def _create_caseroot_tools(self): machines_dir = os.path.abspath(self.get_value("MACHDIR")) toolsdir = os.path.join(self.get_value("CIMEROOT"),"scripts","Tools") # setup executable files in caseroot/ exefiles = (os.path.join(toolsdir, "case.setup"), os.path.join(toolsdir, "case.build"), os.path.join(toolsdir, "case.submit"), os.path.join(toolsdir, "preview_namelists"), os.path.join(toolsdir, "check_input_data"), os.path.join(toolsdir, "check_case"), os.path.join(toolsdir, "archive_metadata.sh"), os.path.join(toolsdir, "xmlchange"), os.path.join(toolsdir, "xmlquery")) try: for exefile in exefiles: destfile = os.path.join(self._caseroot,os.path.basename(exefile)) os.symlink(exefile, destfile) except Exception as e: logger.warning("FAILED to set up exefiles: %s" % str(e)) # set up utility files in caseroot/Tools/ toolfiles = (os.path.join(toolsdir, "check_lockedfiles"), os.path.join(toolsdir, "lt_archive.sh"), os.path.join(toolsdir, "getTiming"), os.path.join(toolsdir, "save_provenance"), os.path.join(machines_dir,"Makefile"), os.path.join(machines_dir,"mkSrcfiles"), os.path.join(machines_dir,"mkDepends")) for toolfile in toolfiles: destfile = os.path.join(self._caseroot,"Tools",os.path.basename(toolfile)) expect(os.path.isfile(toolfile)," File %s does not exist"%toolfile) try: os.symlink(toolfile, destfile) except Exception as e: logger.warning("FAILED to set up toolfiles: %s %s %s" % (str(e), toolfile, destfile)) # Create Macros file. machine = self.get_value("MACH") files = Files() # Use config_build if the environment variable is set, or if there is no # config_compilers file. if os.getenv("CIME_USE_CONFIG_BUILD") == "TRUE" or \ files.get_value("COMPILERS_SPEC_FILE") is None: build_file = files.get_value("BUILD_SPEC_FILE") machobj = Machines(machine=machine, files=files) macro_maker = Build(machobj) macros_path = os.path.join(self._caseroot, "Macros") with open(macros_path, "w") as macros_file: macro_maker.write_macros('Makefile', build_file, macros_file) # Copy any system or compiler Depends files to the case. compiler = self.get_value("COMPILER") for dep in (machine, compiler): dfile = "Depends.%s"%dep if os.path.isfile(os.path.join(machines_dir,dfile)): shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(self._caseroot,dfile)) dfile = "Depends.%s.%s"%(machine,compiler) if os.path.isfile(os.path.join(machines_dir,dfile)): shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(self._caseroot, dfile))
def _set_compset_and_pesfile(self, compset_name, user_compset=False, pesfile=None): """ Loop through all the compset files and find the compset specifation file that matches either the input 'compset_name'. Note that the input compset name (i.e. compset_name) can be either a longname or an alias. This will also set the compsets and pes specfication files. """ files = Files() components = files.get_components("COMPSETS_SPEC_FILE") logger.debug(" Possible components for COMPSETS_SPEC_FILE are %s" % components) # Loop through all of the files listed in COMPSETS_SPEC_FILE and find the file # that has a match for either the alias or the longname in that order for component in components: # Determine the compsets file for this component compsets_filename = files.get_value("COMPSETS_SPEC_FILE", {"component":component}) pes_filename = files.get_value("PES_SPEC_FILE" , {"component":component}) tests_filename = files.get_value("TESTS_SPEC_FILE" , {"component":component}, resolved=False) tests_mods_dir = files.get_value("TESTS_MODS_DIR" , {"component":component}, resolved=False) user_mods_dir = files.get_value("USER_MODS_DIR" , {"component":component}, resolved=False) # If the file exists, read it and see if there is a match for the compset alias or longname if (os.path.isfile(compsets_filename)): compsets = Compsets(compsets_filename) match = compsets.get_compset_match(name=compset_name) if match is not None: self._pesfile = pes_filename self._compsetsfile = compsets_filename self._compsetname = match self.set_value("COMPSETS_SPEC_FILE" , files.get_value("COMPSETS_SPEC_FILE", {"component":component}, resolved=False)) self.set_value("TESTS_SPEC_FILE" , tests_filename) self.set_value("TESTS_MODS_DIR" , tests_mods_dir) self.set_value("USER_MODS_DIR" , user_mods_dir) self.set_value("PES_SPEC_FILE" , files.get_value("PES_SPEC_FILE" , {"component":component}, resolved=False)) logger.info("Compset longname is %s " %(match)) logger.info("Compset specification file is %s" %(compsets_filename)) logger.info("Pes specification file is %s" %(pes_filename)) return if user_compset is True: #Do not error out for user_compset logger.warn("Could not find a compset match for either alias or longname in %s" %(compset_name)) self._compsetname = compset_name self._pesfile = pesfile self.set_value("PES_SPEC_FILE", pesfile) else: expect(False, "Could not find a compset match for either alias or longname in %s" %(compset_name))
def __init__(self, batch_system=None, machine=None, infile=None, files=None): """ initialize an object """ if files is None: files = Files() if infile is None: infile = files.get_value("BATCH_SPEC_FILE") schema = files.get_schema("BATCH_SPEC_FILE") GenericXML.__init__(self, infile, schema=schema) self.batch_system_node = None self.machine_node = None self.batch_system = batch_system self.machine = machine #Append the contents of $HOME/.cime/config_batch.xml if it exists #This could cause problems if node matchs are repeated when only one is expected infile = os.path.join(os.environ.get("HOME"),".cime","config_batch.xml") if os.path.exists(infile): GenericXML.read(self, infile) if self.batch_system is not None: self.set_batch_system(self.batch_system, machine=machine)
def __init__(self, infile=None, files=None): if infile is None: if files is None: files = Files() infile = files.get_value("PIO_SPEC_FILE") EntryID.__init__(self, infile)
def setup(self, env_archive, components, files=None): if files is None: files = Files() components_node = ET.Element("components") components_node.set("version", "2.0") model = get_model() if 'cpl' not in components: components.append('cpl') if 'dart' not in components and model == 'cesm': components.append('dart') for comp in components: infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp}) if infile is not None and os.path.isfile(infile): arch = Archive(infile=infile, files=files) specs = arch.get_node("comp_archive_spec", {"compname":comp}) else: if infile is None: logger.debug("No archive file defined for component %s"%comp) else: logger.debug("Archive file %s for component %s not found"%(infile,comp)) specs = self.get_optional_node("comp_archive_spec", attributes={"compname":comp}) if specs is None: logger.debug("No archive specs found for component %s"%comp) else: logger.debug("adding archive spec for %s"%comp) components_node.append(specs) env_archive.add_child(components_node)
def saveLogs(case, lid): ############################################################################### logdir = case.get_value("LOGDIR") if logdir is not None and len(logdir) > 0: if not os.path.isdir(logdir): os.makedirs(logdir) caseroot = case.get_value("CASEROOT") rundir = case.get_value("RUNDIR") # get components files = Files() config_file = files.get_value("CONFIG_DRV_FILE") component = Component(config_file) comps = [x.lower() for x in component.get_valid_model_components()] comps = [x.replace('drv', 'cpl') for x in comps] model = [case.get_value("MODEL")] comps = comps + model # for each component, compress log files and copy to logdir for comp in comps: logfile = os.path.join(rundir, comp + '.log.' + lid) if os.path.isfile(logfile): f_in = open(logfile) f_out = gzip.open(logfile + '.gz', 'wb') f_out.writelines(f_in) f_out.close() f_in.close() os.remove(logfile) logfile_copy = logfile + '.gz' shutil.copy(logfile_copy, os.path.join(caseroot, logdir, os.path.basename(logfile_copy)))
def __init__(self, infile=None, files=None, machine=None): """ initialize an object if a filename is provided it will be used, otherwise if a files object is provided it will be used otherwise create a files object from default values """ self.machine_node = None self.machine = None self.machines_dir = None if infile is None: if files is None: files = Files() infile = files.get_value("MACHINES_SPEC_FILE", resolved=False) infile = files.get_resolved_value(infile) self.machines_dir = os.path.dirname(infile) GenericXML.__init__(self, infile) # Append the contents of $HOME/.cime/config_machines.xml if it exists # This could cause problems if node matchs are repeated when only one is expected local_infile = os.path.join(os.environ.get("HOME"), ".cime", "config_machines.xml") logger.debug("Infile: %s", local_infile) if os.path.exists(local_infile): GenericXML.read(self, local_infile) if machine is None: machine = self.probe_machine_name() expect(machine is not None, "Could not initialize machine object from %s or %s" % (infile, local_infile)) self.set_machine(machine)
def __init__(self, compiler=None, machine=None, os_= None, mpilib=None, infile=None, files=None): """ initialize an object """ if infile is None: if files is None: files = Files() infile = files.get_value("COMPILERS_SPEC_FILE") GenericXML.__init__(self, infile) self.machine = machine self.os = os_ self.mpilib = mpilib self.compiler_nodes = None # Listed from last to first self.compiler = compiler if self.compiler is not None: self.set_compiler(compiler) #Append the contents of $HOME/.cime/config_compilers.xml if it exists #This could cause problems if node matchs are repeated when only one is expected infile = os.path.join(os.environ.get("HOME"),".cime","config_compilers.xml") if os.path.exists(infile): GenericXML.read(self, infile)
def setup(self, env_archive, components, files=None): if files is None: files = Files() components_node = env_archive.make_child("components", attributes={"version":"2.0"}) model = get_model() if 'drv' not in components: components.append('drv') if 'dart' not in components and model == 'cesm': components.append('dart') for comp in components: infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp}) if infile is not None and os.path.isfile(infile): arch = Archive(infile=infile, files=files) specs = arch.get_optional_child(name="comp_archive_spec", attributes={"compname":comp}) else: if infile is None: logger.debug("No archive file defined for component {}".format(comp)) else: logger.debug("Archive file {} for component {} not found".format(infile,comp)) specs = self.get_optional_child(name="comp_archive_spec", attributes={"compname":comp}) if specs is None: logger.debug("No archive specs found for component {}".format(comp)) else: logger.debug("adding archive spec for {}".format(comp)) env_archive.add_child(specs, root=components_node)
def _create_newcase_phase(self, test): ########################################################################### test_dir = self._get_test_dir(test) test_case, case_opts, grid, compset, machine, compiler, test_mods = CIME.utils.parse_test_name(test) if (compiler != self._compiler): raise StandardError("Test '%s' has compiler that does not match instance compliler '%s'" % (test, self._compiler)) if (self._parallel_jobs == 1): scratch_dir = self._machobj.get_value("CESMSCRATCHROOT") if (self._project is not None): scratch_dir = scratch_dir.replace("$PROJECT", self._project) sharedlibroot = os.path.join(scratch_dir, "sharedlibroot.%s" % self._test_id) else: # Parallelizing builds introduces potential sync problems with sharedlibroot # Just let every case build it's own sharedlibroot = os.path.join(test_dir, "sharedlibroot.%s" % self._test_id) create_newcase_cmd = "%s -model %s -case %s -res %s -mach %s -compiler %s -compset %s -testname %s -project %s -sharedlibroot %s" % \ (os.path.join(self._cime_root,"scripts", "create_newcase"), self._cime_model,test_dir, grid, machine, compiler, compset, test_case, self._project, sharedlibroot) if (test_case != 'PFS'): create_newcase_cmd += " -nosavetiming " if (case_opts is not None): create_newcase_cmd += " -confopts _%s" % ("_".join(case_opts)) if (test_mods is not None): files = Files() (component, mods) = test_mods.split('/') testmods_dir = files.get_value("TESTS_MODS_DIR",{"component": component}) test_mod_file = os.path.join(testmods_dir, component, mods) if (not os.path.exists(test_mod_file)): self._log_output(test, "Missing testmod file '%s'" % test_mod_file) return False create_newcase_cmd += " -user_mods_dir %s" % test_mod_file logging.info("Calling create_newcase: "+create_newcase_cmd) return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)
def __init__(self, batch_system=None, machine=None, infile=None, files=None): """ initialize an object """ if files is None: files = Files() if infile is None: infile = files.get_value("BATCH_SPEC_FILE") schema = files.get_schema("BATCH_SPEC_FILE") GenericXML.__init__(self, infile, schema=schema) self.batch_system_node = None self.machine_node = None self.batch_system = batch_system self.machine = machine #Append the contents of $HOME/.cime/config_batch.xml if it exists #This could cause problems if node matchs are repeated when only one is expected infile = os.path.join(os.environ.get("HOME"), ".cime", "config_batch.xml") if os.path.exists(infile): GenericXML.read(self, infile) if self.batch_system is not None: self.set_batch_system(self.batch_system, machine=machine)
def __init__(self, compiler=None, machine=None, os_=None, mpilib=None, infile=None, files=None): """ initialize an object """ if infile is None: if files is None: files = Files() infile = files.get_value("COMPILERS_SPEC_FILE") GenericXML.__init__(self, infile) self.machine = machine self.os = os_ self.mpilib = mpilib self.compiler_nodes = None # Listed from last to first self.compiler = compiler #Append the contents of $HOME/.cime/config_compilers.xml if it exists #This could cause problems if node matchs are repeated when only one is expected infile = os.path.join(os.environ.get("HOME"), ".cime", "config_compilers.xml") if os.path.exists(infile): GenericXML.read(self, infile) if self.compiler is not None: self.set_compiler(compiler)
def __init__(self, infile=None, files=None): """ initialize an object interface to file config_tests.xml """ if infile is None: if files is None: files = Files() infile = files.get_value("CONFIG_TESTS_FILE") GenericXML.__init__(self, infile) # append any component specific config_tests.xml files for comp in files.get_components("CONFIG_TESTS_FILE"): if comp is None: continue infile = files.get_value("CONFIG_TESTS_FILE", attribute={"component":comp}) if os.path.isfile(infile): self.read(infile)
def __init__(self, infile=None, files=None): if infile is None: if files is None: files = Files() infile = files.get_value("GRIDS_SPEC_FILE") logger.debug(" Grid specification file is %s" % infile) GenericXML.__init__(self, infile)
def __init__(self, infile=None): """ initialize an object """ if infile is None: files = Files() infile = files.get_value("CONFIG_DRV_FILE") EntryID.__init__(self,infile)
def __init__(self, infile=None, files=None): """ initialize an object interface to file config_tests.xml """ if infile is None: if files is None: files = Files() infile = files.get_value("CONFIG_TESTS_FILE") GenericXML.__init__(self, infile)
def _create_newcase_phase(self, test): ########################################################################### test_dir = self._get_test_dir(test) _, case_opts, grid, compset,\ machine, compiler, test_mods = CIME.utils.parse_test_name(test) create_newcase_cmd = "%s --case %s --res %s --mach %s --compiler %s --compset %s"\ " --test" % \ (os.path.join(self._cime_root, "scripts", "create_newcase"), test_dir, grid, machine, compiler, compset) if self._project is not None: create_newcase_cmd += " --project %s " % self._project if self._output_root is not None: create_newcase_cmd += " --output-root %s " % self._output_root if test_mods is not None: files = Files() (component,modspath) = test_mods.split('/',1) testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component}) test_mod_file = os.path.join(testmods_dir, component, modspath) if not os.path.exists(test_mod_file): self._log_output(test, "Missing testmod file '%s'" % test_mod_file) return False create_newcase_cmd += " --user-mods-dir %s" % test_mod_file if case_opts is not None: for case_opt in case_opts: # pylint: disable=not-an-iterable if case_opt.startswith('M'): mpilib = case_opt[1:] create_newcase_cmd += " --mpilib %s" % mpilib logger.debug (" MPILIB set to %s" % mpilib) if case_opt.startswith('N'): ninst = case_opt[1:] create_newcase_cmd += " --ninst %s" %ninst logger.debug (" NINST set to %s" % ninst) if case_opt.startswith('P'): pesize = case_opt[1:] create_newcase_cmd += " --pecount %s"%pesize if self._queue is not None: create_newcase_cmd += " --queue=%s" % self._queue if self._walltime is not None: create_newcase_cmd += " --walltime %s" % self._walltime elif test in self._test_data and "options" in self._test_data[test] and \ "wallclock" in self._test_data[test]['options']: create_newcase_cmd += " --walltime %s" % self._test_data[test]['options']['wallclock'] logger.debug("Calling create_newcase: " + create_newcase_cmd) return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)
def __init__(self,infile=None): """ initialize an object >>> files = Files() >>> files.get_value('CASEFILE_HEADERS',resolved=False) '$CIMEROOT/config/config_headers.xml' """ if infile is None: files = Files() infile = files.get_value('CASEFILE_HEADERS', resolved=True) super(Headers, self).__init__(infile)
def __init__(self, infile=None, files=None): if files is None: files = Files() if infile is None: infile = files.get_value("GRIDS_SPEC_FILE") logger.debug(" Grid specification file is %s" % infile) schema = files.get_schema("GRIDS_SPEC_FILE") GenericXML.__init__(self, infile, schema) self._version = self.get_version() self._comp_gridnames = self._get_grid_names()
def _get_component_config_data(self): # attributes used for multi valued defaults ($attlist is a hash reference) attlist = {"compset":self._compsetname, "grid":self._gridname} # Determine list of component classes that this coupler/driver knows how # to deal with. This list follows the same order as compset longnames follow. files = Files() # Add the group and elements for the config_files.xml for env_file in self._env_entryid_files: env_file.add_elements_by_group(files, attlist) drv_config_file = files.get_value("CONFIG_DRV_FILE") drv_comp = Component(drv_config_file) for env_file in self._env_entryid_files: env_file.add_elements_by_group(drv_comp, attributes=attlist) # loop over all elements of both component_classes and components - and get config_component_file for # for each component self._component_classes =drv_comp.get_valid_model_components() if len(self._component_classes) > len(self._components): self._components.append('sesp') for i in xrange(1,len(self._component_classes)): comp_class = self._component_classes[i] comp_name = self._components[i-1] node_name = 'CONFIG_' + comp_class + '_FILE' # Add the group and elements for the config_files.xml comp_config_file = files.get_value(node_name, {"component":comp_name}, resolved=False) self.set_value(node_name, comp_config_file) comp_config_file = self.get_resolved_value(comp_config_file) expect(comp_config_file is not None,"No config file for component %s"%comp_name) compobj = Component(comp_config_file) for env_file in self._env_entryid_files: env_file.add_elements_by_group(compobj, attributes=attlist) for key,value in self.lookups.items(): result = self.set_value(key,value) if result is not None: del self.lookups[key]
def __init__(self, machine, files=None, infile=None): """ initialize an object """ if (infile is None): if files is None: files = Files() infile = files.get_value("LTARCHIVE_SPEC_FILE", resolved=False) infile = files.get_resolved_value(infile) GenericXML.__init__(self, infile) self.machine = machine
def __init__(self, infile=None, files=None): """ initialize a files object given input pes specification file """ if files is None: files = Files() if infile is None: infile = files.get_value("INPUTDATA_SPEC_FILE") schema = files.get_schema("INPUTDATA_SPEC_FILE") logger.debug("DEBUG: infile is {}".format(infile)) GenericXML.__init__(self, infile, schema=schema) self._servernode = None
def __init__(self, machobj, infile=None, compiler=None, mpilib=None, files=None, version=None): """ initialize an object """ if infile is None: if files is None: files = Files() infile = files.get_value("COMPILERS_SPEC_FILE") schema = files.get_schema("COMPILERS_SPEC_FILE") GenericXML.__init__(self, infile, schema) self._machobj = machobj if version is not None: # this is used in scripts_regression_tests to force version 2, it should not be used otherwise self._version = version else: self._version = self.get_version() self.machine = machobj.get_machine_name() self.os = machobj.get_value("OS") if compiler is None: compiler = machobj.get_default_compiler() self.compiler = compiler if mpilib is None: if compiler is None: mpilib = machobj.get_default_MPIlib() else: mpilib = machobj.get_default_MPIlib(attributes={'compiler':compiler}) self.mpilib = mpilib self.compiler_nodes = None # Listed from last to first #Append the contents of $HOME/.cime/config_compilers.xml if it exists #This could cause problems if node matchs are repeated when only one is expected infile = os.path.join(os.environ.get("HOME"),".cime","config_compilers.xml") if os.path.exists(infile): GenericXML.read(self, infile) if self.compiler is not None: self.set_compiler(compiler) if self._version > 1.0: schema_db = GenericXML(infile=schema) compiler_vars = schema_db.get_child("{http://www.w3.org/2001/XMLSchema}group", attributes={"name":"compilerVars"}) choice = schema_db.get_child(name="{http://www.w3.org/2001/XMLSchema}choice", root=compiler_vars) self.flag_vars = set(schema_db.get(elem, "name") for elem in schema_db.get_children(root=choice, attributes={"type":"flagsVar"}))
def __init__(self, machobj, infile=None, compiler=None, mpilib=None, files=None, version=None): """ initialize an object """ if infile is None: if files is None: files = Files() infile = files.get_value("COMPILERS_SPEC_FILE") schema = files.get_schema("COMPILERS_SPEC_FILE") GenericXML.__init__(self, infile, schema) self._machobj = machobj if version is not None: # this is used in scripts_regression_tests to force version 2, it should not be used otherwise self._version = version else: self._version = self.get_version() self.machine = machobj.get_machine_name() self.os = machobj.get_value("OS") if mpilib is None: mpilib = machobj.get_default_MPIlib() self.mpilib = mpilib if compiler is None: compiler = machobj.get_default_compiler() self.compiler = compiler self.compiler_nodes = None # Listed from last to first #Append the contents of $HOME/.cime/config_compilers.xml if it exists #This could cause problems if node matchs are repeated when only one is expected infile = os.path.join(os.environ.get("HOME"),".cime","config_compilers.xml") if os.path.exists(infile): GenericXML.read(self, infile) if self.compiler is not None: self.set_compiler(compiler) if self._version > 1.0: # Run an XPath query to extract the list of flag variable names. ns = {"xs": "http://www.w3.org/2001/XMLSchema"} flag_xpath = ".//xs:group[@name='compilerVars']/xs:choice/xs:element[@type='flagsVar']" flag_elems = ET.parse(schema).getroot().findall(flag_xpath, ns) self.flag_vars = set(elem.get('name') for elem in flag_elems)
def _create_newcase_phase(self, test): ########################################################################### test_dir = self._get_test_dir(test) _, case_opts, grid, compset,\ machine, compiler, test_mods = CIME.utils.parse_test_name(test) if compiler != self._compiler: raise StandardError("Test '%s' has compiler that does" " not match instance compliler '%s'" % (test, self._compiler)) create_newcase_cmd = "%s --case %s --res %s --mach %s --compiler %s --compset %s"\ " --project %s --test"%\ (os.path.join(self._cime_root, "scripts", "create_newcase"), test_dir, grid, machine, compiler, compset, self._project) if test_mods is not None: files = Files() (component,modspath) = test_mods.split('/',1) testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component}) test_mod_file = os.path.join(testmods_dir, component, modspath) if not os.path.exists(test_mod_file): self._log_output(test, "Missing testmod file '%s'" % test_mod_file) return False create_newcase_cmd += " --user-mods-dir %s" % test_mod_file if case_opts is not None: for case_opt in case_opts: if case_opt.startswith('M'): mpilib = case_opt[1:] create_newcase_cmd += " --mpilib %s" % mpilib logger.debug (" MPILIB set to %s" % mpilib) if case_opt.startswith('N'): ninst = case_opt[1:] create_newcase_cmd += " --ninst %s" %ninst logger.debug (" NINST set to %s" % ninst) pesize = re.match('P([SMLX][12]?)', case_opt) if pesize: create_newcase_cmd += " --pecount %s"%pesize.group(1) logger.debug("Calling create_newcase: " + create_newcase_cmd) return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)
def __init__(self, infile=None, files=None, machine=None): """ initialize an object if a filename is provided it will be used, otherwise if a files object is provided it will be used otherwise create a files object from default values """ self.machine_node = None self.machine = None self.machines_dir = None schema = None if files is None: files = Files() if infile is None: infile = files.get_value("MACHINES_SPEC_FILE") schema = files.get_schema("MACHINES_SPEC_FILE") logger.debug("Verifying using schema {}".format(schema)) self.machines_dir = os.path.dirname(infile) GenericXML.__init__(self, infile, schema) # Append the contents of $HOME/.cime/config_machines.xml if it exists # This could cause problems if node matchs are repeated when only one is expected local_infile = os.path.join(os.environ.get("HOME"),".cime","config_machines.xml") logger.debug("Infile: {}".format(local_infile)) if os.path.exists(local_infile): GenericXML.read(self, local_infile, schema) if machine is None: if "CIME_MACHINE" in os.environ: machine = os.environ["CIME_MACHINE"] else: cime_config = get_cime_config() if cime_config.has_option("main", "machine"): machine = cime_config.get("main", "machine") if machine is None: machine = self.probe_machine_name() expect(machine is not None, "Could not initialize machine object from {} or {}".format(infile, local_infile)) self.set_machine(machine)
def get_tests_from_xml(xml_machine=None,xml_category=None,xml_compiler=None, xml_testlist=None, machine=None, compiler=None): """ Parse testlists for a list of tests """ listoftests = [] testlistfiles = [] if(machine is not None): thismach=machine if(compiler is not None): thiscompiler = compiler if(xml_testlist is not None): expect(os.path.isfile(xml_testlist), "Testlist not found or not readable "+xml_testlist) testlistfiles.append(xml_testlist) else: files = Files() comps = files.get_components("TESTS_SPEC_FILE") for comp in comps: test_spec_file = files.get_value("TESTS_SPEC_FILE", {"component":comp}) if(os.path.isfile(test_spec_file)): testlistfiles.append(test_spec_file) for testlistfile in testlistfiles: thistestlistfile = Testlist(testlistfile) logger.debug("Testlist file is "+testlistfile) logger.debug("xml_machine {} xml_category {} xml_compiler {}".format(xml_machine, xml_category, xml_compiler)) newtests = thistestlistfile.get_tests(xml_machine, xml_category, xml_compiler) for test in newtests: if(machine is None): thismach = test["machine"] if(compiler is None): thiscompiler = test["compiler"] test["name"] = CIME.utils.get_full_test_name(test["testname"], grid=test["grid"], compset=test["compset"], machine=thismach, compiler=thiscompiler, testmod=None if "testmods" not in test else test["testmods"]) logger.debug("Adding test {} with compiler {}".format(test["name"], test["compiler"])) listoftests += newtests logger.debug("Found {:d} tests".format(len(listoftests))) return listoftests
def _create_newcase_phase(self, test): ########################################################################### test_dir = self._get_test_dir(test) test_case, case_opts, grid, compset,\ machine, compiler, test_mods = CIME.utils.parse_test_name(test) if compiler != self._compiler: raise StandardError("Test '%s' has compiler that does" " not match instance compliler '%s'" % (test, self._compiler)) scratch_dir = self._machobj.get_value("CESMSCRATCHROOT") if self._project is not None: scratch_dir = scratch_dir.replace("$PROJECT", self._project) sharedlibroot = os.path.join(scratch_dir, "sharedlibroot.%s" % self._test_id) create_newcase_cmd = "%s -model %s -case %s -res %s -mach %s -compiler %s -compset"\ " %s -testname %s -project %s -sharedlibroot %s" % \ (os.path.join(self._cime_root, "scripts", "create_newcase"), self._cime_model, test_dir, grid, machine, compiler, compset, test_case, self._project, sharedlibroot) if test_case != 'PFS': create_newcase_cmd += " -nosavetiming " if case_opts is not None: create_newcase_cmd += " -confopts _%s" % ("_".join(case_opts)) if test_mods is not None: files = Files() (component,modspath) = test_mods.split('/',1) testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component}) test_mod_file = os.path.join(testmods_dir, component, modspath) if not os.path.exists(test_mod_file): self._log_output(test, "Missing testmod file '%s'" % test_mod_file) return False create_newcase_cmd += " -user_mods_dir %s" % test_mod_file logger.debug("Calling create_newcase: "+create_newcase_cmd) return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)
def get_components(self): """ return dictionary of the form [component_class:component], e.g. [atm:cam], for all compset components """ files = Files() drv_comp = Component(files.get_value("CONFIG_DRV_FILE")) # Determine list of component classes that this coupler/driver knows how # to deal with. This list follows the same order as compset longnames follow. component_classes = drv_comp.get_valid_model_components() components = self.get_compset_components() # Note that component classes can have a bigger range than # compents since stub esp (sesp) is an optional component - so # need to take the min of the two below comp_dict = {} for i in xrange(0,len(components)): comp_name = components[i] comp_class = component_classes[i+1] comp_dict[comp_class] = comp_name return comp_dict
def _xml_phase(self, test): ########################################################################### test_case = CIME.utils.parse_test_name(test)[0] envtest = EnvTest(self._get_test_dir(test)) files = Files() drv_config_file = files.get_value("CONFIG_DRV_FILE") logging.info("Found drv_config_file %s" % drv_config_file) drv_comp = Component(drv_config_file) envtest.add_elements_by_group(drv_comp, {}, "env_test.xml") envtest.set_value("TESTCASE", test_case) envtest.set_value("TEST_TESTID", self._test_id) envtest.set_value("CASEBASEID", test) test_argv = "-testname %s -testroot %s" % (test, self._test_root) if (self._generate): test_argv += " -generate %s" % self._baseline_gen_name envtest.set_value("BASELINE_NAME_GEN",self._baseline_gen_name) envtest.set_value("BASEGEN_CASE",os.path.join(self._baseline_gen_name,test)) if (self._compare): test_argv += " -compare %s" % self._baseline_cmp_name envtest.set_value("BASELINE_NAME_CMP",self._baseline_cmp_name) envtest.set_value("BASECMP_CASE",os.path.join(self._baseline_cmp_name,test)) envtest.set_value("TEST_ARGV",test_argv) envtest.set_value("CLEANUP", ("TRUE" if self._clean else "FALSE")) if (self._generate or self._compare): envtest.set_value("BASELINE_ROOT", self._baseline_root) envtest.set_value("GENERATE_BASELINE", "TRUE" if self._generate else "FALSE") envtest.set_value("COMPARE_BASELINE", "TRUE" if self._compare else "FALSE") envtest.set_value("CCSM_CPRNC", self._machobj.get_value("CCSM_CPRNC",resolved=False)) envtest.write() return True
def _xml_phase(self, test): ########################################################################### test_case = CIME.utils.parse_test_name(test)[0] # Create, fill and write an envtest object test_dir = self._get_test_dir(test) envtest = EnvTest(test_dir) # Determine list of component classes that this coupler/driver knows how # to deal with. This list follows the same order as compset longnames follow. files = Files() drv_config_file = files.get_value("CONFIG_CPL_FILE") drv_comp = Component(drv_config_file, "CPL") envtest.add_elements_by_group(files, {}, "env_test.xml") envtest.add_elements_by_group(drv_comp, {}, "env_test.xml") envtest.set_value("TESTCASE", test_case) envtest.set_value("TEST_TESTID", self._test_id) envtest.set_value("CASEBASEID", test) if test in self._test_data and "options" in self._test_data[test] and \ "memleak_tolerance" in self._test_data[test]['options']: envtest.set_value("TEST_MEMLEAK_TOLERANCE", self._test_data[test]['options']['memleak_tolerance']) test_argv = "-testname {} -testroot {}".format(test, self._test_root) if self._baseline_gen_name: test_argv += " -generate {}".format(self._baseline_gen_name) basegen_case_fullpath = os.path.join(self._baseline_root,self._baseline_gen_name, test) logger.debug("basegen_case is {}".format(basegen_case_fullpath)) envtest.set_value("BASELINE_NAME_GEN", self._baseline_gen_name) envtest.set_value("BASEGEN_CASE", os.path.join(self._baseline_gen_name, test)) if self._baseline_cmp_name: test_argv += " -compare {}".format(self._baseline_cmp_name) envtest.set_value("BASELINE_NAME_CMP", self._baseline_cmp_name) envtest.set_value("BASECMP_CASE", os.path.join(self._baseline_cmp_name, test)) envtest.set_value("TEST_ARGV", test_argv) envtest.set_value("CLEANUP", self._clean) envtest.set_value("BASELINE_ROOT", self._baseline_root) envtest.set_value("GENERATE_BASELINE", self._baseline_gen_name is not None) envtest.set_value("COMPARE_BASELINE", self._baseline_cmp_name is not None) envtest.set_value("CCSM_CPRNC", self._machobj.get_value("CCSM_CPRNC", resolved=False)) tput_tolerance = self._machobj.get_value("TEST_TPUT_TOLERANCE", resolved=False) envtest.set_value("TEST_TPUT_TOLERANCE", 0.25 if tput_tolerance is None else tput_tolerance) # Add the test instructions from config_test to env_test in the case config_test = Tests() testnode = config_test.get_test_node(test_case) envtest.add_test(testnode) # Determine the test_case from the test name test_case, case_opts = CIME.utils.parse_test_name(test)[:2] # Determine case_opts from the test_case if case_opts is not None: logger.debug("case_opts are {} ".format(case_opts)) for opt in case_opts: # pylint: disable=not-an-iterable logger.debug("case_opt is {}".format(opt)) if opt == 'D': envtest.set_test_parameter("DEBUG", "TRUE") logger.debug (" DEBUG set to TRUE") elif opt == 'E': envtest.set_test_parameter("USE_ESMF_LIB", "TRUE") logger.debug (" USE_ESMF_LIB set to TRUE") elif opt == 'CG': envtest.set_test_parameter("CALENDAR", "GREGORIAN") logger.debug (" CALENDAR set to {}".format(opt)) elif opt.startswith('L'): match = re.match('L([A-Za-z])([0-9]*)', opt) stop_option = {"y":"nyears", "m":"nmonths", "d":"ndays", "h":"nhours", "s":"nseconds", "n":"nsteps"} opt = match.group(1) envtest.set_test_parameter("STOP_OPTION",stop_option[opt]) opti = match.group(2) envtest.set_test_parameter("STOP_N", opti) logger.debug (" STOP_OPTION set to {}".format(stop_option[opt])) logger.debug (" STOP_N set to {}".format(opti)) elif opt.startswith('R'): # R option is for testing in PTS_MODE or Single Column Model # (SCM) mode envtest.set_test_parameter("PTS_MODE", "TRUE") # For PTS_MODE, compile with mpi-serial envtest.set_test_parameter("MPILIB", "mpi-serial") elif (opt.startswith('I') or # Marker to distinguish tests with same name - ignored opt.startswith('M') or # handled in create_newcase opt.startswith('P') or # handled in create_newcase opt.startswith('N') or # handled in create_newcase opt.startswith('C') or # handled in create_newcase opt.startswith('V')): # handled in create_newcase pass elif opt.startswith('IOP'): logger.warning("IOP test option not yet implemented") else: expect(False, "Could not parse option '{}' ".format(opt)) envtest.write() lock_file("env_run.xml", caseroot=test_dir, newname="env_run.orig.xml") with Case(test_dir, read_only=False) as case: if self._output_root is None: self._output_root = case.get_value("CIME_OUTPUT_ROOT") # if we are running a single test we don't need sharedlibroot if len(self._tests) > 1 and self._cime_model != "e3sm": case.set_value("SHAREDLIBROOT", os.path.join(self._output_root, "sharedlibroot.{}".format(self._test_id))) envtest.set_initial_values(case) case.set_value("TEST", True) case.set_value("SAVE_TIMING", self._save_timing) # Scale back build parallelism on systems with few cores if self._model_build_cost > self._proc_pool: case.set_value("GMAKE_J", self._proc_pool) self._model_build_cost = self._proc_pool return True, ""
def _create_newcase_phase(self, test): ########################################################################### test_dir = self._get_test_dir(test) _, case_opts, grid, compset,\ machine, compiler, test_mods = CIME.utils.parse_test_name(test) create_newcase_cmd = "{} --case {} --res {} --compset {}"\ " --test".format(os.path.join(self._cime_root, "scripts", "create_newcase"), test_dir, grid, compset) if machine is not None: create_newcase_cmd += " --machine {}".format(machine) if compiler is not None: create_newcase_cmd += " --compiler {}".format(compiler) if self._project is not None: create_newcase_cmd += " --project {} ".format(self._project) if self._output_root is not None: create_newcase_cmd += " --output-root {} ".format(self._output_root) if self._input_dir is not None: create_newcase_cmd += " --input-dir {} ".format(self._input_dir) if self._pesfile is not None: create_newcase_cmd += " --pesfile {} ".format(self._pesfile) if test_mods is not None: files = Files() (component, modspath) = test_mods.split('/',1) testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component}) test_mod_file = os.path.join(testmods_dir, component, modspath) if not os.path.exists(test_mod_file): error = "Missing testmod file '{}'".format(test_mod_file) self._log_output(test, error) return False, error create_newcase_cmd += " --user-mods-dir {}".format(test_mod_file) mpilib = None ninst = 1 ncpl = 1 if case_opts is not None: for case_opt in case_opts: # pylint: disable=not-an-iterable if case_opt.startswith('M'): mpilib = case_opt[1:] create_newcase_cmd += " --mpilib {}".format(mpilib) logger.debug (" MPILIB set to {}".format(mpilib)) elif case_opt.startswith('N'): expect(ncpl == 1,"Cannot combine _C and _N options") ninst = case_opt[1:] create_newcase_cmd += " --ninst {}".format(ninst) logger.debug (" NINST set to {}".format(ninst)) elif case_opt.startswith('C'): expect(ninst == 1,"Cannot combine _C and _N options") ncpl = case_opt[1:] create_newcase_cmd += " --ninst {} --multi-driver" .format(ncpl) logger.debug (" NCPL set to {}" .format(ncpl)) elif case_opt.startswith('P'): pesize = case_opt[1:] create_newcase_cmd += " --pecount {}".format(pesize) elif case_opt.startswith('V'): driver = case_opt[1:] create_newcase_cmd += " --driver {}".format(driver) # create_test mpilib option overrides default but not explicitly set case_opt mpilib if mpilib is None and self._mpilib is not None: create_newcase_cmd += " --mpilib {}".format(self._mpilib) logger.debug (" MPILIB set to {}".format(self._mpilib)) if self._queue is not None: create_newcase_cmd += " --queue={}".format(self._queue) if self._walltime is not None: create_newcase_cmd += " --walltime {}".format(self._walltime) else: # model specific ways of setting time if self._cime_model == "e3sm": recommended_time = _get_time_est(test, self._baseline_root) if recommended_time is not None: create_newcase_cmd += " --walltime {}".format(recommended_time) else: if test in self._test_data and "options" in self._test_data[test] and \ "wallclock" in self._test_data[test]['options']: create_newcase_cmd += " --walltime {}".format(self._test_data[test]['options']['wallclock']) logger.debug("Calling create_newcase: " + create_newcase_cmd) return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)
def _xml_phase(self, test): ########################################################################### test_case = CIME.utils.parse_test_name(test)[0] # Create, fill and write an envtest object test_dir = self._get_test_dir(test) envtest = EnvTest(test_dir) # Determine list of component classes that this coupler/driver knows how # to deal with. This list follows the same order as compset longnames follow. files = Files() drv_config_file = files.get_value("CONFIG_DRV_FILE") drv_comp = Component(drv_config_file) envtest.add_elements_by_group(files, {}, "env_test.xml") envtest.add_elements_by_group(drv_comp, {}, "env_test.xml") envtest.set_value("TESTCASE", test_case) envtest.set_value("TEST_TESTID", self._test_id) envtest.set_value("CASEBASEID", test) if test in self._test_data and "options" in self._test_data[test] and \ "memleak_tolerance" in self._test_data[test]['options']: envtest.set_value("TEST_MEMLEAK_TOLERANCE", self._test_data[test]['options']['memleak_tolerance']) test_argv = "-testname %s -testroot %s" % (test, self._test_root) if self._baseline_gen_name: test_argv += " -generate %s" % self._baseline_gen_name basegen_case_fullpath = os.path.join(self._baseline_root,self._baseline_gen_name, test) logger.debug("basegen_case is %s"%basegen_case_fullpath) envtest.set_value("BASELINE_NAME_GEN", self._baseline_gen_name) envtest.set_value("BASEGEN_CASE", os.path.join(self._baseline_gen_name, test)) if self._baseline_cmp_name: test_argv += " -compare %s" % self._baseline_cmp_name envtest.set_value("BASELINE_NAME_CMP", self._baseline_cmp_name) envtest.set_value("BASECMP_CASE", os.path.join(self._baseline_cmp_name, test)) envtest.set_value("TEST_ARGV", test_argv) envtest.set_value("CLEANUP", self._clean) if self._baseline_gen_name or self._baseline_cmp_name: envtest.set_value("BASELINE_ROOT", self._baseline_root) envtest.set_value("GENERATE_BASELINE", self._baseline_gen_name is not None) envtest.set_value("COMPARE_BASELINE", self._baseline_cmp_name is not None) envtest.set_value("CCSM_CPRNC", self._machobj.get_value("CCSM_CPRNC", resolved=False)) # Add the test instructions from config_test to env_test in the case config_test = Tests() testnode = config_test.get_test_node(test_case) envtest.add_test(testnode) # Determine the test_case from the test name test_case, case_opts = CIME.utils.parse_test_name(test)[:2] # Determine case_opts from the test_case if case_opts is not None: logger.debug("case_opts are %s " %case_opts) for opt in case_opts: logger.debug("case_opt is %s" %opt) if opt == 'D': envtest.set_test_parameter("DEBUG", "TRUE") logger.debug (" DEBUG set to TRUE") elif opt == 'E': envtest.set_test_parameter("USE_ESMF_LIB", "TRUE") envtest.set_test_parameter("COMP_INTERFACE", "ESMF") logger.debug (" USE_ESMF_LIB set to TRUE") logger.debug (" COMP_INTERFACE set to ESMF") elif opt == 'CG': envtest.set_test_parameter("CALENDAR", "GREGORIAN") logger.debug (" CALENDAR set to %s" %opt) elif opt.startswith('L'): match = re.match('L([A-Za-z])([0-9]*)', opt) stop_option = {"y":"nyears", "m":"nmonths", "d":"ndays", "h":"nhours", "s":"nseconds", "n":"nsteps"} opt = match.group(1) envtest.set_test_parameter("STOP_OPTION",stop_option[opt]) opti = match.group(2) envtest.set_test_parameter("STOP_N", opti) logger.debug (" STOP_OPTION set to %s" %stop_option[opt]) logger.debug (" STOP_N set to %s" %opti) elif opt.startswith('M'): # M option handled by create newcase continue elif opt.startswith('P'): # P option handled by create newcase continue elif opt.startswith('N'): # handled in create_newcase continue elif opt.startswith('IOP'): logger.warn("IOP test option not yet implemented") else: expect(False, "Could not parse option '%s' " %opt) envtest.write() lockedfiles = os.path.join(test_dir, "LockedFiles") if not os.path.exists(lockedfiles): os.mkdir(lockedfiles) shutil.copy(os.path.join(test_dir,"env_run.xml"), os.path.join(lockedfiles, "env_run.orig.xml")) with Case(test_dir, read_only=False) as case: case.set_value("SHAREDLIBROOT", os.path.join(self._test_root, "sharedlibroot.%s"%self._test_id)) envtest.set_initial_values(case) if self._save_timing: case.set_value("SAVE_TIMING", True) return True