Esempio n. 1
0
    def setup(self, env_archive, components, files=None):
        if files is None:
            files = Files()

        components_node = env_archive.make_child("components", attributes={"version":"2.0"})

        model = get_model()
        if 'drv' not in components:
            components.append('drv')
        if 'dart' not in components and model == 'cesm':
            components.append('dart')

        for comp in components:
            infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp})

            if infile is not None and os.path.isfile(infile):
                arch = Archive(infile=infile, files=files)
                specs = arch.get_optional_child(name="comp_archive_spec", attributes={"compname":comp})
            else:
                if infile is None:
                    logger.debug("No archive file defined for component {}".format(comp))
                else:
                    logger.debug("Archive file {} for component {} not found".format(infile,comp))

                specs = self.get_optional_child(name="comp_archive_spec", attributes={"compname":comp})

            if specs is None:
                logger.debug("No archive specs found for component {}".format(comp))
            else:
                logger.debug("adding archive spec for {}".format(comp))
                env_archive.add_child(specs, root=components_node)
Esempio n. 2
0
    def __init__(self,
                 compiler=None,
                 machine=None,
                 os_=None,
                 mpilib=None,
                 infile=None,
                 files=None):
        """
        initialize an object
        """
        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("COMPILERS_SPEC_FILE")

        GenericXML.__init__(self, infile)

        self.machine = machine
        self.os = os_
        self.mpilib = mpilib
        self.compiler_nodes = None  # Listed from last to first
        self.compiler = compiler
        #Append the contents of $HOME/.cime/config_compilers.xml if it exists
        #This could cause problems if node matchs are repeated when only one is expected
        infile = os.path.join(os.environ.get("HOME"), ".cime",
                              "config_compilers.xml")
        if os.path.exists(infile):
            GenericXML.read(self, infile)

        if self.compiler is not None:
            self.set_compiler(compiler)
Esempio n. 3
0
    def setup(self, env_archive, components, files=None):
        if files is None:
            files = Files()

        components_node = env_archive.make_child("components", attributes={"version":"2.0"})

        model = get_model()
        if 'drv' not in components:
            components.append('drv')
        if 'dart' not in components and model == 'cesm':
            components.append('dart')

        for comp in components:
            infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp})

            if infile is not None and os.path.isfile(infile):
                arch = Archive(infile=infile, files=files)
                specs = arch.get_child(name="comp_archive_spec", attributes={"compname":comp})
            else:
                if infile is None:
                    logger.debug("No archive file defined for component {}".format(comp))
                else:
                    logger.debug("Archive file {} for component {} not found".format(infile,comp))

                specs = self.get_optional_child(name="comp_archive_spec", attributes={"compname":comp})

            if specs is None:
                logger.debug("No archive specs found for component {}".format(comp))
            else:
                logger.debug("adding archive spec for {}".format(comp))
                env_archive.add_child(specs, root=components_node)
Esempio n. 4
0
File: case.py Progetto: erika72/cime
    def _create_caseroot_tools(self):
        machines_dir = os.path.abspath(self.get_value("MACHDIR"))
        toolsdir = os.path.join(self.get_value("CIMEROOT"),"scripts","Tools")
        # setup executable files in caseroot/
        exefiles = (os.path.join(toolsdir, "case.setup"),
                    os.path.join(toolsdir, "case.build"),
                    os.path.join(toolsdir, "case.submit"),
                    os.path.join(toolsdir, "preview_namelists"),
                    os.path.join(toolsdir, "check_input_data"),
                    os.path.join(toolsdir, "check_case"),
                    os.path.join(toolsdir, "archive_metadata.sh"),
                    os.path.join(toolsdir, "xmlchange"),
                    os.path.join(toolsdir, "xmlquery"))
        try:
            for exefile in exefiles:
                destfile = os.path.join(self._caseroot,os.path.basename(exefile))
                os.symlink(exefile, destfile)
        except Exception as e:
            logger.warning("FAILED to set up exefiles: %s" % str(e))

        # set up utility files in caseroot/Tools/
        toolfiles = (os.path.join(toolsdir, "check_lockedfiles"),
                     os.path.join(toolsdir, "lt_archive.sh"),
                     os.path.join(toolsdir, "getTiming"),
                     os.path.join(toolsdir, "save_provenance"),
                     os.path.join(machines_dir,"Makefile"),
                     os.path.join(machines_dir,"mkSrcfiles"),
                     os.path.join(machines_dir,"mkDepends"))

        for toolfile in toolfiles:
            destfile = os.path.join(self._caseroot,"Tools",os.path.basename(toolfile))
            expect(os.path.isfile(toolfile)," File %s does not exist"%toolfile)
            try:
                os.symlink(toolfile, destfile)
            except Exception as e:
                logger.warning("FAILED to set up toolfiles: %s %s %s" % (str(e), toolfile, destfile))

        # Create Macros file.
        machine = self.get_value("MACH")
        files = Files()
        # Use config_build if the environment variable is set, or if there is no
        # config_compilers file.
        if os.getenv("CIME_USE_CONFIG_BUILD") == "TRUE" or \
           files.get_value("COMPILERS_SPEC_FILE") is None:
            build_file = files.get_value("BUILD_SPEC_FILE")
            machobj = Machines(machine=machine, files=files)
            macro_maker = Build(machobj)
            macros_path = os.path.join(self._caseroot, "Macros")
            with open(macros_path, "w") as macros_file:
                macro_maker.write_macros('Makefile', build_file, macros_file)

        # Copy any system or compiler Depends files to the case.
        compiler = self.get_value("COMPILER")
        for dep in (machine, compiler):
            dfile = "Depends.%s"%dep
            if os.path.isfile(os.path.join(machines_dir,dfile)):
                shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(self._caseroot,dfile))
        dfile = "Depends.%s.%s"%(machine,compiler)
        if os.path.isfile(os.path.join(machines_dir,dfile)):
            shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(self._caseroot, dfile))
Esempio n. 5
0
    def __init__(self,
                 batch_system=None,
                 machine=None,
                 infile=None,
                 files=None):
        """
        initialize an object
        """
        if files is None:
            files = Files()
        if infile is None:
            infile = files.get_value("BATCH_SPEC_FILE")

        schema = files.get_schema("BATCH_SPEC_FILE")

        GenericXML.__init__(self, infile, schema=schema)

        self.batch_system_node = None
        self.machine_node = None
        self.batch_system = batch_system
        self.machine = machine

        #Append the contents of $HOME/.cime/config_batch.xml if it exists
        #This could cause problems if node matchs are repeated when only one is expected
        infile = os.path.join(os.environ.get("HOME"), ".cime",
                              "config_batch.xml")
        if os.path.exists(infile):
            GenericXML.read(self, infile)

        if self.batch_system is not None:
            self.set_batch_system(self.batch_system, machine=machine)
Esempio n. 6
0
    def __init__(self, infile=None, files=None):
        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("PIO_SPEC_FILE")

        EntryID.__init__(self, infile)
Esempio n. 7
0
    def __init__(self, infile=None, files=None, machine=None):
        """
        initialize an object
        if a filename is provided it will be used,
        otherwise if a files object is provided it will be used
        otherwise create a files object from default values
        """

        self.machine_node = None
        self.machine = None
        self.machines_dir = None

        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("MACHINES_SPEC_FILE", resolved=False)
            infile = files.get_resolved_value(infile)

        self.machines_dir = os.path.dirname(infile)

        GenericXML.__init__(self, infile)

        # Append the contents of $HOME/.cime/config_machines.xml if it exists
        # This could cause problems if node matchs are repeated when only one is expected
        local_infile = os.path.join(os.environ.get("HOME"), ".cime", "config_machines.xml")
        logger.debug("Infile: %s", local_infile)
        if os.path.exists(local_infile):
            GenericXML.read(self, local_infile)

        if machine is None:
            machine = self.probe_machine_name()

        expect(machine is not None, "Could not initialize machine object from %s or %s" % (infile, local_infile))
        self.set_machine(machine)
Esempio n. 8
0
    def __init__(self, compiler=None, machine=None, os_= None, mpilib=None, infile=None, files=None):
        """
        initialize an object
        """
        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("COMPILERS_SPEC_FILE")

        GenericXML.__init__(self, infile)

        self.machine        = machine
        self.os             = os_
        self.mpilib         = mpilib
        self.compiler_nodes = None # Listed from last to first
        self.compiler       = compiler

        if self.compiler is not None:
            self.set_compiler(compiler)

        #Append the contents of $HOME/.cime/config_compilers.xml if it exists
        #This could cause problems if node matchs are repeated when only one is expected
        infile = os.path.join(os.environ.get("HOME"),".cime","config_compilers.xml")
        if os.path.exists(infile):
            GenericXML.read(self, infile)
Esempio n. 9
0
    def _create_newcase_phase(self, test):
    ###########################################################################
        test_dir = self._get_test_dir(test)

        test_case, case_opts, grid, compset, machine, compiler, test_mods = CIME.utils.parse_test_name(test)
        if (compiler != self._compiler):
            raise StandardError("Test '%s' has compiler that does not match instance compliler '%s'" % (test, self._compiler))
        if (self._parallel_jobs == 1):
            scratch_dir = self._machobj.get_value("CESMSCRATCHROOT")
            if (self._project is not None):
                scratch_dir = scratch_dir.replace("$PROJECT", self._project)
            sharedlibroot = os.path.join(scratch_dir, "sharedlibroot.%s" % self._test_id)
        else:
            # Parallelizing builds introduces potential sync problems with sharedlibroot
            # Just let every case build it's own
            sharedlibroot = os.path.join(test_dir, "sharedlibroot.%s" % self._test_id)
        create_newcase_cmd = "%s -model %s -case %s -res %s -mach %s -compiler %s -compset %s -testname %s -project %s -sharedlibroot %s" % \
                              (os.path.join(self._cime_root,"scripts", "create_newcase"),
                               self._cime_model,test_dir, grid, machine, compiler, compset, test_case, self._project,
                               sharedlibroot)
        if (test_case != 'PFS'):
            create_newcase_cmd += " -nosavetiming "
        if (case_opts is not None):
            create_newcase_cmd += " -confopts _%s" % ("_".join(case_opts))
        if (test_mods is not None):
            files = Files()
            (component, mods) = test_mods.split('/')
            testmods_dir = files.get_value("TESTS_MODS_DIR",{"component": component})
            test_mod_file = os.path.join(testmods_dir, component, mods)
            if (not os.path.exists(test_mod_file)):
                self._log_output(test, "Missing testmod file '%s'" % test_mod_file)
                return False
            create_newcase_cmd += " -user_mods_dir %s" % test_mod_file
        logging.info("Calling create_newcase: "+create_newcase_cmd)
        return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)
Esempio n. 10
0
    def error(self, message):
        self.print_usage(sys.stderr)
        # missing argument
        # TODO: assumes comp_interface='mct'
        if "expected one argument" in message:
            if "compset" in message:
                components = get_compsets(Files(comp_interface="mct"))
                self.exit(
                    2,
                    "{}: error: {}\nValid input arguments are {}\n".format(
                        self.prog, message, components),
                )
            elif "component" in message:
                files = Files(comp_interface="mct")
                components = get_components(files)
                # Loop through the elements for each component class (in config_files.xml)
                valid_components = []
                for comp in components:
                    string = "CONFIG_{}_FILE".format(comp)

                    # determine all components in string
                    components = files.get_components(string)
                    for item in components:
                        valid_components.append(item)
                self.exit(
                    2,
                    "{}: error: {}\nValid input arguments are {}\n".format(
                        self.prog, message, valid_components),
                )
        # for all other errors
        self.exit(2, "{}: error: {}\n".format(self.prog, message))
Esempio n. 11
0
    def setup(self, env_archive, components, files=None):
        if files is None:
            files = Files()

        components_node = ET.Element("components")
        components_node.set("version", "2.0")


        model = get_model()
        if 'cpl' not in components:
            components.append('cpl')
        if 'dart' not in components and model == 'cesm':
            components.append('dart')

        for comp in components:
            infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp})

            if infile is not None and os.path.isfile(infile):
                arch = Archive(infile=infile, files=files)
                specs = arch.get_node("comp_archive_spec", {"compname":comp})
            else:
                if infile is None:
                    logger.debug("No archive file defined for component %s"%comp)
                else:
                    logger.debug("Archive file %s for component %s not found"%(infile,comp))

                specs = self.get_optional_node("comp_archive_spec", attributes={"compname":comp})
            if specs is None:
                logger.debug("No archive specs found for component %s"%comp)
            else:
                logger.debug("adding archive spec for %s"%comp)
                components_node.append(specs)
        env_archive.add_child(components_node)
Esempio n. 12
0
    def __init__(self, batch_system=None, machine=None, infile=None, files=None):
        """
        initialize an object
        """
        if files is None:
            files = Files()
        if infile is None:
            infile = files.get_value("BATCH_SPEC_FILE")

        schema = files.get_schema("BATCH_SPEC_FILE")

        GenericXML.__init__(self, infile, schema=schema)

        self.batch_system_node = None
        self.machine_node      = None
        self.batch_system      = batch_system
        self.machine           = machine

        #Append the contents of $HOME/.cime/config_batch.xml if it exists
        #This could cause problems if node matchs are repeated when only one is expected
        infile = os.path.join(os.environ.get("HOME"),".cime","config_batch.xml")
        if os.path.exists(infile):
            GenericXML.read(self, infile)

        if self.batch_system is not None:
            self.set_batch_system(self.batch_system, machine=machine)
Esempio n. 13
0
File: pio.py Progetto: Katetc/cime
    def __init__(self, infile=None, files=None):
        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("PIO_SPEC_FILE")

        EntryID.__init__(self, infile)
Esempio n. 14
0
    def _create_caseroot_tools(self):
        machines_dir = os.path.abspath(self.get_value("MACHDIR"))
        toolsdir = os.path.join(self.get_value("CIMEROOT"),"scripts","Tools")
        # setup executable files in caseroot/
        exefiles = (os.path.join(toolsdir, "case.setup"),
                    os.path.join(toolsdir, "case.build"),
                    os.path.join(toolsdir, "case.submit"),
                    os.path.join(toolsdir, "preview_namelists"),
                    os.path.join(toolsdir, "check_input_data"),
                    os.path.join(toolsdir, "check_case"),
                    os.path.join(toolsdir, "archive_metadata.sh"),
                    os.path.join(toolsdir, "xmlchange"),
                    os.path.join(toolsdir, "xmlquery"))
        try:
            for exefile in exefiles:
                destfile = os.path.join(self._caseroot,os.path.basename(exefile))
                os.symlink(exefile, destfile)
        except Exception as e:
            logger.warning("FAILED to set up exefiles: %s" % str(e))

        # set up utility files in caseroot/Tools/
        toolfiles = (os.path.join(toolsdir, "check_lockedfiles"),
                     os.path.join(toolsdir, "lt_archive.sh"),
                     os.path.join(toolsdir, "getTiming"),
                     os.path.join(toolsdir, "save_provenance"),
                     os.path.join(machines_dir,"Makefile"),
                     os.path.join(machines_dir,"mkSrcfiles"),
                     os.path.join(machines_dir,"mkDepends"))

        for toolfile in toolfiles:
            destfile = os.path.join(self._caseroot,"Tools",os.path.basename(toolfile))
            expect(os.path.isfile(toolfile)," File %s does not exist"%toolfile)
            try:
                os.symlink(toolfile, destfile)
            except Exception as e:
                logger.warning("FAILED to set up toolfiles: %s %s %s" % (str(e), toolfile, destfile))

        # Create Macros file.
        machine = self.get_value("MACH")
        files = Files()
        # Use config_build if the environment variable is set, or if there is no
        # config_compilers file.
        if os.getenv("CIME_USE_CONFIG_BUILD") == "TRUE" or \
           files.get_value("COMPILERS_SPEC_FILE") is None:
            build_file = files.get_value("BUILD_SPEC_FILE")
            machobj = Machines(machine=machine, files=files)
            macro_maker = Build(machobj)
            macros_path = os.path.join(self._caseroot, "Macros")
            with open(macros_path, "w") as macros_file:
                macro_maker.write_macros('Makefile', build_file, macros_file)

        # Copy any system or compiler Depends files to the case.
        compiler = self.get_value("COMPILER")
        for dep in (machine, compiler):
            dfile = "Depends.%s"%dep
            if os.path.isfile(os.path.join(machines_dir,dfile)):
                shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(self._caseroot,dfile))
        dfile = "Depends.%s.%s"%(machine,compiler)
        if os.path.isfile(os.path.join(machines_dir,dfile)):
            shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(self._caseroot, dfile))
Esempio n. 15
0
    def __init__(self, infile=None, files=None, machine=None):
        """
        initialize an object
        if a filename is provided it will be used,
        otherwise if a files object is provided it will be used
        otherwise create a files object from default values
        """

        self.machine_node = None
        self.machine = None
        self.machines_dir = None

        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("MACHINES_SPEC_FILE", resolved=False)
            infile = files.get_resolved_value(infile)

        self.machines_dir = os.path.dirname(infile)

        GenericXML.__init__(self, infile)

        # Append the contents of $HOME/.cime/config_machines.xml if it exists
        # This could cause problems if node matchs are repeated when only one is expected
        infile = os.path.join(os.environ.get("HOME"), ".cime",
                              "config_machines.xml")
        logger.debug("Infile: %s", infile)
        if os.path.exists(infile):
            GenericXML.read(self, infile)

        if machine is None:
            machine = self.probe_machine_name()

        expect(machine is not None, "Could not initialize machine object")
        self.set_machine(machine)
Esempio n. 16
0
def saveLogs(case, lid):
###############################################################################

    logdir = case.get_value("LOGDIR")
    if logdir is not None and len(logdir) > 0:
        if not os.path.isdir(logdir):
            os.makedirs(logdir)

        caseroot = case.get_value("CASEROOT")
        rundir = case.get_value("RUNDIR")

        # get components
        files = Files()
        config_file = files.get_value("CONFIG_DRV_FILE")
        component = Component(config_file)
        comps = [x.lower() for x in component.get_valid_model_components()]
        comps = [x.replace('drv', 'cpl') for x in comps]
        model = [case.get_value("MODEL")]
        comps = comps + model

        # for each component, compress log files and copy to logdir
        for comp in comps:
            logfile = os.path.join(rundir, comp + '.log.' + lid)
            if os.path.isfile(logfile):
                f_in = open(logfile)
                f_out = gzip.open(logfile + '.gz', 'wb')
                f_out.writelines(f_in)
                f_out.close()
                f_in.close()
                os.remove(logfile)
                logfile_copy = logfile + '.gz'
                shutil.copy(logfile_copy,
                            os.path.join(caseroot, logdir, os.path.basename(logfile_copy)))
Esempio n. 17
0
    def print_values(self, machine_name="all"):  # pylint: disable=arguments-differ
        # set flag to look for single machine
        if "all" not in machine_name:
            single_machine = True
            if machine_name == "current":
                machine_name = self.probe_machine_name(warn=False)
        else:
            single_machine = False

        # if we can't find the specified machine
        if single_machine and machine_name is None:
            files = Files()
            config_file = files.get_value("MACHINES_SPEC_FILE")
            print(
                "Machine is not listed in config file: {}".format(config_file))
        else:  # write out machines
            if single_machine:
                machine_names = [machine_name]
            else:
                machine_names = self.list_available_machines()
            print("Machine(s)\n")
            for name in machine_names:
                self.set_machine(name)
                desc = self.text(self.get_child("DESC"))
                os_ = self.text(self.get_child("OS"))
                compilers = self.text(self.get_child("COMPILERS"))
                mpilibnodes = self.get_children("MPILIBS",
                                                root=self.machine_node)
                mpilibs = []
                for node in mpilibnodes:
                    mpilibs.extend(self.text(node).split(","))
                # This does not include the possible depedancy of mpilib on compiler
                # it simply provides a list of mpilibs available on the machine
                mpilibs = list(set(mpilibs))
                max_tasks_per_node = self.text(
                    self.get_child("MAX_TASKS_PER_NODE"))
                mpitasks_node = self.get_optional_child(
                    "MAX_MPITASKS_PER_NODE", root=self.machine_node)
                max_mpitasks_per_node = (self.text(mpitasks_node) if
                                         mpitasks_node else max_tasks_per_node)
                max_gpus_node = self.get_optional_child("MAX_GPUS_PER_NODE",
                                                        root=self.machine_node)
                max_gpus_per_node = self.text(
                    max_gpus_node) if max_gpus_node else 0

                current_machine = self.probe_machine_name(warn=False)
                name += (" (current)" if current_machine
                         and current_machine in name else "")
                print("  {} : {} ".format(name, desc))
                print("      os             ", os_)
                print("      compilers      ", compilers)
                print("      mpilibs        ", mpilibs)
                if max_mpitasks_per_node is not None:
                    print("      pes/node       ", max_mpitasks_per_node)
                if max_tasks_per_node is not None:
                    print("      max_tasks/node ", max_tasks_per_node)
                if max_gpus_per_node is not None:
                    print("      max_gpus/node ", max_gpus_per_node)
                print("")
    def _run_and_assert_query_testlist(self, extra_args=""):
        """Ensure that query_testlist runs successfully with the given extra arguments"""
        files = Files()
        testlist_drv = files.get_value("TESTS_SPEC_FILE", {"component": "drv"})

        self.run_cmd_assert_result(
            "{}/query_testlists --xml-testlist {} {}".format(
                self.SCRIPT_DIR, testlist_drv, extra_args))
Esempio n. 19
0
 def __init__(self, infile=None, files=None):
     """
     initialize an object
     """
     if files is None:
         files = Files()
     schema = files.get_schema("ARCHIVE_SPEC_FILE")
     super(Archive, self).__init__(infile, schema)
Esempio n. 20
0
    def __init__(self, infile=None, files=None):
        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("GRIDS_SPEC_FILE")
        logger.debug(" Grid specification file is %s" % infile)

        GenericXML.__init__(self, infile)
Esempio n. 21
0
    def __init__(self, infile=None, files=None):
        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("GRIDS_SPEC_FILE")
        logger.debug(" Grid specification file is %s" % infile)

        GenericXML.__init__(self, infile)
Esempio n. 22
0
def _main_func(options, work_dir):
    ###############################################################################
    """Construct machines html from an XML file."""

    # Initialize a variables for the html template
    mach_dict = dict()
    model_version = options.version[0]

    # get the machine config file
    files = Files()
    config_file = files.get_value("MACHINES_SPEC_FILE")
    expect(os.path.isfile(config_file),
           "Cannot find config_file {} on disk".format(config_file))

    # instantiate a machines object and read XML values into a dictionary
    machines = Machines(config_file, machine="Query")
    mach_list = machines.list_available_machines()

    # get all the machine values loaded into the mach_dict
    mach_dict = machines.return_values()

    # intialize the support keys
    for machine in mach_list:
        mach_dict[(machine, 'support')] = "Unsupported"

    # loop through the list of supported machines and flag in the dictionary
    supported = options.supported[0].split(',')
    for machine in supported:
        mach_dict[(machine, 'support')] = "Scientific"

    # loop through the list of tested machines and flag in the dictionary
    tested = options.tested[0].split(',')
    for machine in tested:
        mach_dict[(machine, 'support')] = "Tested"

    # load up jinja template
    templateLoader = jinja2.FileSystemLoader(
        searchpath='{0}/templates'.format(work_dir))
    templateEnv = jinja2.Environment(loader=templateLoader)

    # TODO - get the cesm_version for the CIME root
    tmplFile = 'machdef2html.tmpl'
    template = templateEnv.get_template(tmplFile)
    templateVars = {
        'mach_list': mach_list,
        'mach_dict': mach_dict,
        'today': _now,
        'model_version': model_version
    }

    # render the template
    mach_tmpl = template.render(templateVars)

    # write the output file
    with open(options.htmlfile[0], 'w') as html:
        html.write(mach_tmpl)

    return 0
Esempio n. 23
0
 def list_compsets(self):
     files = Files()
     components = files.get_components("COMPSETS_SPEC_FILE")
     for comp in components:
         infile = files.get_value("COMPSETS_SPEC_FILE", {"component": comp})
         compsetobj = Compsets(infile=infile, files=files)
         _, compsets = compsetobj.return_all_values()
         self.CompsetList.addItems(sorted(compsets.keys()))
         self.CompsetList.insertSeparator(999)
Esempio n. 24
0
    def __init__(self, infile=None, files=None):
        """
        initialize an object
        """
        if files is None:
            files = Files()
        schema = files.get_schema("ARCHIVE_SPEC_FILE")

        GenericXML.__init__(self, infile, schema)
Esempio n. 25
0
File: pes.py Progetto: rgknox/cime
 def __init__(self, infile, files=None):
     """
     initialize a files object given input pes specification file
     """
     if files is None:
         files = Files()
     schema = files.get_schema("PES_SPEC_FILE")
     logger.debug("DEBUG: infile is {}".format(infile))
     GenericXML.__init__(self, infile, schema=schema)
Esempio n. 26
0
File: pes.py Progetto: Katetc/cime
 def __init__(self, infile, files=None):
     """
     initialize a files object given input pes specification file
     """
     if files is None:
         files = Files()
     schema = files.get_schema("PES_SPEC_FILE")
     logger.debug("DEBUG: infile is {}".format(infile))
     GenericXML.__init__(self, infile, schema=schema)
Esempio n. 27
0
File: pio.py Progetto: zswolff/E3SM
    def __init__(self, comp_classes, infile=None, files=None):
        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("PIO_SPEC_FILE")

        EntryID.__init__(self, infile)

        self._components = list(comp_classes)
Esempio n. 28
0
    def __init__(self, infile=None):
        """
        initialize an object
        """
        if infile is None:
            files = Files()
            infile = files.get_value("CONFIG_DRV_FILE")

        EntryID.__init__(self,infile)
Esempio n. 29
0
 def __init__(self,infile, files=None):
     """
     initialize an object
     """
     schema = None
     if files is None:
         files = Files()
     schema = files.get_schema("TESTS_SPEC_FILE")
     GenericXML.__init__(self, infile, schema=schema)
Esempio n. 30
0
    def _create_newcase_phase(self, test):
        ###########################################################################
        test_dir = self._get_test_dir(test)

        _, case_opts, grid, compset,\
            machine, compiler, test_mods = CIME.utils.parse_test_name(test)

        create_newcase_cmd = "%s --case %s --res %s --mach %s --compiler %s --compset %s"\
                               " --test" % \
                              (os.path.join(self._cime_root, "scripts", "create_newcase"),
                               test_dir, grid, machine, compiler, compset)
        if self._project is not None:
            create_newcase_cmd += " --project %s " % self._project
        if self._output_root is not None:
            create_newcase_cmd += " --output-root %s " % self._output_root

        if test_mods is not None:
            files = Files()
            (component, modspath) = test_mods.split('/', 1)
            testmods_dir = files.get_value("TESTS_MODS_DIR",
                                           {"component": component})

            test_mod_file = os.path.join(testmods_dir, component, modspath)
            if not os.path.exists(test_mod_file):
                self._log_output(test,
                                 "Missing testmod file '%s'" % test_mod_file)
                return False
            create_newcase_cmd += " --user-mods-dir %s" % test_mod_file

        if case_opts is not None:
            for case_opt in case_opts:  # pylint: disable=not-an-iterable
                if case_opt.startswith('M'):
                    mpilib = case_opt[1:]
                    create_newcase_cmd += " --mpilib %s" % mpilib
                    logger.debug(" MPILIB set to %s" % mpilib)
                if case_opt.startswith('N'):
                    ninst = case_opt[1:]
                    create_newcase_cmd += " --ninst %s" % ninst
                    logger.debug(" NINST set to %s" % ninst)
                if case_opt.startswith('P'):
                    pesize = case_opt[1:]
                    create_newcase_cmd += " --pecount %s" % pesize

        if self._queue is not None:
            create_newcase_cmd += " --queue=%s" % self._queue

        if self._walltime is not None:
            create_newcase_cmd += " --walltime %s" % self._walltime
        elif test in self._test_data and "options" in self._test_data[test] and \
                "wallclock" in self._test_data[test]['options']:
            create_newcase_cmd += " --walltime %s" % self._test_data[test][
                'options']['wallclock']

        logger.debug("Calling create_newcase: " + create_newcase_cmd)
        return self._shell_cmd_for_phase(test, create_newcase_cmd,
                                         CREATE_NEWCASE_PHASE)
Esempio n. 31
0
    def __init__(self, infile=None, files=None):
        """
        initialize an object interface to file config_tests.xml
        """
        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("CONFIG_TESTS_FILE")

        GenericXML.__init__(self, infile)
Esempio n. 32
0
    def __init__(self,  infile=None, files=None):
        """
        initialize an object interface to file config_tests.xml
        """
        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("CONFIG_TESTS_FILE")

        GenericXML.__init__(self,  infile)
Esempio n. 33
0
 def __init__(self, infile):
     """
     initialize an object
     """
     files = Files()
     schema = None
     # not checking schema on external components yet
     cimeroot = get_cime_root()
     if cimeroot in os.path.abspath(infile):
         schema = files.get_schema("CONFIG_CPL_FILE")
     EntryID.__init__(self, infile, schema=schema)
Esempio n. 34
0
def get_tests_from_xml(xml_machine=None,
                       xml_category=None,
                       xml_compiler=None,
                       xml_testlist=None,
                       machine=None,
                       compiler=None):
    """
    Parse testlists for a list of tests
    """
    listoftests = []
    testlistfiles = []
    if (machine is not None):
        thismach = machine
    if (compiler is not None):
        thiscompiler = compiler

    if (xml_testlist is not None):
        expect(os.path.isfile(xml_testlist),
               "Testlist not found or not readable " + xml_testlist)
        testlistfiles.append(xml_testlist)
    else:
        files = Files()
        comps = files.get_components("TESTS_SPEC_FILE")
        for comp in comps:
            test_spec_file = files.get_value("TESTS_SPEC_FILE",
                                             {"component": comp})
            if (os.path.isfile(test_spec_file)):
                testlistfiles.append(test_spec_file)

    for testlistfile in testlistfiles:
        thistestlistfile = Testlist(testlistfile)
        logger.debug("Testlist file is " + testlistfile)
        logger.debug("xml_machine %s xml_category %s xml_compiler %s" %
                     (xml_machine, xml_category, xml_compiler))
        newtests = thistestlistfile.get_tests(xml_machine, xml_category,
                                              xml_compiler)
        for test in newtests:
            if (machine is None):
                thismach = test["machine"]
            if (compiler is None):
                thiscompiler = test["compiler"]
            test["name"] = CIME.utils.get_full_test_name(
                test["testname"],
                grid=test["grid"],
                compset=test["compset"],
                machine=thismach,
                compiler=thiscompiler,
                testmod=None if "testmods" not in test else test["testmods"])
            logger.debug("Adding test %s with compiler %s" %
                         (test["name"], test["compiler"]))
        listoftests += newtests
        logger.debug("Found %d tests" % len(listoftests))

    return listoftests
Esempio n. 35
0
 def __init__(self,infile, files=None):
     """
     initialize an object
     """
     schema = None
     if files is None:
         files = Files()
     schema = files.get_schema("TESTS_SPEC_FILE")
     GenericXML.__init__(self, infile, schema=schema)
     expect(self.get_version() >= 2.0,
            "{} is an unsupported version of the testfile format and will be ignored".format(infile))
Esempio n. 36
0
    def __init__(self, infile=None, files=None):
        if files is None:
            files = Files()
        if infile is None:
            infile = files.get_value("GRIDS_SPEC_FILE")
        logger.debug(" Grid specification file is %s" % infile)
        schema = files.get_schema("GRIDS_SPEC_FILE")

        GenericXML.__init__(self, infile, schema)
        self._version = self.get_version()

        self._comp_gridnames = self._get_grid_names()
Esempio n. 37
0
    def __init__(self, infile):
        """
        initialize an object
        """
        files = Files()
        schema = None
        # not checking schema on external components yet
        cimeroot = get_cime_root()
        if  cimeroot in os.path.abspath(infile):
            schema = files.get_schema("CONFIG_DRV_FILE")

        EntryID.__init__(self, infile, schema=schema)
Esempio n. 38
0
    def _create_newcase_phase(self, test):
    ###########################################################################
        test_dir = self._get_test_dir(test)

        _, case_opts, grid, compset,\
            machine, compiler, test_mods = CIME.utils.parse_test_name(test)

        create_newcase_cmd = "%s --case %s --res %s --mach %s --compiler %s --compset %s"\
                               " --test" % \
                              (os.path.join(self._cime_root, "scripts", "create_newcase"),
                               test_dir, grid, machine, compiler, compset)
        if self._project is not None:
            create_newcase_cmd += " --project %s " % self._project
        if self._output_root is not None:
            create_newcase_cmd += " --output-root %s " % self._output_root


        if test_mods is not None:
            files = Files()
            (component,modspath) = test_mods.split('/',1)
            testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component})

            test_mod_file = os.path.join(testmods_dir, component, modspath)
            if not os.path.exists(test_mod_file):
                self._log_output(test, "Missing testmod file '%s'" % test_mod_file)
                return False
            create_newcase_cmd += " --user-mods-dir %s" % test_mod_file

        if case_opts is not None:
            for case_opt in case_opts: # pylint: disable=not-an-iterable
                if case_opt.startswith('M'):
                    mpilib = case_opt[1:]
                    create_newcase_cmd += " --mpilib %s" % mpilib
                    logger.debug (" MPILIB set to %s" % mpilib)
                if case_opt.startswith('N'):
                    ninst = case_opt[1:]
                    create_newcase_cmd += " --ninst %s" %ninst
                    logger.debug (" NINST set to %s" % ninst)
                if case_opt.startswith('P'):
                    pesize = case_opt[1:]
                    create_newcase_cmd += " --pecount %s"%pesize

        if self._queue is not None:
            create_newcase_cmd += " --queue=%s" % self._queue

        if self._walltime is not None:
            create_newcase_cmd += " --walltime %s" % self._walltime
        elif test in self._test_data and "options" in self._test_data[test] and \
                "wallclock" in self._test_data[test]['options']:
            create_newcase_cmd += " --walltime %s" % self._test_data[test]['options']['wallclock']

        logger.debug("Calling create_newcase: " + create_newcase_cmd)
        return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)
Esempio n. 39
0
    def __init__(self, infile=None):
        """
        initialize an object

        >>> files = Files()
        >>> files.get_value('CASEFILE_HEADERS',resolved=False)
        '$CIMEROOT/CIME/data/config/config_headers.xml'
        """
        if infile is None:
            files = Files()
            infile = files.get_value("CASEFILE_HEADERS", resolved=True)
        super(Headers, self).__init__(infile)
Esempio n. 40
0
    def __init__(self, infile=None):
        """
        initialize an object

        >>> files = Files()
        >>> files.get_value('CASEFILE_HEADERS',resolved=False)
        '$CIMEROOT/cime_config/config_headers.xml'
        """
        if infile is None:
            files = Files()
            infile = files.get_value('CASEFILE_HEADERS', resolved=True)
        EntryID.__init__(self, infile)
Esempio n. 41
0
    def __init__(self,infile=None):
        """
        initialize an object

        >>> files = Files()
        >>> files.get_value('CASEFILE_HEADERS',resolved=False)
        '$CIMEROOT/config/config_headers.xml'
        """
        if infile is None:
            files = Files()
            infile = files.get_value('CASEFILE_HEADERS', resolved=True)
        super(Headers, self).__init__(infile)
Esempio n. 42
0
    def __init__(self, infile=None, files=None):
        if files is None:
            files = Files()
        if infile is None:
            infile = files.get_value("GRIDS_SPEC_FILE")
        logger.debug(" Grid specification file is %s" % infile)
        schema = files.get_schema("GRIDS_SPEC_FILE")

        GenericXML.__init__(self, infile, schema)
        self._version = self.get_version()

        self._comp_gridnames = self._get_grid_names()
Esempio n. 43
0
    def __init__(self, machine, files=None, infile=None):
        """
        initialize an object
        """
        if (infile is None):
            if files is None:
                files = Files()
            infile = files.get_value("LTARCHIVE_SPEC_FILE", resolved=False)
            infile = files.get_resolved_value(infile)

        GenericXML.__init__(self, infile)

        self.machine = machine
Esempio n. 44
0
    def __init__(self, infile=None, files=None):
        """
        initialize a files object given input pes specification file
        """
        if files is None:
            files = Files()
        if infile is None:
            infile = files.get_value("INPUTDATA_SPEC_FILE")
        schema = files.get_schema("INPUTDATA_SPEC_FILE")
        logger.debug("DEBUG: infile is {}".format(infile))
        GenericXML.__init__(self, infile, schema=schema)

        self._servernode = None
Esempio n. 45
0
    def __init__(self, infile):
        """
        initialize an object
        """
        files = Files()
        schema = files.get_schema("CONFIG_DRV_FILE")
        if schema is not None:
            # not checking schema on external components yet
            cimeroot = get_cime_root()
            if cimeroot in os.path.abspath(infile):
                self.validate_xml_file(infile, schema)

        EntryID.__init__(self, infile)
Esempio n. 46
0
    def __init__(self, machine, files=None, infile=None):
        """
        initialize an object
        """
        if (infile is None):
            if files is None:
                files = Files()
            infile = files.get_value("LTARCHIVE_SPEC_FILE", resolved=False)
            infile = files.get_resolved_value(infile)

        GenericXML.__init__(self, infile)

        self.machine = machine
Esempio n. 47
0
    def __init__(self, infile=None, files=None):
        """
        initialize a files object given input pes specification file
        """
        if files is None:
            files = Files()
        if infile is None:
            infile = files.get_value("INPUTDATA_SPEC_FILE")
        schema = files.get_schema("INPUTDATA_SPEC_FILE")
        logger.debug("DEBUG: infile is {}".format(infile))
        GenericXML.__init__(self, infile, schema=schema)

        self._servernode = None
Esempio n. 48
0
def test_st_archive(self, testdir="st_archive_test"):
    archive = Archive()
    files = Files()
    components = []
#    expect(not self.get_value("MULTI_DRIVER"),"Test not configured for multi-driver cases")

    config_archive_files = archive.get_all_config_archive_files(files)
    # create the run directory testdir and populate it with rest_file and hist_file from
    # config_archive.xml test_file_names
    if os.path.exists(testdir):
        logger.info("Removing existing test directory {}".format(testdir))
        shutil.rmtree(testdir)
    dout_s_root=os.path.join(testdir,"archive")
    archive = Archive()
    schema = files.get_schema("ARCHIVE_SPEC_FILE")
    for config_archive_file in config_archive_files:
        archive.read(config_archive_file, schema)
    comp_archive_specs = archive.get_children("comp_archive_spec")
    for comp_archive_spec in comp_archive_specs:
        components.append(archive.get(comp_archive_spec, 'compname'))
        test_file_names = archive.get_optional_child("test_file_names", root=comp_archive_spec)
        if test_file_names is not None:
            if not os.path.exists(testdir):
                os.makedirs(os.path.join(testdir,"archive"))

            for file_node in archive.get_children("tfile", root=test_file_names):
                fname = os.path.join(testdir,archive.text(file_node))
                disposition = archive.get(file_node, "disposition")
                logger.info("Create file {} with disposition {}".
                            format(fname, disposition))
                with open(fname, 'w') as fd:
                    fd.write(disposition+"\n")

    logger.info("testing components: {} ".format(list(set(components))))
    _archive_process(self, archive, None, False, False,components=list(set(components)),
                     dout_s_root=dout_s_root,
                     casename="casename", rundir=testdir, testonly=True)

    _check_disposition(testdir)

    # Now test the restore capability
    testdir2 = os.path.join(testdir,"run2")
    os.makedirs(testdir2)

    restore_from_archive(self, rundir=testdir2, dout_s_root=dout_s_root)

    restfiles = [f for f in os.listdir(os.path.join(testdir,"archive","rest","1976-01-01-00000"))]
    for _file in restfiles:
        expect(os.path.isfile(os.path.join(testdir2,_file)), "Expected file {} to be restored from rest dir".format(_file))

    return True
Esempio n. 49
0
    def __init__(self,
                 machobj,
                 infile=None,
                 compiler=None,
                 mpilib=None,
                 files=None,
                 version=None):
        """
        initialize an object
        """

        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("COMPILERS_SPEC_FILE")
            schema = files.get_schema("COMPILERS_SPEC_FILE")

        GenericXML.__init__(self, infile, schema)
        self._machobj = machobj
        if version is not None:
            # this is used in scripts_regression_tests to force version 2, it should not be used otherwise
            self._version = version
        else:
            self._version = self.get_version()

        self.machine = machobj.get_machine_name()
        self.os = machobj.get_value("OS")
        if mpilib is None:
            mpilib = machobj.get_default_MPIlib()
        self.mpilib = mpilib
        if compiler is None:
            compiler = machobj.get_default_compiler()
        self.compiler = compiler

        self.compiler_nodes = None  # Listed from last to first
        #Append the contents of $HOME/.cime/config_compilers.xml if it exists
        #This could cause problems if node matchs are repeated when only one is expected
        infile = os.path.join(os.environ.get("HOME"), ".cime",
                              "config_compilers.xml")
        if os.path.exists(infile):
            GenericXML.read(self, infile)

        if self.compiler is not None:
            self.set_compiler(compiler)

        if self._version != "1.0":
            # Run an XPath query to extract the list of flag variable names.
            ns = {"xs": "http://www.w3.org/2001/XMLSchema"}
            flag_xpath = ".//xs:group[@name='compilerVars']/xs:choice/xs:element[@type='flagsVar']"
            flag_elems = ET.parse(schema).getroot().findall(flag_xpath, ns)
            self.flag_vars = set(elem.get('name') for elem in flag_elems)
Esempio n. 50
0
def test_st_archive(self, testdir="st_archive_test"):
    files = Files()
    archive = Archive(files=files)
    components = []
#    expect(not self.get_value("MULTI_DRIVER"),"Test not configured for multi-driver cases")

    config_archive_files = archive.get_all_config_archive_files(files)
    # create the run directory testdir and populate it with rest_file and hist_file from
    # config_archive.xml test_file_names
    if os.path.exists(testdir):
        logger.info("Removing existing test directory {}".format(testdir))
        shutil.rmtree(testdir)
    dout_s_root=os.path.join(testdir,"archive")
    archive = Archive()
    schema = files.get_schema("ARCHIVE_SPEC_FILE")
    for config_archive_file in config_archive_files:
        archive.read(config_archive_file, schema)
    comp_archive_specs = archive.get_children("comp_archive_spec")
    for comp_archive_spec in comp_archive_specs:
        components.append(archive.get(comp_archive_spec, 'compname'))
        test_file_names = archive.get_optional_child("test_file_names", root=comp_archive_spec)
        if test_file_names is not None:
            if not os.path.exists(testdir):
                os.makedirs(os.path.join(testdir,"archive"))

            for file_node in archive.get_children("tfile", root=test_file_names):
                fname = os.path.join(testdir,archive.text(file_node))
                disposition = archive.get(file_node, "disposition")
                logger.info("Create file {} with disposition {}".
                            format(fname, disposition))
                with open(fname, 'w') as fd:
                    fd.write(disposition+"\n")

    logger.info("testing components: {} ".format(list(set(components))))
    _archive_process(self, archive, None, False, False,components=list(set(components)),
                     dout_s_root=dout_s_root,
                     casename="casename", rundir=testdir, testonly=True)

    _check_disposition(testdir)

    # Now test the restore capability
    testdir2 = os.path.join(testdir,"run2")
    os.makedirs(testdir2)

    restore_from_archive(self, rundir=testdir2, dout_s_root=dout_s_root, test=True)

    restfiles = [f for f in os.listdir(os.path.join(testdir,"archive","rest","1976-01-01-00000"))]
    for _file in restfiles:
        expect(os.path.isfile(os.path.join(testdir2,_file)), "Expected file {} to be restored from rest dir".format(_file))

    return True
Esempio n. 51
0
File: tests.py Progetto: Katetc/cime
 def __init__(self,  infile=None, files=None):
     """
     initialize an object interface to file config_tests.xml
     """
     if infile is None:
         if files is None:
             files = Files()
         infile = files.get_value("CONFIG_TESTS_FILE")
     GenericXML.__init__(self,  infile)
     # append any component specific config_tests.xml files
     for comp in files.get_components("CONFIG_TESTS_FILE"):
         if comp is None:
             continue
         infile = files.get_value("CONFIG_TESTS_FILE", attribute={"component":comp})
         if os.path.isfile(infile):
             self.read(infile)
Esempio n. 52
0
    def __init__(self, infile, comp_class):
        """
        initialize a Component obect from the component xml file in infile
        associate the component class with comp_class if provided.
        """
        self._comp_class = comp_class
        if infile == 'testingonly':
            self.filename = infile
            return
        files = Files()
        schema = None
        EntryID.__init__(self, infile)
        schema = files.get_schema("CONFIG_{}_FILE".format(comp_class), attributes={"version":"{}".format(self.get_version())})

        if schema is not None:
            self.validate_xml_file(infile, schema)
Esempio n. 53
0
    def __init__(self, machobj, infile=None, compiler=None, mpilib=None, files=None, version=None):
        """
        initialize an object
        """

        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("COMPILERS_SPEC_FILE")
            schema = files.get_schema("COMPILERS_SPEC_FILE")

        GenericXML.__init__(self, infile, schema)
        self._machobj = machobj
        if version is not None:
            # this is used in scripts_regression_tests to force version 2, it should not be used otherwise
            self._version = version
        else:
            self._version = self.get_version()

        self.machine  = machobj.get_machine_name()
        self.os = machobj.get_value("OS")
        if compiler is None:
            compiler = machobj.get_default_compiler()
        self.compiler       = compiler

        if mpilib is None:
            if compiler is None:
                mpilib = machobj.get_default_MPIlib()
            else:
                mpilib = machobj.get_default_MPIlib(attributes={'compiler':compiler})
        self.mpilib = mpilib

        self.compiler_nodes = None # Listed from last to first
        #Append the contents of $HOME/.cime/config_compilers.xml if it exists
        #This could cause problems if node matchs are repeated when only one is expected
        infile = os.path.join(os.environ.get("HOME"),".cime","config_compilers.xml")
        if os.path.exists(infile):
            GenericXML.read(self, infile)

        if self.compiler is not None:
            self.set_compiler(compiler)

        if self._version > 1.0:
            schema_db = GenericXML(infile=schema)
            compiler_vars = schema_db.get_child("{http://www.w3.org/2001/XMLSchema}group", attributes={"name":"compilerVars"})
            choice  = schema_db.get_child(name="{http://www.w3.org/2001/XMLSchema}choice", root=compiler_vars)
            self.flag_vars = set(schema_db.get(elem, "name") for elem in schema_db.get_children(root=choice, attributes={"type":"flagsVar"}))
Esempio n. 54
0
    def __init__(self, machobj, infile=None, compiler=None, mpilib=None, files=None, version=None):
        """
        initialize an object
        """

        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("COMPILERS_SPEC_FILE")
            schema = files.get_schema("COMPILERS_SPEC_FILE")

        GenericXML.__init__(self, infile, schema)
        self._machobj = machobj
        if version is not None:
            # this is used in scripts_regression_tests to force version 2, it should not be used otherwise
            self._version = version
        else:
            self._version = self.get_version()

        self.machine  = machobj.get_machine_name()
        self.os = machobj.get_value("OS")
        if mpilib is None:
            mpilib = machobj.get_default_MPIlib()
        self.mpilib = mpilib
        if compiler is None:
            compiler = machobj.get_default_compiler()
        self.compiler       = compiler

        self.compiler_nodes = None # Listed from last to first
        #Append the contents of $HOME/.cime/config_compilers.xml if it exists
        #This could cause problems if node matchs are repeated when only one is expected
        infile = os.path.join(os.environ.get("HOME"),".cime","config_compilers.xml")
        if os.path.exists(infile):
            GenericXML.read(self, infile)

        if self.compiler is not None:
            self.set_compiler(compiler)

        if self._version > 1.0:
            # Run an XPath query to extract the list of flag variable names.
            ns = {"xs": "http://www.w3.org/2001/XMLSchema"}
            flag_xpath = ".//xs:group[@name='compilerVars']/xs:choice/xs:element[@type='flagsVar']"
            flag_elems = ET.parse(schema).getroot().findall(flag_xpath, ns)
            self.flag_vars = set(elem.get('name') for elem in flag_elems)
Esempio n. 55
0
    def __init__(self, infile, files=None):
        """Construct a `NamelistDefinition` from an XML file."""

        # if the file is invalid we may not be able to check the version
        # but we need to do it this way until we remove the version 1 files
        schema = None
        if files is None:
            files = Files()
        schema = files.get_schema("NAMELIST_DEFINITION_FILE")
        expect(os.path.isfile(infile), "File {} does not exist".format(infile))
        super(NamelistDefinition, self).__init__(infile, schema=schema)

        self._attributes = {}
        self._entry_nodes = []
        self._entry_ids = []
        self._valid_values = {}
        self._entry_types = {}
        self._group_names = {}
        self._nodes = {}
Esempio n. 56
0
    def _create_newcase_phase(self, test):
    ###########################################################################
        test_dir = self._get_test_dir(test)

        _, case_opts, grid, compset,\
            machine, compiler, test_mods = CIME.utils.parse_test_name(test)
        if compiler != self._compiler:
            raise StandardError("Test '%s' has compiler that does"
                                " not match instance compliler '%s'" % (test, self._compiler))

        create_newcase_cmd = "%s --case %s --res %s --mach %s --compiler %s --compset %s"\
                               " --project %s --test"%\
                              (os.path.join(self._cime_root, "scripts", "create_newcase"),
                               test_dir, grid, machine, compiler, compset, self._project)

        if test_mods is not None:
            files = Files()
            (component,modspath) = test_mods.split('/',1)
            testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component})

            test_mod_file = os.path.join(testmods_dir, component, modspath)
            if not os.path.exists(test_mod_file):
                self._log_output(test, "Missing testmod file '%s'" % test_mod_file)
                return False
            create_newcase_cmd += " --user-mods-dir %s" % test_mod_file

        if case_opts is not None:
            for case_opt in case_opts:
                if case_opt.startswith('M'):
                    mpilib = case_opt[1:]
                    create_newcase_cmd += " --mpilib %s" % mpilib
                    logger.debug (" MPILIB set to %s" % mpilib)
                if case_opt.startswith('N'):
                    ninst = case_opt[1:]
                    create_newcase_cmd += " --ninst %s" %ninst
                    logger.debug (" NINST set to %s" % ninst)
                pesize = re.match('P([SMLX][12]?)', case_opt)
                if pesize:
                    create_newcase_cmd += " --pecount %s"%pesize.group(1)


        logger.debug("Calling create_newcase: " + create_newcase_cmd)
        return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)
Esempio n. 57
0
    def __init__(self, infile=None, files=None, machine=None):
        """
        initialize an object
        if a filename is provided it will be used,
        otherwise if a files object is provided it will be used
        otherwise create a files object from default values
        """

        self.machine_node = None
        self.machine = None
        self.machines_dir = None
        schema = None
        if files is None:
            files = Files()
        if infile is None:
            infile = files.get_value("MACHINES_SPEC_FILE")
        schema = files.get_schema("MACHINES_SPEC_FILE")
        logger.debug("Verifying using schema {}".format(schema))

        self.machines_dir = os.path.dirname(infile)

        GenericXML.__init__(self, infile, schema)

        # Append the contents of $HOME/.cime/config_machines.xml if it exists
        # This could cause problems if node matchs are repeated when only one is expected
        local_infile = os.path.join(os.environ.get("HOME"),".cime","config_machines.xml")
        logger.debug("Infile: {}".format(local_infile))
        if os.path.exists(local_infile):
            GenericXML.read(self, local_infile, schema)

        if machine is None:
            if "CIME_MACHINE" in os.environ:
                machine = os.environ["CIME_MACHINE"]
            else:
                cime_config = get_cime_config()
                if cime_config.has_option("main", "machine"):
                    machine = cime_config.get("main", "machine")
                if machine is None:
                    machine = self.probe_machine_name()

        expect(machine is not None, "Could not initialize machine object from {} or {}".format(infile, local_infile))
        self.set_machine(machine)
Esempio n. 58
0
File: case.py Progetto: ekluzek/cime
    def _set_compset_and_pesfile(self, compset_name, user_compset=False, pesfile=None):
        """
        Loop through all the compset files and find the compset
        specifation file that matches either the input 'compset_name'.
        Note that the input compset name (i.e. compset_name) can be
        either a longname or an alias.  This will also set the
        compsets and pes specfication files.
        """
        files = Files()
        components = files.get_components("COMPSETS_SPEC_FILE")
        logger.debug(" Possible components for COMPSETS_SPEC_FILE are %s" % components)

        # Loop through all of the files listed in COMPSETS_SPEC_FILE and find the file
        # that has a match for either the alias or the longname in that order
        for component in components:

            # Determine the compsets file for this component
            compsets_filename = files.get_value("COMPSETS_SPEC_FILE", {"component":component})

            # If the file exists, read it and see if there is a match for the compset alias or longname
            if (os.path.isfile(compsets_filename)):
                compsets = Compsets(compsets_filename)
                match = compsets.get_compset_match(name=compset_name)
                pesfile = files.get_value("PES_SPEC_FILE"     , {"component":component})
                if match is not None:
                    self._pesfile = pesfile
                    self._compsetsfile = compsets_filename
                    self._compsetname = match
                    tests_filename    = files.get_value("TESTS_SPEC_FILE"   , {"component":component}, resolved=False)
                    tests_mods_dir    = files.get_value("TESTS_MODS_DIR"    , {"component":component}, resolved=False)
                    user_mods_dir     = files.get_value("USER_MODS_DIR"     , {"component":component}, resolved=False)
                    self.set_lookup_value("COMPSETS_SPEC_FILE" ,
                                   files.get_value("COMPSETS_SPEC_FILE", {"component":component}, resolved=False))
                    self.set_lookup_value("TESTS_SPEC_FILE"    , tests_filename)
                    self.set_lookup_value("TESTS_MODS_DIR"     , tests_mods_dir)
                    self.set_lookup_value("USER_MODS_DIR"      , user_mods_dir)
                    self.set_lookup_value("PES_SPEC_FILE"      ,
                                   files.get_value("PES_SPEC_FILE"     , {"component":component}, resolved=False))
                    logger.info("Compset longname is %s " %(match))
                    logger.info("Compset specification file is %s" %(compsets_filename))
                    logger.info("Pes     specification file is %s" %(pesfile))
                    return

        if user_compset is True:
            #Do not error out for user_compset
            logger.warn("Could not find a compset match for either alias or longname in %s" %(compset_name))
            self._compsetname = compset_name
            self._pesfile = pesfile
            self.set_lookup_value("PES_SPEC_FILE", pesfile)
        else:
            expect(False,
                   "Could not find a compset match for either alias or longname in %s" %(compset_name))
Esempio n. 59
0
def get_tests_from_xml(xml_machine=None,xml_category=None,xml_compiler=None, xml_testlist=None,
                       machine=None, compiler=None):
    """
    Parse testlists for a list of tests
    """
    listoftests = []
    testlistfiles = []
    if(machine is not None):
        thismach=machine
    if(compiler is not None):
        thiscompiler = compiler

    if(xml_testlist is not None):
        expect(os.path.isfile(xml_testlist), "Testlist not found or not readable "+xml_testlist)
        testlistfiles.append(xml_testlist)
    else:
        files = Files()
        comps = files.get_components("TESTS_SPEC_FILE")
        for comp in comps:
            test_spec_file = files.get_value("TESTS_SPEC_FILE", {"component":comp})
            if(os.path.isfile(test_spec_file)):
                testlistfiles.append(test_spec_file)

    for testlistfile in testlistfiles:
        thistestlistfile = Testlist(testlistfile)
        logger.debug("Testlist file is "+testlistfile)
        logger.debug("xml_machine {} xml_category {} xml_compiler {}".format(xml_machine, xml_category, xml_compiler))
        newtests =  thistestlistfile.get_tests(xml_machine, xml_category, xml_compiler)
        for test in newtests:
            if(machine is None):
                thismach = test["machine"]
            if(compiler is None):
                thiscompiler = test["compiler"]
            test["name"] = CIME.utils.get_full_test_name(test["testname"], grid=test["grid"], compset=test["compset"],
                                                         machine=thismach, compiler=thiscompiler,
                                                         testmod=None if "testmods" not in test else test["testmods"])
            logger.debug("Adding test {} with compiler {}".format(test["name"], test["compiler"]))
        listoftests += newtests
        logger.debug("Found {:d} tests".format(len(listoftests)))

    return listoftests