Example #1
0
    def get_resolved_value(self, raw_value):
        """
        A value in the xml file may contain references to other xml
        variables or to environment variables. These are refered to in
        the perl style with $name and $ENV{name}.

        >>> obj = GenericXML()
        >>> os.environ["FOO"] = "BAR"
        >>> os.environ["BAZ"] = "BARF"
        >>> obj.get_resolved_value("one $ENV{FOO} two $ENV{BAZ} three")
        'one BAR two BARF three'
        >>> obj.get_resolved_value("2 + 3 - 1")
        '4'
        >>> obj.get_resolved_value("0001-01-01")
        '0001-01-01'
        """
        logger.debug("raw_value %s" % raw_value)
        reference_re = re.compile(r'\${?(\w+)}?')
        env_ref_re   = re.compile(r'\$ENV\{(\w+)\}')
        math_re = re.compile(r'\s[+-/*]\s')
        item_data = raw_value

        if item_data is None:
            return None

        if type(item_data) is not str:
            return item_data

        for m in env_ref_re.finditer(item_data):
            logger.debug("look for %s in env" % item_data)
            env_var = m.groups()[0]
            expect(env_var in os.environ, "Undefined env var '%s'" % env_var)
            item_data = item_data.replace(m.group(), os.environ[env_var])

        for m in reference_re.finditer(item_data):
            var = m.groups()[0]
            logger.debug("find: %s" % var)
            ref = self.get_value(var)
            if ref is not None:
                logger.debug("resolve: " + str(ref))
                item_data = item_data.replace(m.group(), self.get_resolved_value(str(ref)))
            elif var == "CIMEROOT":
                cimeroot = get_cime_root()
                item_data = item_data.replace(m.group(), cimeroot)
            elif var == "SRCROOT":
                srcroot = os.path.join(get_cime_root(),"..")
                item_data = item_data.replace(m.group(), srcroot)
            elif var in os.environ:
                # this is a list of suppressed warnings (things normally expected to be resolved in env)
                if var not in ("USER",):
                    logging.warn("Resolved from env: " + var)
                item_data = item_data.replace(m.group(), os.environ[var])
        if math_re.search(item_data):
            try:
                tmp = eval(item_data)
            except:
                tmp = item_data
            item_data = str(tmp)

        return item_data
Example #2
0
    def get_resolved_value(self, raw_value):
        """
        A value in the xml file may contain references to other xml
        variables or to environment variables. These are refered to in
        the perl style with $name and $ENV{name}.

        >>> obj = GenericXML()
        >>> os.environ["FOO"] = "BAR"
        >>> os.environ["BAZ"] = "BARF"
        >>> obj.get_resolved_value("one $ENV{FOO} two $ENV{BAZ} three")
        'one BAR two BARF three'
        >>> obj.get_resolved_value("2 + 3 - 1")
        '4'
        >>> obj.get_resolved_value("0001-01-01")
        '0001-01-01'
        """
        logger.debug("raw_value %s" % raw_value)
        reference_re = re.compile(r'\${?(\w+)}?')
        env_ref_re   = re.compile(r'\$ENV\{(\w+)\}')
        math_re = re.compile(r'\s[+-/*]\s')
        item_data = raw_value

        if item_data is None:
            return None

        if type(item_data) is not str:
            return item_data

        for m in env_ref_re.finditer(item_data):
            logger.debug("look for %s in env" % item_data)
            env_var = m.groups()[0]
            expect(env_var in os.environ, "Undefined env var '%s'" % env_var)
            item_data = item_data.replace(m.group(), os.environ[env_var])

        for m in reference_re.finditer(item_data):
            var = m.groups()[0]
            logger.debug("find: %s" % var)
            ref = self.get_value(var)
            if ref is not None:
                logger.debug("resolve: " + str(ref))
                item_data = item_data.replace(m.group(), self.get_resolved_value(str(ref)))
            elif var == "CIMEROOT":
                cimeroot = get_cime_root()
                item_data = item_data.replace(m.group(), cimeroot)
            elif var == "SRCROOT":
                srcroot = os.path.join(get_cime_root(),"..")
                item_data = item_data.replace(m.group(), srcroot)
            elif var in os.environ:
                # this is a list of suppressed warnings (things normally expected to be resolved in env)
                if var not in ("USER",):
                    logging.debug("Resolved from env: " + var)
                item_data = item_data.replace(m.group(), os.environ[var])
        if math_re.search(item_data):
            try:
                tmp = eval(item_data)
            except:
                tmp = item_data
            item_data = str(tmp)

        return item_data
Example #3
0
File: case.py Project: erika72/cime
    def __init__(self, case_root=None, read_only=True):

        if case_root is None:
            case_root = os.getcwd()
        self._caseroot = case_root
        logger.debug("Initializing Case.")
        self._env_files_that_need_rewrite = set()
        self._read_only_mode = True
        self._force_read_only = read_only

        self._env_entryid_files = []
        self._env_generic_files = []
        self._files = []

        self.read_xml()

        # Hold arbitary values. In create_newcase we may set values
        # for xml files that haven't been created yet. We need a place
        # to store them until we are ready to create the file. At file
        # creation we get the values for those fields from this lookup
        # table and then remove the entry.
        self.lookups = {}
        self.set_lookup_value('CIMEROOT',os.path.abspath(get_cime_root()))

        self._compsetname = None
        self._gridname = None
        self._compsetsfile = None
        self._pesfile = None
        self._gridfile = None
        self._components = []
        self._component_classes = []
Example #4
0
    def run_phase(self):

        rundir = self._case.get_value("RUNDIR")
        exeroot = self._case.get_value("EXEROOT")
        mach = self._case.get_value("MACH")

        log = os.path.join(rundir, "funit.log")
        if os.path.exists(log):
            os.remove(log)

        test_spec_dir = self.get_test_spec_dir()
        unit_test_tool = os.path.abspath(
            os.path.join(get_cime_root(), "scripts", "fortran_unit_testing",
                         "run_tests.py"))
        args = "--build-dir {} --test-spec-dir {} --machine {}".format(
            exeroot, test_spec_dir, mach)

        # BUG(wjs, 2022-01-07, ESMCI/CIME#4136) For now, these Fortran unit tests only
        # work with the old config_compilers.xml-based configuration
        my_env = os.environ.copy()
        my_env["CIME_NO_CMAKE_MACRO"] = "ON"

        stat = run_cmd(
            "{} {} >& funit.log".format(unit_test_tool, args),
            from_dir=rundir,
            env=my_env,
        )[0]

        append_testlog(open(os.path.join(rundir, "funit.log"), "r").read())

        expect(stat == 0, "RUN FAIL for FUNIT")
Example #5
0
def _run_pylint(on_file, interactive):
###############################################################################
    pylint = find_executable("pylint")

    cmd_options = " --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import"
    cmd_options += ",fixme,broad-except,bare-except,eval-used,exec-used,global-statement"
    cmd_options += ",logging-format-interpolation,no-name-in-module"
    cimeroot = get_cime_root()

    if "scripts/Tools" in on_file:
        cmd_options +=",relative-import"

    # add init-hook option
    cmd_options += " --init-hook='sys.path.extend((\"%s\",\"%s\",\"%s\"))'"%\
        (os.path.join(cimeroot,"scripts","lib"),
         os.path.join(cimeroot,"scripts","Tools"),
         os.path.join(cimeroot,"scripts","fortran_unit_testing","python"))

    cmd = "%s %s %s" % (pylint, cmd_options, on_file)
    logger.debug("pylint command is %s"%cmd)
    stat, out, err = run_cmd(cmd, verbose=False, from_dir=cimeroot)
    if stat != 0:
        if interactive:
            logger.info("File %s has pylint problems, please fix\n    Use command: %s" % (on_file, cmd))
            logger.info(out + "\n" + err)
        return (on_file, out + "\n" + err)
    else:
        if interactive:
            logger.info("File %s has no pylint problems" % on_file)
        return (on_file, "")
Example #6
0
File: case.py Project: ekluzek/cime
    def __init__(self, case_root=None, read_only=True):

        if case_root is None:
            case_root = os.getcwd()
        self._caseroot = case_root
        logger.debug("Initializing Case.")
        self._env_files_that_need_rewrite = set()
        self._read_only_mode = True
        self._force_read_only = read_only

        self._env_entryid_files = []
        self._env_generic_files = []
        self._files = []

        self.read_xml()

        # Hold arbitary values. In create_newcase we may set values
        # for xml files that haven't been created yet. We need a place
        # to store them until we are ready to create the file. At file
        # creation we get the values for those fields from this lookup
        # table and then remove the entry.
        self.lookups = {}
        self.set_lookup_value('CIMEROOT',os.path.abspath(get_cime_root()))

        self._compsetname = None
        self._gridname = None
        self._compsetsfile = None
        self._pesfile = None
        self._gridfile = None
        self._components = []
        self._component_classes = []
Example #7
0
def get_all_checkable_files():
    ###############################################################################
    cimeroot = get_cime_root()
    all_git_files = run_cmd_no_fail("git ls-files",
                                    from_dir=cimeroot,
                                    verbose=False).splitlines()
    if get_cime_default_driver() == "nuopc":
        srcroot = get_src_root()
        nuopc_git_files = []
        try:
            nuopc_git_files = run_cmd_no_fail(
                "git ls-files",
                from_dir=os.path.join(srcroot, "components", "cmeps"),
                verbose=False,
            ).splitlines()
        except:
            logger.warning("No nuopc driver found in source")
        all_git_files.extend([
            os.path.join(srcroot, "components", "cmeps", _file)
            for _file in nuopc_git_files
        ])
    files_to_test = [
        item for item in all_git_files
        if ((item.endswith(".py") or is_python_executable(
            os.path.join(cimeroot, item))) and not _should_pylint_skip(item))
    ]

    return files_to_test
    def test_k_append_config(self):
        machlist_before = self.MACHINE.list_available_machines()
        self.assertEqual(
            len(machlist_before) > 1, True, msg="Problem reading machine list"
        )

        newmachfile = os.path.join(
            utils.get_cime_root(),
            "config",
            "xml_schemas",
            "config_machines_template.xml",
        )
        self.MACHINE.read(newmachfile)
        machlist_after = self.MACHINE.list_available_machines()

        self.assertEqual(
            len(machlist_after) - len(machlist_before),
            1,
            msg="Not able to append config_machines.xml {} {}".format(
                len(machlist_after), len(machlist_before)
            ),
        )
        self.assertEqual(
            "mymachine" in machlist_after,
            True,
            msg="Not able to append config_machines.xml",
        )
Example #9
0
    def __init__(self,
                 caseroot=None,
                 infile="env_mach_specific.xml",
                 components=None,
                 unit_testing=False,
                 read_only=False,
                 standalone_configure=False,
                 comp_interface=None):
        """
        initialize an object interface to file env_mach_specific.xml in the case directory

        Notes on some arguments:
        standalone_configure: logical - whether this is being called from the standalone
            configure utility, outside of a case
        """
        schema = os.path.join(get_cime_root(), "config", "xml_schemas",
                              "env_mach_specific.xsd")
        EnvBase.__init__(self,
                         caseroot,
                         infile,
                         schema=schema,
                         read_only=read_only)
        self._allowed_mpi_attributes = ("compiler", "mpilib", "threaded",
                                        "unit_testing", "queue",
                                        "comp_interface")
        self._comp_interface = comp_interface
        self._unit_testing = unit_testing
        self._standalone_configure = standalone_configure
Example #10
0
def _run_pylint(on_file, interactive):
    ###############################################################################
    pylint = find_executable("pylint")

    cmd_options = " --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement"
    cimeroot = get_cime_root()

    if "scripts/Tools" in on_file:
        cmd_options += ",relative-import"

    # add init-hook option
    cmd_options += " --init-hook='sys.path.extend((\"%s\",\"%s\"))'"%\
        (os.path.join(cimeroot,"utils","python"),
         os.path.join(cimeroot,"scripts","Tools"))

    cmd = "%s %s %s" % (pylint, cmd_options, on_file)
    logger.debug("pylint command is %s" % cmd)
    stat, out, err = run_cmd(cmd, verbose=False, from_dir=cimeroot)
    if stat != 0:
        if interactive:
            logger.info(
                "File %s has pylint problems, please fix\n    Use command: %s"
                % (on_file, cmd))
            logger.info(out + "\n" + err)
        return (on_file, out + "\n" + err)
    else:
        if interactive:
            logger.info("File %s has no pylint problems" % on_file)
        return (on_file, "")
Example #11
0
    def test_configure(self):
        testname = "SMS.f09_g16.X"
        casedir = self._create_test([testname, "--no-build"],
                                    test_id=self._baseline_name)

        manual_config_dir = os.path.join(casedir, "manual_config")
        os.mkdir(manual_config_dir)

        utils.run_cmd_no_fail(
            "{} --machine={} --compiler={}".format(
                os.path.join(utils.get_cime_root(), "tools", "configure"),
                self._machine,
                self._compiler,
            ),
            from_dir=manual_config_dir,
        )

        with open(os.path.join(casedir, "env_mach_specific.xml"), "r") as fd:
            case_env_contents = fd.read()

        with open(os.path.join(manual_config_dir, "env_mach_specific.xml"),
                  "r") as fd:
            man_env_contents = fd.read()

        self.assertEqual(case_env_contents, man_env_contents)
Example #12
0
    def __init__(self, comp_interface="mct"):
        """
        initialize an object

        >>> files = Files()
        >>> files.get_value('CASEFILE_HEADERS',resolved=False)
        '$CIMEROOT/config/config_headers.xml'
        """
        cimeroot = get_cime_root()
        infile = os.path.join(cimeroot, "config", get_model(),
                              "config_files.xml")
        expect(os.path.isfile(infile),
               "Could not find or open file {}".format(infile))
        schema = os.path.join(cimeroot, "config", "xml_schemas",
                              "entry_id.xsd")
        EntryID.__init__(self, infile, schema=schema)
        config_files_override = os.path.join(os.path.dirname(cimeroot),
                                             ".config_files.xml")
        # variables COMP_ROOT_DIR_{} are mutable, all other variables are read only
        self.COMP_ROOT_DIR = {}
        self._comp_interface = comp_interface
        # .config_file.xml at the top level may overwrite COMP_ROOT_DIR_ nodes in config_files

        if os.path.isfile(config_files_override):
            self.read(config_files_override)
            self.overwrite_existing_entries()
Example #13
0
    def __init__(self, machobj, schema_path=None):
        """Construct a Build given machine-specific information.

        In the process some information about possible variables is read in
        from the schema file.

        Arguments:
        machobj - A Machines object for this machine.
        schema_path (optional) - Path to config_build.xsd within CIME.

        >>> "CFLAGS" in Build('MyMach').flag_vars
        True
        >>> "MPICC" in Build('MyMach').flag_vars
        False
        """
        self.machobj = machobj

        # The schema is used to figure out which variables contain
        # command-line arguments (e.g. compiler flags), since these are
        # processed in a more complex manner than other variables.
        if schema_path is None:
            schema_path = os.path.join(get_cime_root(), "cime_config",
                                       "xml_schemas", "config_build.xsd")

        # Run an XPath query to extract the list of flag variable names.
        ns = {"xs": "http://www.w3.org/2001/XMLSchema"}
        flag_xpath = ".//xs:group[@name='compilerVars']/xs:choice/xs:element[@type='flagsVar']"
        flag_elems = ET.parse(schema_path).getroot().findall(flag_xpath, ns)
        self.flag_vars = set(elem.get('name') for elem in flag_elems)
Example #14
0
    def get_resolved_value(self, raw_value):
        """
        A value in the xml file may contain references to other xml
        variables or to environment variables. These are refered to in
        the perl style with $name and $ENV{name}.

        >>> obj = GenericXML()
        >>> os.environ["FOO"] = "BAR"
        >>> os.environ["BAZ"] = "BARF"
        >>> obj.get_resolved_value("one $ENV{FOO} two $ENV{BAZ} three")
        'one BAR two BARF three'
        """
        logger.debug("raw_value %s" % raw_value)
        reference_re = re.compile(r'\${?(\w+)}?')
        env_ref_re   = re.compile(r'\$ENV\{(\w+)\}')
        item_data = raw_value

        if item_data is None:
            return None

        if type(item_data) is not str:
            return item_data

        for m in env_ref_re.finditer(item_data):
            logger.debug("look for %s in env" % item_data)
            env_var = m.groups()[0]
            expect(env_var in os.environ, "Undefined env var '%s'" % env_var)
            item_data = item_data.replace(m.group(), os.environ[env_var])

        for m in reference_re.finditer(item_data):
            var = m.groups()[0]
            logger.debug("find: %s" % var)
            ref = self.get_value(var)
            if ref is not None:
                logger.debug("resolve: " + str(ref))
                item_data = item_data.replace(m.group(), self.get_resolved_value(str(ref)))
            elif var == "CIMEROOT":
                cimeroot = get_cime_root()
                item_data = item_data.replace(m.group(), cimeroot)
            elif var == "SRCROOT":
                srcroot = os.path.join(get_cime_root(),"..")
                item_data = item_data.replace(m.group(), srcroot)
            elif var in os.environ:
                logging.debug("resolve from env: " + var)
                item_data = item_data.replace(m.group(), os.environ[var])

        return item_data
Example #15
0
    def __init__(self, infile=None):
        """
        initialize an object
        """
        if infile is None:
            infile = os.path.join(get_cime_root(), "cime_config", get_model(), "archive.xml")

        GenericXML.__init__(self, infile)
Example #16
0
def get_all_checkable_files():
###############################################################################
    cimeroot = get_cime_root()
    all_git_files = run_cmd_no_fail("git ls-files", from_dir=cimeroot, verbose=False).splitlines()

    files_to_test = [item for item in all_git_files
                     if ((item.endswith(".py") or is_python_executable(os.path.join(cimeroot, item))) and not _should_pylint_skip(item))]
    return files_to_test
Example #17
0
def get_all_checkable_files():
###############################################################################
    cimeroot = get_cime_root()
    all_git_files = run_cmd_no_fail("git ls-files", from_dir=cimeroot, verbose=False).splitlines()

    files_to_test = [item for item in all_git_files
                     if ((item.endswith(".py") or is_python_executable(os.path.join(cimeroot, item))) and not _should_pylint_skip(item))]
    return files_to_test
Example #18
0
    def __init__(self, infile=None):
        """
        initialize an object
        """
        if infile is None:
            infile = os.path.join(get_cime_root(), "cime_config", get_model(),
                                  "config_archive.xml")

        GenericXML.__init__(self, infile)
Example #19
0
 def __init__(self, case_root=None, infile="env_batch.xml", read_only=False):
     """
     initialize an object interface to file env_batch.xml in the case directory
     """
     self._batchtype = None
     # This arbitrary setting should always be overwritten
     self._default_walltime = "00:20:00"
     schema = os.path.join(get_cime_root(), "config", "xml_schemas", "env_batch.xsd")
     super(EnvBatch,self).__init__(case_root, infile, schema=schema, read_only=read_only)
Example #20
0
 def __init__(self, caseroot=None, infile="env_mach_specific.xml",
              components=None, unit_testing=False, read_only=False):
     """
     initialize an object interface to file env_mach_specific.xml in the case directory
     """
     schema = os.path.join(get_cime_root(), "config", "xml_schemas", "env_mach_specific.xsd")
     EnvBase.__init__(self, caseroot, infile, schema=schema, read_only=read_only)
     self._allowed_mpi_attributes = ("compiler", "mpilib", "threaded", "unit_testing", "queue")
     self._unit_testing = unit_testing
Example #21
0
 def __init__(self, case_root=None, infile="env_batch.xml", read_only=False):
     """
     initialize an object interface to file env_batch.xml in the case directory
     """
     self._batchtype = None
     # This arbitrary setting should always be overwritten
     self._default_walltime = "00:20:00"
     schema = os.path.join(get_cime_root(), "config", "xml_schemas", "env_batch.xsd")
     super(EnvBatch,self).__init__(case_root, infile, schema=schema, read_only=read_only)
Example #22
0
 def __init__(self, caseroot=None, infile="env_mach_specific.xml",
              components=None, unit_testing=False):
     """
     initialize an object interface to file env_mach_specific.xml in the case directory
     """
     schema = os.path.join(get_cime_root(), "config", "xml_schemas", "env_mach_specific.xsd")
     EnvBase.__init__(self, caseroot, infile, schema=schema)
     self._allowed_mpi_attributes = ("compiler", "mpilib", "threaded", "unit_testing")
     self._unit_testing = unit_testing
 def __init__(self, infile, attributes=None):
     """Construct a `NamelistDefinition` from an XML file."""
     super(NamelistDefinition, self).__init__(infile)
     self._attributes = attributes
     # if the file is invalid we may not be able to check the version
     # but we need to do it this way until we remove the version 1 files
     if self.get_version() == "2.0":
         cimeroot = get_cime_root()
         schema = os.path.join(cimeroot,"cime_config","xml_schemas","entry_id_namelist.xsd")
         self.validate_xml_file(infile, schema)
Example #24
0
 def __init__(self, case_root=None, infile="env_batch.xml"):
     """
     initialize an object interface to file env_batch.xml in the case directory
     """
     self.prereq_jobid = None
     self.batchtype = None
     # This arbitrary setting should always be overwritten
     self._default_walltime = "00:20:00"
     schema = os.path.join(get_cime_root(), "config", "xml_schemas", "env_batch.xsd")
     EnvBase.__init__(self, case_root, infile, schema=schema)
Example #25
0
 def __init__(self, case_root=None, infile="env_batch.xml"):
     """
     initialize an object interface to file env_batch.xml in the case directory
     """
     self._prereq_jobid = None
     self._batchtype = None
     # This arbitrary setting should always be overwritten
     self._default_walltime = "00:20:00"
     schema = os.path.join(get_cime_root(), "config", "xml_schemas", "env_batch.xsd")
     EnvBase.__init__(self, case_root, infile, schema=schema)
Example #26
0
    def __init__(self, machine, infile=None):
        """
        initialize an object
        """
        if (infile is None):
            infile = os.path.join(get_cime_root(), "cime_config", get_model(), "machines", "config_lt_archive.xml")

        GenericXML.__init__(self, infile)

        self.machine = machine
Example #27
0
    def __init__(self):
        """
        initialize an object

        >>> files = Files()
        >>> files.get_value('CASEFILE_HEADERS',resolved=False)
        '$CIMEROOT/cime_config/config_headers.xml'
        """
        infile = os.path.join(get_cime_root(), "cime_config", get_model(), "config_files.xml")
        EntryID.__init__(self, infile)
Example #28
0
 def __init__(self, infile):
     """
     initialize an object
     """
     files = Files()
     schema = None
     # not checking schema on external components yet
     cimeroot = get_cime_root()
     if cimeroot in os.path.abspath(infile):
         schema = files.get_schema("CONFIG_CPL_FILE")
     EntryID.__init__(self, infile, schema=schema)
Example #29
0
    def __init__(self):
        """
        initialize an object

        >>> files = Files()
        >>> files.get_value('CASEFILE_HEADERS',resolved=False)
        '$CIMEROOT/cime_config/config_headers.xml'
        """
        infile = os.path.join(get_cime_root(), "cime_config", get_model(),
                              "config_files.xml")
        EntryID.__init__(self, infile)
Example #30
0
 def __init__(self, infile, attributes=None):
     """Construct a `NamelistDefinition` from an XML file."""
     super(NamelistDefinition, self).__init__(infile)
     self._attributes = attributes
     # if the file is invalid we may not be able to check the version
     # but we need to do it this way until we remove the version 1 files
     if self.get_version() == "2.0":
         cimeroot = get_cime_root()
         schema = os.path.join(cimeroot, "cime_config", "xml_schemas",
                               "entry_id_namelist.xsd")
         self.validate_xml_file(infile, schema)
Example #31
0
def _run_pylint(all_files, interactive):
    ###############################################################################
    pylint = find_executable("pylint")

    cmd_options = (
        " --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import"
    )
    cmd_options += (
        ",fixme,broad-except,bare-except,eval-used,exec-used,global-statement")
    cmd_options += ",logging-format-interpolation,no-name-in-module,arguments-renamed"
    cmd_options += " -j 0 -f json"
    cimeroot = get_cime_root()
    srcroot = get_src_root()

    # if "scripts/Tools" in on_file:
    #     cmd_options +=",relative-import"

    # add init-hook option
    cmd_options += ' --init-hook=\'sys.path.extend(("%s","%s","%s","%s"))\'' % (
        os.path.join(cimeroot, "CIME"),
        os.path.join(cimeroot, "CIME", "Tools"),
        os.path.join(cimeroot, "scripts", "fortran_unit_testing", "python"),
        os.path.join(srcroot, "components", "cmeps", "cime_config", "runseq"),
    )

    files = " ".join(all_files)
    cmd = "%s %s %s" % (pylint, cmd_options, files)
    logger.debug("pylint command is %s" % cmd)
    stat, out, err = run_cmd(cmd, verbose=False, from_dir=cimeroot)

    data = json.loads(out)

    result = {}

    for item in data:
        if item["type"] != "error":
            continue

        path = item["path"]
        message = item["message"]
        line = item["line"]

        if path in result:
            result[path].append(f"{message}:{line}")
        else:
            result[path] = [
                message,
            ]

    for k in result.keys():
        result[k] = "\n".join(set(result[k]))

    return result
Example #32
0
    def __init__(self, infile):
        """
        initialize an object
        """
        files = Files()
        schema = None
        # not checking schema on external components yet
        cimeroot = get_cime_root()
        if  cimeroot in os.path.abspath(infile):
            schema = files.get_schema("CONFIG_DRV_FILE")

        EntryID.__init__(self, infile, schema=schema)
Example #33
0
 def create_new_case(self):
     srcroot = os.path.dirname(os.path.abspath(get_cime_root()))
     casename = self.CreateNewcaseArgs["case"]
     with Case(os.path.join(self._casedir, casename),
               read_only=False) as case:
         case.create(casename,
                     srcroot,
                     self.CreateNewcaseArgs["compset"],
                     self.CreateNewcaseArgs["res"],
                     machine_name=self.CreateNewcaseArgs["machine"],
                     compiler=self.CreateNewcaseArgs["compiler"],
                     mpilib=self.CreateNewcaseArgs["mpilib"])
Example #34
0
    def __init__(self, infile):
        """
        initialize an object
        """
        files = Files()
        schema = files.get_schema("CONFIG_DRV_FILE")
        if schema is not None:
            # not checking schema on external components yet
            cimeroot = get_cime_root()
            if cimeroot in os.path.abspath(infile):
                self.validate_xml_file(infile, schema)

        EntryID.__init__(self, infile)
Example #35
0
    def __init__(self):
        """
        initialize an object

        >>> files = Files()
        >>> files.get_value('CASEFILE_HEADERS',resolved=False)
        '$CIMEROOT/config/config_headers.xml'
        """
        cimeroot = get_cime_root()
        infile = os.path.join(cimeroot, "config", get_model(), "config_files.xml")
        expect(os.path.isfile(infile), "Could not find or open file {}".format(infile))
        schema = os.path.join(cimeroot, "config", "xml_schemas", "entry_id.xsd")
        EntryID.__init__(self, infile, schema=schema)
Example #36
0
    def submit_single_job(self, case, job, depid=None, no_batch=False):
        logger.warn("Submit job %s"%job)
        caseroot = case.get_value("CASEROOT")
        batch_system = self.get_value("BATCH_SYSTEM", subgroup=None)
        if batch_system is None or batch_system == "none" or no_batch:
            # Import here to avoid circular include
            from CIME.case_test       import case_test # pylint: disable=unused-variable
            from CIME.case_run        import case_run # pylint: disable=unused-variable
            from CIME.case_st_archive import case_st_archive # pylint: disable=unused-variable
            from CIME.case_lt_archive import case_lt_archive # pylint: disable=unused-variable

            logger.info("Starting job script %s" % job)

            # Hack until all testcases are ported to python
            testcase = case.get_value("TESTCASE")
            cimeroot = get_cime_root()
            testscript = os.path.join(cimeroot, "scripts", "Testing", "Testcases", "%s_script" % testcase)
            if job == "case.test" and testcase is not None and os.path.exists(testscript):
                run_cmd_no_fail("%s --caseroot %s" % (os.path.join(".", job), caseroot))
            else:
                # This is what we want longterm
                function_name = job.replace(".", "_")
                success = locals()[function_name](case)
                expect(success, "%s failed" % function_name)

            return

        submitargs = self.get_submit_args(case, job)

        if depid is not None:
            dep_string = self.get_value("depend_string", subgroup=None)
            dep_string = dep_string.replace("jobid",depid.strip()) # pylint: disable=maybe-no-member
            submitargs += " " + dep_string

        batchsubmit = self.get_value("batch_submit", subgroup=None)
        expect(batchsubmit is not None,
               "Unable to determine the correct command for batch submission.")
        batchredirect = self.get_value("batch_redirect", subgroup=None)
        submitcmd = ''
        for string in (batchsubmit, submitargs, batchredirect, job):
            if  string is not None:
                submitcmd += string + " "

        logger.info("Submitting job script %s"%submitcmd)
        output = run_cmd_no_fail(submitcmd)
        jobid = self.get_job_id(output)
        logger.debug("Submitted job id is %s"%jobid)
        return jobid
Example #37
0
    def __init__(self, batch_system=None, machine=None, infile=None):
        """
        initialize an object
        """
        if infile is None:
            infile = os.path.join(get_cime_root(), "cime_config", get_model(), "machines", "config_batch.xml")

        GenericXML.__init__(self, infile)

        self.batch_system_node = None
        self.machine_node      = None
        self.batch_system      = batch_system
        self.machine           = machine

        if self.batch_system is not None:
            self.set_batch_system(self.batch_system, machine=machine)
Example #38
0
    def __init__(self):
        """
        initialize an object

        >>> files = Files()
        >>> files.get_value('CASEFILE_HEADERS',resolved=False)
        '$CIMEROOT/config/config_headers.xml'
        """
        cimeroot = get_cime_root()
        infile = os.path.join(cimeroot, "config", get_model(),
                              "config_files.xml")
        expect(os.path.isfile(infile),
               "Could not find or open file {}".format(infile))
        schema = os.path.join(cimeroot, "config", "xml_schemas",
                              "entry_id.xsd")
        EntryID.__init__(self, infile, schema=schema)
Example #39
0
def check_code(files, num_procs=10, interactive=False):
    ###############################################################################
    """
    Check all python files in the given directory

    Returns True if all files had no problems
    """
    # Get list of files to check, we look to see if user-provided file argument
    # is a valid file, if not, we search the repo for a file with similar name.
    files_to_check = []
    if files:
        repo_files = run_cmd_no_fail('git ls-files',
                                     from_dir=get_cime_root(),
                                     verbose=False).splitlines()
        for filearg in files:
            if os.path.exists(filearg):
                files_to_check.append(os.path.abspath(filearg))
            else:
                found = False
                for repo_file in repo_files:
                    if repo_file.endswith(filearg):
                        found = True
                        files_to_check.append(
                            repo_file)  # could have multiple matches

                if not found:
                    logger.warning(
                        "Could not find file matching argument '%s'" % filearg)
    else:
        # Check every python file
        files_to_check = get_all_checkable_files()

    if "scripts/lib/six.py" in files_to_check:
        files_to_check.remove("scripts/lib/six.py")
        logger.info("Not checking contributed file six.py")

    expect(len(files_to_check) > 0, "No matching files found")

    # No point in using more threads than files
    if len(files_to_check) < num_procs:
        num_procs = len(files_to_check)

    pool = ThreadPool(num_procs)
    results = pool.map(lambda x: _run_pylint(x, interactive), files_to_check)
    pool.close()
    pool.join()
    return dict(results)
    def test_m_createnewcase_alternate_drivers(self):
        # Test that case.setup runs for nuopc and moab drivers
        cls = self.__class__
        model = utils.get_model()
        for driver in ("nuopc", "moab"):
            if not os.path.exists(
                os.path.join(utils.get_cime_root(), "src", "drivers", driver)
            ):
                self.skipTest(
                    "Skipping driver test for {}, driver not found".format(driver)
                )
            if (model == "cesm" and driver == "moab") or (
                model == "e3sm" and driver == "nuopc"
            ):
                continue

            testdir = os.path.join(cls._testroot, "testcreatenewcase.{}".format(driver))
            if os.path.exists(testdir):
                shutil.rmtree(testdir)
            args = " --driver {} --case {} --compset X --res f19_g16 --output-root {} --handle-preexisting-dirs=r".format(
                driver, testdir, cls._testroot
            )
            if model == "cesm":
                args += " --run-unsupported"
            if self.TEST_COMPILER is not None:
                args = args + " --compiler %s" % self.TEST_COMPILER
            if self.TEST_MPILIB is not None:
                args = args + " --mpilib %s" % self.TEST_MPILIB

            args += f" --machine {self.MACHINE.get_machine_name()}"

            cls._testdirs.append(testdir)
            self.run_cmd_assert_result(
                "./create_newcase %s" % (args), from_dir=self.SCRIPT_DIR
            )
            self.assertTrue(os.path.exists(testdir))
            self.assertTrue(os.path.exists(os.path.join(testdir, "case.setup")))

            self.run_cmd_assert_result("./case.setup", from_dir=testdir)
            with Case(testdir, read_only=False) as case:
                comp_interface = case.get_value("COMP_INTERFACE")
                self.assertTrue(
                    driver == comp_interface, msg="%s != %s" % (driver, comp_interface)
                )

            cls._do_teardown.append(testdir)
Example #41
0
    def run_phase(self):

        rundir   = self._case.get_value("RUNDIR")
        exeroot  = self._case.get_value("EXEROOT")
        mach     = self._case.get_value("MACH")

        log = os.path.join(rundir, "funit.log")
        if os.path.exists(log):
            os.remove(log)

        test_spec_dir = self.get_test_spec_dir()
        unit_test_tool = os.path.abspath(os.path.join(get_cime_root(),"scripts","fortran_unit_testing","run_tests.py"))
        args = "--build-dir {} --test-spec-dir {} --machine {}".format(exeroot, test_spec_dir, mach)
        stat = run_cmd("{} {} >& funit.log".format(unit_test_tool, args), from_dir=rundir)[0]

        append_testlog(open(os.path.join(rundir, "funit.log"), "r").read())

        expect(stat == 0, "RUN FAIL for FUNIT")
    def test_gen_domain(self):
        if utils.get_model() != "e3sm":
            self.skipTest("Skipping gen_domain test. Depends on E3SM tools")
        cime_root = utils.get_cime_root()
        inputdata = self.MACHINE.get_value("DIN_LOC_ROOT")

        tool_name = "test_gen_domain"
        tool_location = os.path.join(cime_root, "tools", "mapping",
                                     "gen_domain_files", "test_gen_domain.sh")
        args = "--cime_root={} --inputdata_root={}".format(
            cime_root, inputdata)

        cls = self.__class__
        test_dir = os.path.join(cls._testroot, tool_name)
        cls._testdirs.append(test_dir)
        os.makedirs(test_dir)
        self.run_cmd_assert_result("{} {}".format(tool_location, args),
                                   from_dir=test_dir)
        cls._do_teardown.append(test_dir)
Example #43
0
def check_code(files, num_procs=10, interactive=False):
###############################################################################
    """
    Check all python files in the given directory

    Returns True if all files had no problems
    """
    # Get list of files to check, we look to see if user-provided file argument
    # is a valid file, if not, we search the repo for a file with similar name.
    files_to_check = []
    if files:
        repo_files = run_cmd_no_fail('git ls-files', from_dir=get_cime_root(), verbose=False).splitlines()
        for filearg in files:
            if os.path.exists(filearg):
                files_to_check.append(os.path.abspath(filearg))
            else:
                found = False
                for repo_file in repo_files:
                    if repo_file.endswith(filearg):
                        found = True
                        files_to_check.append(repo_file) # could have multiple matches

                if not found:
                    logger.warning("Could not find file matching argument '%s'" % filearg)
    else:
        # Check every python file
        files_to_check = get_all_checkable_files()

    if "scripts/lib/six.py" in files_to_check:
        files_to_check.remove("scripts/lib/six.py")
        logger.info("Not checking contributed file six.py")

    expect(len(files_to_check) > 0, "No matching files found")

    # No point in using more threads than files
    if len(files_to_check) < num_procs:
        num_procs = len(files_to_check)

    pool = ThreadPool(num_procs)
    results = pool.map(lambda x : _run_pylint(x, interactive), files_to_check)
    pool.close()
    pool.join()
    return dict(results)
Example #44
0
 def test_a_unit_test(self):
     cls = self.__class__
     if not self._has_unit_test_support():
         self.skipTest(
             "Skipping TestUnitTest - PFUNIT_PATH not found for the default compiler on this machine"
         )
     test_dir = os.path.join(cls._testroot, "unit_tester_test")
     cls._testdirs.append(test_dir)
     os.makedirs(test_dir)
     unit_test_tool = os.path.abspath(
         os.path.join(utils.get_cime_root(), "scripts",
                      "fortran_unit_testing", "run_tests.py"))
     test_spec_dir = os.path.join(os.path.dirname(unit_test_tool),
                                  "Examples", "interpolate_1d", "tests")
     args = "--build-dir {} --test-spec-dir {}".format(
         test_dir, test_spec_dir)
     args += " --machine {}".format(self.MACHINE.get_machine_name())
     utils.run_cmd_no_fail("{} {}".format(unit_test_tool, args))
     cls._do_teardown.append(test_dir)
Example #45
0
    def __init__(self, case_root=None, read_only=True):

        if case_root is None:
            case_root = os.getcwd()
        self._caseroot = case_root
        logger.debug("Initializing Case.")
        self._env_files_that_need_rewrite = set()
        self._read_only_mode = True
        self._force_read_only = read_only

        self._env_entryid_files = []
        self._env_generic_files = []
        self._files = []

        self.read_xml()

        # Hold arbitary values. In create_newcase we may set values
        # for xml files that haven't been created yet. We need a place
        # to store them until we are ready to create the file. At file
        # creation we get the values for those fields from this lookup
        # table and then remove the entry.
        self.lookups = {}
        self.set_lookup_value('CIMEROOT',os.path.abspath(get_cime_root()))

        self._compsetname = None
        self._gridname = None
        self._compsetsfile = None
        self._pesfile = None
        self._gridfile = None
        self._components = []
        self._component_classes = []
        self._is_env_loaded = False

        self.thread_count = None
        self.tasks_per_node = None
        self.num_nodes = None
        self.tasks_per_numa = None
        self.cores_per_task = None

        # check if case has been configured and if so initialize derived
        if self.get_value("CASEROOT") is not None:
            self.initialize_derived_attributes()
Example #46
0
    def __init__(self, case_root=None, read_only=True):

        if case_root is None:
            case_root = os.getcwd()
        self._caseroot = case_root
        logger.debug("Initializing Case.")
        self._env_files_that_need_rewrite = set()
        self._read_only_mode = True
        self._force_read_only = read_only

        self._env_entryid_files = []
        self._env_generic_files = []
        self._files = []

        self.read_xml()

        # Hold arbitary values. In create_newcase we may set values
        # for xml files that haven't been created yet. We need a place
        # to store them until we are ready to create the file. At file
        # creation we get the values for those fields from this lookup
        # table and then remove the entry.
        self.lookups = {}
        self.set_lookup_value('CIMEROOT', os.path.abspath(get_cime_root()))

        self._compsetname = None
        self._gridname = None
        self._compsetsfile = None
        self._pesfile = None
        self._gridfile = None
        self._components = []
        self._component_classes = []
        self._is_env_loaded = False

        self.thread_count = None
        self.tasks_per_node = None
        self.num_nodes = None
        self.tasks_per_numa = None
        self.cores_per_task = None

        # check if case has been configured and if so initialize derived
        if self.get_value("CASEROOT") is not None:
            self.initialize_derived_attributes()
Example #47
0
    def test_b_cime_f90_unit_tests(self):
        cls = self.__class__
        if self.FAST_ONLY:
            self.skipTest("Skipping slow test")

        if not self._has_unit_test_support():
            self.skipTest(
                "Skipping TestUnitTest - PFUNIT_PATH not found for the default compiler on this machine"
            )

        test_dir = os.path.join(cls._testroot, "driver_f90_tests")
        cls._testdirs.append(test_dir)
        os.makedirs(test_dir)
        test_spec_dir = utils.get_cime_root()
        unit_test_tool = os.path.abspath(
            os.path.join(test_spec_dir, "scripts", "fortran_unit_testing",
                         "run_tests.py"))
        args = "--build-dir {} --test-spec-dir {}".format(
            test_dir, test_spec_dir)
        args += " --machine {}".format(self.MACHINE.get_machine_name())
        utils.run_cmd_no_fail("{} {}".format(unit_test_tool, args))
        cls._do_teardown.append(test_dir)
Example #48
0
    def __init__(self, batch_system=None, machine=None, infile=None):
        """
        initialize an object
        """
        if infile is None:
            infile = os.path.join(get_cime_root(), "cime_config", get_model(), "machines", "config_batch.xml")

        GenericXML.__init__(self, infile)

        self.batch_system_node = None
        self.machine_node      = None
        self.batch_system      = batch_system
        self.machine           = machine

        #Append the contents of $HOME/.cime/config_batch.xml if it exists
        #This could cause problems if node matchs are repeated when only one is expected
        infile = os.path.join(os.environ.get("HOME"),".cime","config_batch.xml")
        if os.path.exists(infile):
            GenericXML.read(self, infile)

        if self.batch_system is not None:
            self.set_batch_system(self.batch_system, machine=machine)
Example #49
0
    def run_phase(self):

        rundir = self._case.get_value("RUNDIR")
        exeroot = self._case.get_value("EXEROOT")
        mach = self._case.get_value("MACH")

        log = os.path.join(rundir, "funit.log")
        if os.path.exists(log):
            os.remove(log)

        test_spec_dir = self.get_test_spec_dir()
        unit_test_tool = os.path.abspath(
            os.path.join(get_cime_root(), "scripts", "fortran_unit_testing",
                         "run_tests.py"))
        args = "--build-dir {} --test-spec-dir {} --machine {}".format(
            exeroot, test_spec_dir, mach)
        stat = run_cmd("{} {} >& funit.log".format(unit_test_tool, args),
                       from_dir=rundir)[0]

        append_testlog(open(os.path.join(rundir, "funit.log"), "r").read())

        expect(stat == 0, "RUN FAIL for FUNIT")
Example #50
0
    def __init__(self, case_root=None):

        if case_root is None:
            case_root = os.getcwd()

        # Init first, if no valid case_root expect fails and tears down object, __del__ expects self._env_files_that_need_rewrite
        self._env_files_that_need_rewrite = set()

        logger.debug("Initializing Case.")

        self._env_entryid_files = []
        self._env_entryid_files.append(EnvRun(case_root))
        self._env_entryid_files.append(EnvBuild(case_root))
        self._env_entryid_files.append(EnvMachPes(case_root))
        self._env_entryid_files.append(EnvCase(case_root))
        self._env_entryid_files.append(EnvBatch(case_root))
        if os.path.isfile(os.path.join(case_root,"env_test.xml")):
            self._env_entryid_files.append(EnvTest(case_root))
        self._env_generic_files = []
        self._env_generic_files.append(EnvMachSpecific(case_root))
        self._env_generic_files.append(EnvArchive(case_root))
        self._files = self._env_entryid_files + self._env_generic_files

        # Hold arbitary values. In create_newcase we may set values
        # for xml files that haven't been created yet. We need a place
        # to store them until we are ready to create the file. At file
        # creation we get the values for those fields from this lookup
        # table and then remove the entry. This was what I came up
        # with in the perl anyway and I think that we still need it here.
        self.lookups = {}
        self.lookups['CIMEROOT'] = os.path.abspath(get_cime_root())

        self._compsetname = None
        self._gridname = None
        self._compsetsfile = None
        self._pesfile = None
        self._gridfile = None
        self._components = []
        self._component_config_files = []
Example #51
0
File: files.py Project: Katetc/cime
    def __init__(self):
        """
        initialize an object

        >>> files = Files()
        >>> files.get_value('CASEFILE_HEADERS',resolved=False)
        '$CIMEROOT/config/config_headers.xml'
        """
        cimeroot = get_cime_root()
        infile = os.path.join(cimeroot, "config", get_model(), "config_files.xml")
        expect(os.path.isfile(infile), "Could not find or open file {}".format(infile))
        schema = os.path.join(cimeroot, "config", "xml_schemas", "entry_id.xsd")
        EntryID.__init__(self, infile, schema=schema)
        config_files_override = os.path.join(os.path.dirname(cimeroot),".config_files.xml")
        # variables COMP_ROOT_DIR_{} are mutable, all other variables are read only
        self.COMP_ROOT_DIR = {}

        # .config_file.xml at the top level may overwrite COMP_ROOT_DIR_ nodes in config_files

        if os.path.isfile(config_files_override):
            self.read(config_files_override)
            self.overwrite_existing_entries()
Example #52
0
 def __init__(self, infile=None):
     """
     Initialize an object
     """
     self.tree = None
     self.lookups = {}
     self.lookups["CIMEROOT"] = get_cime_root()
     if infile == None:
         # if file is not defined just return
         self.filename = None
         return
     if os.path.isfile(infile) and os.access(infile, os.R_OK):
         # If file is defined and exists, read it
         self.filename = infile
         self.read(infile)
     else:
         # if file does not exist create a root xml element
         # and set it's id to file
         self.filename = infile
         root = ET.Element("xml")
         root.set("version", "1.0")
         self.root = ET.SubElement(root, "file")
         self.root.set("id", infile)
         self.tree = ET.ElementTree(root)
Example #53
0
import CIME.utils
from CIME.utils import expect, convert_to_seconds, parse_test_name, get_cime_root, get_model
from CIME.XML.machines import Machines
import six, sys, os

# Expect that, if a model wants to use python-based test lists, they will have a file
# config/$model/tests.py , containing a test dictionary called _TESTS

sys.path.insert(0, os.path.join(get_cime_root(), "config", get_model()))
_ALL_TESTS = {}
try:
    from tests import _TESTS # pylint: disable=import-error
    _ALL_TESTS.update(_TESTS)
except:
    pass

# Here are the tests belonging to e3sm suites. Format is
# <test>.<grid>.<compset>.
# suite_name -> (inherits_from, timelimit, [test [, mods[, machines]]])
#   To elaborate, if no mods are needed, a string representing the testname is all that is needed.
#   If testmods are needed, a 2-ple must be provided  (test, mods)
#   If you want to restrict the test mods to certain machines, than a 3-ple is needed (test, mods, [machines])

_CIME_TESTS = {

    "cime_tiny" : (None, "0:10:00",
                   ("ERS.f19_g16_rx1.A",
                    "NCK.f19_g16_rx1.A")
                   ),

    "cime_test_only_pass" : (None, "0:10:00",
Example #54
0
def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False):
###############################################################################
    os.chdir(caseroot)
    msg = "case.setup starting"
    append_status(msg, caseroot=caseroot, sfile="CaseStatus")

    cimeroot = get_cime_root(case)

    # Check that $DIN_LOC_ROOT exists - and abort if not a namelist compare tests
    din_loc_root = case.get_value("DIN_LOC_ROOT")
    testcase     = case.get_value("TESTCASE")
    expect(not (not os.path.isdir(din_loc_root) and testcase != "SBN"),
           "inputdata root is not a directory: \"$din_loc_root\" ")

    # Check that userdefine settings are specified before expanding variable
    for vid, value in case:
        expect(not (type(value) is str and "USERDEFINED_required_build" in value),
               "Parameter '%s' must be defined" % vid)

    # Create batch script
    if reset or clean:
        # Clean batch script

        backup_dir = "PESetupHist/b.%s" % time.strftime("%y%m%d-%H%M%S")
        if not os.path.isdir(backup_dir):
            os.makedirs(backup_dir)

        # back up relevant files
        for fileglob in ["case.run", "env_build.xml", "env_mach_pes.xml", "Macros*"]:
            for filename in glob.glob(fileglob):
                shutil.copy(filename, backup_dir)
        if os.path.exists("case.run"):
            os.remove("case.run")

        # only do the following if are NOT in testmode
        if not test_mode:
            # rebuild the models (even on restart)
            case.set_value("BUILD_COMPLETE", False)

            # backup and then clean test script
            if os.path.exists("case.test"):
                shutil.copy("case.test", backup_dir)
                os.remove("case.test")
                logger.info("Successfully cleaned test script case.test")

            if os.path.exists("case.testdriver"):
                shutil.copy("case.testdriver", backup_dir)
                os.remove("case.testdriver")
                logger.info("Successfully cleaned test script case.testdriver")

        logger.info("Successfully cleaned batch script case.run")

        logger.info("Successfully cleaned batch script case.run")
        logger.info("Some files have been saved to %s" % backup_dir)

        msg = "case.setup clean complete"
        append_status(msg, caseroot=caseroot, sfile="CaseStatus")

    if not clean:
        case.load_env()

        models = case.get_values("COMP_CLASSES")
        mach = case.get_value("MACH")
        compiler = case.get_value("COMPILER")
        debug = case.get_value("DEBUG")
        mpilib = case.get_value("MPILIB")
        sysos = case.get_value("OS")
        expect(mach is not None, "xml variable MACH is not set")

        # creates the Macros.make, Depends.compiler, Depends.machine, Depends.machine.compiler
        # and env_mach_specific.xml if they don't already exist.
        if not os.path.isfile("Macros.make") or not os.path.isfile("env_mach_specific.xml"):
            configure(Machines(machine=mach), caseroot, ["Makefile"], compiler, mpilib, debug, sysos)

        # Set tasks to 1 if mpi-serial library
        if mpilib == "mpi-serial":
            for vid, value in case:
                if vid.startswith("NTASKS_") and value != 1:
                    case.set_value(vid, 1)

        # Check ninst.
        # In CIME there can be multiple instances of each component model (an ensemble) NINST is the instance of that component.
        for comp in models:
            if comp == "DRV":
                continue
            ninst  = case.get_value("NINST_%s" % comp)
            ntasks = case.get_value("NTASKS_%s" % comp)
            if ninst > ntasks:
                if ntasks == 1:
                    case.set_value("NTASKS_%s" % comp, ninst)
                else:
                    expect(False, "NINST_%s value %d greater than NTASKS_%s %d" % (comp, ninst, comp, ntasks))

        if os.path.exists("case.run"):
            logger.info("Machine/Decomp/Pes configuration has already been done ...skipping")
        else:
            _check_pelayouts_require_rebuild(case, models)

            if os.path.exists("LockedFiles/env_build.xml"):
                os.remove("LockedFiles/env_build.xml")

            case.flush()
            check_lockedfiles()
            env_mach_pes = case.get_env("mach_pes")
            pestot = env_mach_pes.get_total_tasks(models)
            logger.debug("at update TOTALPES = %s"%pestot)
            case.set_value("TOTALPES", pestot)
            thread_count = env_mach_pes.get_max_thread_count(models)
            if thread_count > 1:
                case.set_value("BUILD_THREADED", True)

            expect(not (case.get_value("BUILD_THREADED")  and compiler == "nag"),
                   "it is not possible to run with OpenMP if using the NAG Fortran compiler")


            cost_pes = env_mach_pes.get_cost_pes(pestot, thread_count, machine=case.get_value("MACH"))
            case.set_value("COST_PES", cost_pes)

            # create batch file
            logger.info("Creating batch script case.run")

            # Use BatchFactory to get the appropriate instance of a BatchMaker,
            # use it to create our batch scripts
            env_batch = case.get_env("batch")
            num_nodes = env_mach_pes.get_total_nodes(pestot, thread_count)
            tasks_per_node = env_mach_pes.get_tasks_per_node(pestot, thread_count)
            for job in env_batch.get_jobs():
                input_batch_script  = os.path.join(case.get_value("MACHDIR"), env_batch.get_value('template', subgroup=job))
                if job == "case.test" and testcase is not None and not test_mode:
                    logger.info("Writing %s script" % job)
                    testscript = os.path.join(cimeroot, "scripts", "Testing", "Testcases", "%s_script" % testcase)
                    # Short term fix to be removed when csh tests are removed
                    if not os.path.exists(testscript):
                        env_batch.make_batch_script(input_batch_script, job, case, pestot, tasks_per_node, num_nodes, thread_count)
                elif job != "case.test":
                    logger.info("Writing %s script from input template %s" % (job, input_batch_script))
                    env_batch.make_batch_script(input_batch_script, job, case, pestot, tasks_per_node, num_nodes, thread_count)

            # Make a copy of env_mach_pes.xml in order to be able
            # to check that it does not change once case.setup is invoked
            logger.info("Locking file env_mach_pes.xml")
            case.flush()
            logger.debug("at copy TOTALPES = %s"%case.get_value("TOTALPES"))
            shutil.copy("env_mach_pes.xml", "LockedFiles")

        # Create user_nl files for the required number of instances
        if not os.path.exists("user_nl_cpl"):
            logger.info("Creating user_nl_xxx files for components and cpl")
        # loop over models
        for model in models:
            comp = case.get_value("COMP_%s" % model)
            logger.info("Building %s usernl files"%model)
            _build_usernl_files(case, model, comp)
            if comp == "cism":
                run_cmd_no_fail("%s/../components/cism/cime_config/cism.template %s" % (cimeroot, caseroot))

        _build_usernl_files(case, "drv", "cpl")

        # Create needed directories for case
        create_dirs(case)

        logger.info("If an old case build already exists, might want to run \'case.build --clean\' before building")

        # Create test script if appropriate
        # Short term fix to be removed when csh tests are removed
        if os.path.exists("env_test.xml"):
            if not os.path.exists("case.test"):
                logger.info("Starting testcase.setup")
                run_cmd_no_fail("./testcase.setup -caseroot %s" % caseroot)
                logger.info("Finished testcase.setup")

        # some tests need namelists created here (ERP)
        if test_mode:
            create_namelists(case)

        msg = "case.setup complete"
        append_status(msg, caseroot=caseroot, sfile="CaseStatus")

        # Record env information
        env_module = case.get_env("mach_specific")
        env_module.make_env_mach_specific_file(compiler, debug, mpilib, "sh")
        env_module.make_env_mach_specific_file(compiler, debug, mpilib, "csh")
        with open("software_environment.txt", "w") as f:
            f.write(env_module.list_modules())
        run_cmd_no_fail("echo -e '\n' >> software_environment.txt && \
                         env >> software_environment.txt")
Example #55
0
def write_macros_file_v1(macros, compiler, os_, machine, macros_file="Macros", output_format="make"):
    """
    Parse the config_compiler.xml file into a Macros file for the
    given machine and compiler.
    """
    # A few things can be used from environ if not in XML
    for item in ["MPI_PATH", "NETCDF_PATH"]:
        if not item in macros and item in os.environ:
            logger.warning("Setting {} from Environment".format(item))
            macros[item] = os.environ[item]

    with open(macros_file, "w") as fd:
        fd.write(
"""#
# COMPILER={}
# OS={}
# MACH={}
""".format(compiler, os_, machine)
)
        if output_format == "make":
            fd.write("#\n# Makefile Macros \n")

            # print the settings out to the Macros file
            for key, value in sorted(macros.items()):
                if key == "_COND_":
                    pass
                elif key.startswith("ADD_"):
                    fd.write("{}+={}\n\n".format(key[4:], value))
                else:
                    fd.write("{}:={}\n\n".format(key, value))

        elif output_format == "cmake":
            fd.write(
'''#
# cmake Macros generated from $compiler_file
#
set(CMAKE_MODULE_PATH %s)
include(Compilers)
set(CMAKE_C_FLAGS_RELEASE "" CACHE STRING "Flags used by c compiler." FORCE)
set(CMAKE_C_FLAGS_DEBUG "" CACHE STRING "Flags used by c compiler." FORCE)
set(CMAKE_Fortran_FLAGS_RELEASE "" CACHE STRING "Flags used by Fortran compiler." FORCE)
set(CMAKE_Fortran_FLAGS_DEBUG "" CACHE STRING "Flags used by Fortran compiler." FORCE)
set(all_build_types "None Debug Release RelWithDebInfo MinSizeRel")
set(CMAKE_BUILD_TYPE "${CMAKE_BUILD_TYPE}" CACHE STRING "Choose the type of build, options are: ${all_build_types}." FORCE)
''' % os.path.join(get_cime_root(), "src", "CMake"))

            # print the settings out to the Macros file, do it in
            # two passes so that path values appear first in the
            # file.
            for key, value in sorted(macros.items()):
                if key == "_COND_":
                    pass
                else:
                    value = value.replace("(", "{").replace(")", "}")
                    if key.endswith("_PATH"):
                        if value.startswith("$"):
                            value = "$ENV{}".format(value[1:])


                        cmake_var = key.replace("NETCDF_PATH", "NetCDF_PATH").replace("PNETCDF_PATH", "Pnetcdf_PATH")
                        fd.write("set({} {})\n".format(cmake_var, value))
                        fd.write("list(APPEND CMAKE_PREFIX_PATH {})\n\n".format(value))

            for key, value in sorted(macros.items()):
                if key == "_COND_":
                    pass
                else:
                    value = value.replace("(", "{").replace(")", "}")
                    if "CFLAGS" in key or "FFLAGS" in key or "CPPDEFS" in key or "SLIBS" in key or "LDFLAGS" in key:
                        if "shell " in value:
                            components = _get_components(value)

                            idx = 0
                            for is_shell, component in components:
                                component = component.replace("NETCDF", "NetCDF").replace("PNETCDF_PATH", "Pnetcdf_PATH")
                                if is_shell:
                                    fd.write('execute_process(COMMAND {} OUTPUT_VARIABLE TEMP{:d})\n'.format(component, idx))
                                    fd.write('string(REGEX REPLACE "\\n$" "" TEMP{:d} "${{TEMP{:d}}}")\n'.format(idx, idx))
                                else:
                                    fd.write('set(TEMP{:d} "{}")\n'.format(idx, component))

                                idx += 1

                            fd.write('set(TEMP "{}")\n'.format(" ".join(["${{TEMP{:d}}}".format(i) for i in range(idx)])))
                        else:
                            fd.write('set(TEMP "{}")\n'.format(value))

                        if "CFLAGS" in key:
                            fd.write("add_flags(CFLAGS ${TEMP})\n\n")
                        elif "FFLAGS" in key:
                            fd.write("add_flags(FFLAGS ${TEMP})\n\n")
                        elif "CPPDEFS" in key:
                            fd.write("list(APPEND CPPDEFS ${TEMP})\n\n")
                        elif "SLIBS" in key or "LDFLAGS" in key:
                            fd.write("add_flags(LDFLAGS ${TEMP})\n\n")

        # Recursively print the conditionals, combining tests to avoid repetition
        _parse_hash(macros["_COND_"], fd, 0, output_format)
Example #56
0
File: case.py Project: ekluzek/cime
    def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None):

        newcaseroot = os.path.abspath(newcase)
        expect(not os.path.isdir(newcaseroot),
               "New caseroot directory %s already exists" % newcaseroot)
        newcasename = os.path.basename(newcaseroot)
        newcase_cimeroot = os.path.abspath(get_cime_root())

        # create clone from self to case
        clone_cimeroot = self.get_value("CIMEROOT")
        if newcase_cimeroot != clone_cimeroot:
            logger.warning(" case  CIMEROOT is %s " %newcase_cimeroot)
            logger.warning(" clone CIMEROOT is %s " %clone_cimeroot)
            logger.warning(" It is NOT recommended to clone cases from different versions of CIME.")


        # *** create case object as deepcopy of clone object ***
        srcroot = os.path.join(newcase_cimeroot,"..")
        newcase = self.copy(newcasename, newcaseroot, newsrcroot=srcroot)
        newcase.set_value("CIMEROOT", newcase_cimeroot)

        # determine if will use clone executable or not
        if keepexe:
            orig_exeroot = self.get_value("EXEROOT")
            newcase.set_value("EXEROOT", orig_exeroot)
            newcase.set_value("BUILD_COMPLETE","TRUE")
        else:
            newcase.set_value("BUILD_COMPLETE","FALSE")

        # set machdir
        if mach_dir is not None:
            newcase.set_value("MACHDIR", mach_dir)

        # Set project id
        # Note: we do not just copy this from the clone because it seems likely that
        # users will want to change this sometimes, especially when cloning another
        # user's case. However, note that, if a project is not given, the fallback will
        # be to copy it from the clone, just like other xml variables are copied.
        if project is None:
            project = self.get_value("PROJECT", subgroup="case.run")
        if project is not None:
            newcase.set_value("PROJECT", project)

        # create caseroot
        newcase.create_caseroot(clone=True)
        newcase.flush(flushall=True)

        # copy user_nl_files
        cloneroot = self._caseroot
        files = glob.glob(cloneroot + '/user_nl_*')
        for item in files:
            shutil.copy(item, newcaseroot)

        # copy SourceMod and Buildconf files
        for casesub in ("SourceMods", "Buildconf"):
            shutil.copytree(os.path.join(cloneroot, casesub), os.path.join(newcaseroot, casesub))

        # copy env_case.xml to LockedFiles
        shutil.copy(os.path.join(newcaseroot,"env_case.xml"), os.path.join(newcaseroot,"LockedFiles"))

        # Update README.case
        fclone   = open(cloneroot + "/README.case", "r")
        fnewcase = open(newcaseroot  + "/README.case", "a")
        fnewcase.write("\n    *** original clone README follows ****")
        fnewcase.write("\n " +  fclone.read())

        clonename = self.get_value("CASE")
        logger.info(" Successfully created new case %s from clone case %s " %(newcasename, clonename))

        case_setup(newcase, clean=False, test_mode=False)

        return newcase
Example #57
0
    def __init__(self, test_names, test_data=None,
                 no_run=False, no_build=False, no_setup=False, no_batch=None,
                 test_root=None, test_id=None,
                 machine_name=None, compiler=None,
                 baseline_root=None, baseline_cmp_name=None, baseline_gen_name=None,
                 clean=False, namelists_only=False,
                 project=None, parallel_jobs=None,
                 walltime=None, proc_pool=None,
                 use_existing=False, save_timing=False, queue=None,
                 allow_baseline_overwrite=False, output_root=None,
                 force_procs=None, force_threads=None, mpilib=None,
                 input_dir=None, pesfile=None, mail_user=None, mail_type=None, allow_pnl=False, non_local=False):
    ###########################################################################
        self._cime_root       = get_cime_root()
        self._cime_model      = get_model()
        self._cime_driver     = "mct"
        self._save_timing     = save_timing
        self._queue           = queue
        self._test_data       = {} if test_data is None else test_data # Format:  {test_name -> {data_name -> data}}
        self._mpilib          = mpilib  # allow override of default mpilib
        self._completed_tests = 0
        self._input_dir       = input_dir
        self._pesfile         = pesfile
        self._allow_baseline_overwrite = allow_baseline_overwrite
        self._allow_pnl       = allow_pnl
        self._non_local       = non_local

        self._mail_user = mail_user
        self._mail_type = mail_type

        self._machobj = Machines(machine=machine_name)

        self._model_build_cost = 4

        # If user is forcing procs or threads, re-write test names to reflect this.
        if force_procs or force_threads:
            test_names = _translate_test_names_for_new_pecount(test_names, force_procs, force_threads)

        self._no_setup = no_setup
        self._no_build = no_build or no_setup or namelists_only
        self._no_run   = no_run or self._no_build
        self._output_root = output_root
        # Figure out what project to use
        if project is None:
            self._project = get_project()
            if self._project is None:
                self._project = self._machobj.get_value("PROJECT")
        else:
            self._project = project

        # We will not use batch system if user asked for no_batch or if current
        # machine is not a batch machine
        self._no_batch = no_batch or not self._machobj.has_batch_system()
        expect(not (self._no_batch and self._queue is not None),
               "Does not make sense to request a queue without batch system")

        # Determine and resolve test_root
        if test_root is not None:
            self._test_root = test_root
        elif self._output_root is not None:
            self._test_root = self._output_root
        else:
            self._test_root = self._machobj.get_value("CIME_OUTPUT_ROOT")

        if self._project is not None:
            self._test_root = self._test_root.replace("$PROJECT", self._project)

        self._test_root = os.path.abspath(self._test_root)
        self._test_id   = test_id if test_id is not None else get_timestamp()

        self._compiler = self._machobj.get_default_compiler() if compiler is None else compiler

        self._clean          = clean
        self._namelists_only = namelists_only

        self._walltime = walltime

        if parallel_jobs is None:
            self._parallel_jobs = min(len(test_names),
                                      self._machobj.get_value("MAX_MPITASKS_PER_NODE"))
        else:
            self._parallel_jobs = parallel_jobs

        self._baseline_cmp_name = baseline_cmp_name # Implies comparison should be done if not None
        self._baseline_gen_name = baseline_gen_name # Implies generation should be done if not None

        # Compute baseline_root
        self._baseline_root = baseline_root if baseline_root is not None \
                              else self._machobj.get_value("BASELINE_ROOT")

        if self._project is not None:
            self._baseline_root = self._baseline_root.replace("$PROJECT", self._project)

        self._baseline_root = os.path.abspath(self._baseline_root)

        if baseline_cmp_name or baseline_gen_name:
            if self._baseline_cmp_name:
                full_baseline_dir = os.path.join(self._baseline_root, self._baseline_cmp_name)
                expect(os.path.isdir(full_baseline_dir),
                       "Missing baseline comparison directory {}".format(full_baseline_dir))

            # the following is to assure that the existing generate directory is not overwritten
            if self._baseline_gen_name:
                full_baseline_dir = os.path.join(self._baseline_root, self._baseline_gen_name)
                existing_baselines = []
                for test_name in test_names:
                    test_baseline = os.path.join(full_baseline_dir, test_name)
                    if os.path.isdir(test_baseline):
                        existing_baselines.append(test_baseline)

                expect(allow_baseline_overwrite or len(existing_baselines) == 0,
                       "Baseline directories already exists {}\n" \
                       "Use -o to avoid this error".format(existing_baselines))

        if self._cime_model == "e3sm":
            _order_tests_by_runtime(test_names, self._baseline_root)

        # This is the only data that multiple threads will simultaneously access
        # Each test has it's own value and setting/retrieving items from a dict
        # is atomic, so this should be fine to use without mutex.
        # name -> (phase, status)
        self._tests = OrderedDict()
        for test_name in test_names:
            self._tests[test_name] = (TEST_START, TEST_PASS_STATUS)

        # Oversubscribe by 1/4
        if proc_pool is None:
            pes = int(self._machobj.get_value("MAX_TASKS_PER_NODE"))
            self._proc_pool = int(pes * 1.25)
        else:
            self._proc_pool = int(proc_pool)

        self._procs_avail = self._proc_pool

        # Setup phases
        self._phases = list(PHASES)
        if self._no_setup:
            self._phases.remove(SETUP_PHASE)
        if self._no_build:
            self._phases.remove(SHAREDLIB_BUILD_PHASE)
            self._phases.remove(MODEL_BUILD_PHASE)
        if self._no_run:
            self._phases.remove(RUN_PHASE)

        if use_existing:
            for test in self._tests:
                with TestStatus(self._get_test_dir(test)) as ts:
                    for phase, status in ts:
                        if phase in CORE_PHASES:
                            if status in [TEST_PEND_STATUS, TEST_FAIL_STATUS]:
                                if status == TEST_FAIL_STATUS:
                                    # Import for potential subsequent waits
                                    ts.set_status(phase, TEST_PEND_STATUS)

                                # We need to pick up here
                                break

                            else:
                                if phase != SUBMIT_PHASE:
                                    # Somewhat subtle. Create_test considers submit/run to be the run phase,
                                    # so don't try to update test status for a passed submit phase
                                    self._update_test_status(test, phase, TEST_PEND_STATUS)
                                    self._update_test_status(test, phase, status)

                                    if phase == RUN_PHASE:
                                        logger.info("Test {} passed and will not be re-run".format(test))

                logger.info("Using existing test directory {}".format(self._get_test_dir(test)))
        else:
            # None of the test directories should already exist.
            for test in self._tests:
                expect(not os.path.exists(self._get_test_dir(test)),
                       "Cannot create new case in directory '{}', it already exists."
                       " Pick a different test-id".format(self._get_test_dir(test)))
                logger.info("Creating test directory {}".format(self._get_test_dir(test)))
Example #58
0
import CIME.utils
from CIME.utils import expect, convert_to_seconds, parse_test_name, get_cime_root, get_model
from CIME.XML.machines import Machines
import six, sys, os

# Expect that, if a model wants to use python-based test lists, they will have a file
# config/$model/tests.py , containing a test dictionary called _TESTS

sys.path.append(os.path.join(get_cime_root(), "config", get_model()))
_ALL_TESTS = {}
try:
    from tests import _TESTS # pylint: disable=import-error
    _ALL_TESTS.update(_TESTS)
except:
    pass

# Here are the tests belonging to e3sm suites. Format is
# <test>.<grid>.<compset>.
# suite_name -> (inherits_from, timelimit, [test [, mods[, machines]]])
#   To elaborate, if no mods are needed, a string representing the testname is all that is needed.
#   If testmods are needed, a 2-ple must be provided  (test, mods)
#   If you want to restrict the test mods to certain machines, than a 3-ple is needed (test, mods, [machines])

_CIME_TESTS = {

    "cime_tiny" : (None, "0:10:00",
                   ("ERS.f19_g16_rx1.A",
                    "NCK.f19_g16_rx1.A")
                   ),

    "cime_test_only_pass" : (None, "0:10:00",