def _has_unit_test_support(self): if self.TEST_COMPILER is None: default_compiler = self.MACHINE.get_default_compiler() compiler = Compilers(self.MACHINE, compiler=default_compiler) else: compiler = Compilers(self.MACHINE, compiler=self.TEST_COMPILER) attrs = {"MPILIB": "mpi-serial", "compile_threaded": "FALSE"} pfunit_path = compiler.get_optional_compiler_node("PFUNIT_PATH", attributes=attrs) if pfunit_path is None: return False else: return True
def configure(machobj, output_dir, macros_format, compiler, mpilib, debug, sysos, unit_testing=False): """Add Macros, Depends, and env_mach_specific files to a directory. Arguments: machobj - Machines argument for this machine. output_dir - Directory in which to place output. macros_format - Container containing the string 'Makefile' to produce Makefile Macros output, and/or 'CMake' for CMake output. compiler - String containing the compiler vendor to configure for. mpilib - String containing the MPI implementation to configure for. debug - Boolean specifying whether debugging options are enabled. unit_testing - Boolean specifying whether we're running unit tests (as opposed to a system run) """ # Macros generation. suffixes = {'Makefile': 'make', 'CMake': 'cmake'} macro_maker = Compilers(machobj, compiler=compiler, mpilib=mpilib) for form in macros_format: out_file_name = os.path.join(output_dir, "Macros." + suffixes[form]) macro_maker.write_macros_file(macros_file=out_file_name, output_format=suffixes[form]) _copy_depends_files(machobj.get_machine_name(), machobj.machines_dir, output_dir, compiler) _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug, sysos, unit_testing)
def test_script_is_callable(self): """The test script can be called on valid output without dying.""" # This is really more a smoke test of this script than anything else. maker = Compilers(test_utils.MockMachines("mymachine", "SomeOS"), version=2.0) test_xml = test_utils._wrap_config_compilers_xml( "<compiler><SUPPORTS_CXX>FALSE</SUPPORTS_CXX></compiler>") test_utils.get_macros(maker, test_xml, "Makefile")
def test_script_rejects_bad_build_system(self): """The macro writer rejects a bad build system string.""" maker = Compilers(test_utils.MockMachines("mymachine", "SomeOS"), version=2.0) bad_string = "argle-bargle." with self.assertRaisesRegex( utils.CIMEError, "Unrecognized build system provided to write_macros: " + bad_string, ): test_utils.get_macros(maker, "This string is irrelevant.", bad_string)
def setUp(self): self.test_os = "SomeOS" self.test_machine = "mymachine" self.test_compiler = (self.MACHINE.get_default_compiler() if self.TEST_COMPILER is None else self.TEST_COMPILER) self.test_mpilib = (self.MACHINE.get_default_MPIlib( attributes={"compiler": self.test_compiler}) if self.TEST_MPILIB is None else self.TEST_MPILIB) self._maker = Compilers(test_utils.MockMachines( self.test_machine, self.test_os), version=2.0) super().setUp()
def setUp(self): super().setUp() if "CIME_NO_CMAKE_MACRO" not in os.environ: self.skipTest("Skipping test of old macro system") self.test_os = "SomeOS" self.test_machine = "mymachine" self.test_compiler = (self.MACHINE.get_default_compiler() if self.TEST_COMPILER is None else self.TEST_COMPILER) self.test_mpilib = (self.MACHINE.get_default_MPIlib( attributes={"compiler": self.test_compiler}) if self.TEST_MPILIB is None else self.TEST_MPILIB) self._maker = Compilers(test_utils.MockMachines( self.test_machine, self.test_os), version=2.0)
def _main(): output, build_dir, build_optimized, clean,\ cmake_args, compiler, enable_genf90, machine, machines_dir,\ make_j, use_mpi, mpilib, mpirun_command, test_spec_dir, ctest_args,\ use_openmp, xml_test_list, verbose \ = parse_command_line(sys.argv) #================================================= # Find directory and file paths. #================================================= suite_specs = [] # TODO: this violates cime policy of direct access to xml # should be moved to CIME/XML if xml_test_list is not None: test_xml_tree = ElementTree() test_xml_tree.parse(xml_test_list) known_paths = { "here": os.path.abspath(os.path.dirname(xml_test_list)), } suite_specs.extend(suites_from_xml(test_xml_tree, known_paths)) if test_spec_dir is not None: suite_specs.append( TestSuiteSpec("__command_line_test__", ["__command_line_test__"], [os.path.abspath(test_spec_dir)])) if machines_dir is not None: machines_file = os.path.join(machines_dir, "config_machines.xml") machobj = Machines(infile=machines_file, machine=machine) else: machobj = Machines(machine=machine) # Create build directory if necessary. build_dir = os.path.abspath(build_dir) if not os.path.isdir(build_dir): os.mkdir(build_dir) # Switch to the build directory. os.chdir(build_dir) if clean: pwd_contents = os.listdir(os.getcwd()) # Clear CMake cache. for file_ in pwd_contents: if file_ in ("Macros.cmake", "env_mach_specific.xml") \ or file_.startswith('Depends') or file_.startswith(".env_mach_specific"): os.remove(file_) #================================================= # Functions to perform various stages of build. #================================================= if not use_mpi: mpilib = "mpi-serial" elif mpilib is None: mpilib = machobj.get_default_MPIlib() logger.info("Using mpilib: {}".format(mpilib)) if compiler is None: compiler = machobj.get_default_compiler() logger.info("Compiler is {}".format(compiler)) compilerobj = Compilers(machobj, compiler=compiler, mpilib=mpilib) pfunit_path = find_pfunit(compilerobj, mpilib=mpilib, use_openmp=use_openmp) debug = not build_optimized os_ = machobj.get_value("OS") # Create the environment, and the Macros.cmake file # # configure(machobj, build_dir, ["CMake"], compiler, mpilib, debug, os_, unit_testing=True) machspecific = EnvMachSpecific(build_dir, unit_testing=True) fake_case = FakeCase(compiler, mpilib, debug) machspecific.load_env(fake_case) os.environ["OS"] = os_ os.environ["COMPILER"] = compiler os.environ["DEBUG"] = stringify_bool(debug) os.environ["MPILIB"] = mpilib if use_openmp: os.environ["compile_threaded"] = "true" else: os.environ["compile_threaded"] = "false" os.environ["UNIT_TEST_HOST"] = socket.gethostname() if "NETCDF_PATH" in os.environ and not "NETCDF" in os.environ: # The CMake Netcdf find utility that we use (from pio2) seems to key off # of the environment variable NETCDF, but not NETCDF_PATH logger.info("Setting NETCDF environment variable: {}".format( os.environ["NETCDF_PATH"])) os.environ["NETCDF"] = os.environ["NETCDF_PATH"] if not use_mpi: mpirun_command = "" elif mpirun_command is None: mpi_attribs = { "compiler": compiler, "mpilib": mpilib, "threaded": use_openmp, "unit_testing": True } # We can get away with specifying case=None since we're using exe_only=True mpirun_command, _ = machspecific.get_mpirun(None, mpi_attribs, None, exe_only=True) mpirun_command = machspecific.get_resolved_value(mpirun_command) logger.info("mpirun command is '{}'".format(mpirun_command)) #================================================= # Run tests. #================================================= for spec in suite_specs: os.chdir(build_dir) if os.path.isdir(spec.name): if clean: rmtree(spec.name) if not os.path.isdir(spec.name): os.mkdir(spec.name) for label, directory in spec: os.chdir(os.path.join(build_dir, spec.name)) if not os.path.isdir(label): os.mkdir(label) os.chdir(label) name = spec.name + "/" + label if not os.path.islink("Macros.cmake"): os.symlink(os.path.join(build_dir, "Macros.cmake"), "Macros.cmake") use_mpiserial = not use_mpi cmake_stage(name, directory, build_optimized, use_mpiserial, mpirun_command, output, pfunit_path, verbose=verbose, enable_genf90=enable_genf90, cmake_args=cmake_args) make_stage(name, output, make_j, clean=clean, verbose=verbose) for spec in suite_specs: os.chdir(os.path.join(build_dir, spec.name)) for label, directory in spec: name = spec.name + "/" + label output.print_header("Running CTest tests for " + name + ".") ctest_command = ["ctest", "--output-on-failure"] if verbose: ctest_command.append("-VV") if ctest_args is not None: ctest_command.extend(ctest_args.split(" ")) run_cmd_no_fail(" ".join(ctest_command), from_dir=label, arg_stdout=None, arg_stderr=subprocess.STDOUT)
msg = "case.setup clean complete" append_status(msg, caseroot=caseroot, sfile="CaseStatus") if not clean: models = case.get_values("COMP_CLASSES") mach, compiler, debug, mpilib = \ case.get_value("MACH"), case.get_value("COMPILER"), case.get_value("DEBUG"), case.get_value("MPILIB") expect(mach is not None, "xml variable MACH is not set") # Create Macros file only if it does not exist if not os.path.exists("Macros"): logger.debug("Creating Macros file for %s" % mach) compilers = Compilers(compiler=compiler, machine=mach, os_=case.get_value("OS"), mpilib=mpilib) compilers.write_macros_file() else: logger.debug("Macros script already created ...skipping") # Set tasks to 1 if mpi-serial library if mpilib == "mpi-serial": for vid, value in case: if vid.startswith("NTASKS_") and value != 1: case.set_value(vid, 1) # Check ninst. # In CIME there can be multiple instances of each component model (an ensemble) NINST is the instance of that component. # Save ninst in a dict to use later in apply_user_mods
def test_script_rejects_bad_xml(self): """The macro writer rejects input that's not valid XML.""" maker = Compilers(test_utils.MockMachines("mymachine", "SomeOS"), version=2.0) with self.assertRaises(ParseError): test_utils.get_macros(maker, "This is not valid XML.", "Makefile")
def configure(machobj, output_dir, macros_format, compiler, mpilib, debug, comp_interface, sysos, unit_testing=False, noenv=False, threaded=False, extra_machines_dir=None): """Add Macros, Depends, and env_mach_specific files to a directory. Arguments: machobj - Machines argument for this machine. output_dir - Directory in which to place output. macros_format - Container containing the string 'Makefile' to produce Makefile Macros output, and/or 'CMake' for CMake output. compiler - String containing the compiler vendor to configure for. mpilib - String containing the MPI implementation to configure for. debug - Boolean specifying whether debugging options are enabled. unit_testing - Boolean specifying whether we're running unit tests (as opposed to a system run) extra_machines_dir - String giving path to an additional directory that will be searched for a config_compilers.xml file. """ # Macros generation. suffixes = {'Makefile': 'make', 'CMake': 'cmake'} new_cmake_macros_dir = Files( comp_interface=comp_interface).get_value("CMAKE_MACROS_DIR") macro_maker = None for form in macros_format: if form == "CMake" and new_cmake_macros_dir is not None and os.path.exists( new_cmake_macros_dir ) and not "CIME_NO_CMAKE_MACRO" in os.environ: if not os.path.isfile(os.path.join(output_dir, "Macros.cmake")): safe_copy(os.path.join(new_cmake_macros_dir, "Macros.cmake"), output_dir) if not os.path.exists(os.path.join(output_dir, "cmake_macros")): shutil.copytree(new_cmake_macros_dir, os.path.join(output_dir, "cmake_macros")) else: logger.warning("Using deprecated CIME makefile generators") if macro_maker is None: macro_maker = Compilers(machobj, compiler=compiler, mpilib=mpilib, extra_machines_dir=extra_machines_dir) out_file_name = os.path.join(output_dir, "Macros." + suffixes[form]) macro_maker.write_macros_file(macros_file=out_file_name, output_format=suffixes[form]) copy_depends_files(machobj.get_machine_name(), machobj.machines_dir, output_dir, compiler) generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug, comp_interface, sysos, unit_testing, threaded, noenv=noenv)
def configure( machobj, output_dir, macros_format, compiler, mpilib, debug, comp_interface, sysos, unit_testing=False, noenv=False, threaded=False, extra_machines_dir=None, ): """Add Macros, Depends, and env_mach_specific files to a directory. Arguments: machobj - Machines argument for this machine. output_dir - Directory in which to place output. macros_format - Container containing the string 'Makefile' to produce Makefile Macros output, and/or 'CMake' for CMake output. compiler - String containing the compiler vendor to configure for. mpilib - String containing the MPI implementation to configure for. debug - Boolean specifying whether debugging options are enabled. unit_testing - Boolean specifying whether we're running unit tests (as opposed to a system run) extra_machines_dir - String giving path to an additional directory that will be searched for a config_compilers.xml file. """ # Macros generation. suffixes = {"Makefile": "make", "CMake": "cmake"} new_cmake_macros_dir = Files( comp_interface=comp_interface).get_value("CMAKE_MACROS_DIR") macro_maker = None for form in macros_format: if (new_cmake_macros_dir is not None and os.path.exists(new_cmake_macros_dir) and not "CIME_NO_CMAKE_MACRO" in os.environ): if not os.path.isfile(os.path.join(output_dir, "Macros.cmake")): safe_copy(os.path.join(new_cmake_macros_dir, "Macros.cmake"), output_dir) if not os.path.exists(os.path.join(output_dir, "cmake_macros")): shutil.copytree(new_cmake_macros_dir, os.path.join(output_dir, "cmake_macros")) # Grab macros from extra machine dir if it was provided if extra_machines_dir: extra_cmake_macros = glob.glob( "{}/cmake_macros/*.cmake".format(extra_machines_dir)) for extra_cmake_macro in extra_cmake_macros: safe_copy(extra_cmake_macro, new_cmake_macros_dir) if form == "Makefile": # Use the cmake macros to generate the make macros cmake_args = " -DOS={} -DMACH={} -DCOMPILER={} -DDEBUG={} -DMPILIB={} -Dcompile_threaded={} -DCASEROOT={}".format( sysos, machobj.get_machine_name(), compiler, stringify_bool(debug), mpilib, stringify_bool(threaded), output_dir, ) with CmakeTmpBuildDir(macroloc=output_dir) as cmaketmp: output = cmaketmp.get_makefile_vars(cmake_args=cmake_args) with open(os.path.join(output_dir, "Macros.make"), "w") as fd: fd.write(output) else: logger.warning("Using deprecated CIME makefile generators") if macro_maker is None: macro_maker = Compilers( machobj, compiler=compiler, mpilib=mpilib, extra_machines_dir=extra_machines_dir, ) out_file_name = os.path.join(output_dir, "Macros." + suffixes[form]) macro_maker.write_macros_file(macros_file=out_file_name, output_format=suffixes[form]) copy_depends_files(machobj.get_machine_name(), machobj.machines_dir, output_dir, compiler) generate_env_mach_specific( output_dir, machobj, compiler, mpilib, debug, comp_interface, sysos, unit_testing, threaded, noenv=noenv, )