def _clean_impl(case, cleanlist, clean_all, clean_depends): ############################################################################### exeroot = os.path.abspath(case.get_value("EXEROOT")) if clean_all: # If cleanlist is empty just remove the bld directory expect(exeroot is not None,"No EXEROOT defined in case") if os.path.isdir(exeroot): logging.info("cleaning directory {}".format(exeroot)) shutil.rmtree(exeroot) # if clean_all is True also remove the sharedlibpath sharedlibroot = os.path.abspath(case.get_value("SHAREDLIBROOT")) expect(sharedlibroot is not None,"No SHAREDLIBROOT defined in case") if sharedlibroot != exeroot and os.path.isdir(sharedlibroot): logging.warning("cleaning directory {}".format(sharedlibroot)) shutil.rmtree(sharedlibroot) else: expect((cleanlist is not None and len(cleanlist) > 0) or (clean_depends is not None and len(clean_depends)),"Empty cleanlist not expected") debug = case.get_value("DEBUG") use_esmf_lib = case.get_value("USE_ESMF_LIB") build_threaded = case.get_build_threaded() gmake = case.get_value("GMAKE") caseroot = os.path.abspath(case.get_value("CASEROOT")) casetools = case.get_value("CASETOOLS") clm_config_opts = case.get_value("CLM_CONFIG_OPTS") os.environ["DEBUG"] = stringify_bool(debug) os.environ["USE_ESMF_LIB"] = stringify_bool(use_esmf_lib) os.environ["BUILD_THREADED"] = stringify_bool(build_threaded) os.environ["CASEROOT"] = caseroot os.environ["COMP_INTERFACE"] = case.get_value("COMP_INTERFACE") os.environ["PIO_VERSION"] = str(case.get_value("PIO_VERSION")) os.environ["CLM_CONFIG_OPTS"] = clm_config_opts if clm_config_opts is not None else "" cmd = gmake + " -f " + os.path.join(casetools, "Makefile") if cleanlist is not None: for item in cleanlist: tcmd = cmd + " clean" + item logger.info("calling {} ".format(tcmd)) run_cmd_no_fail(tcmd) else: for item in clean_depends: tcmd = cmd + " clean_depends" + item logger.info("calling {} ".format(tcmd)) run_cmd_no_fail(tcmd) # unlink Locked files directory unlock_file("env_build.xml") # reset following values in xml files case.set_value("SMP_BUILD",str(0)) case.set_value("NINST_BUILD",str(0)) case.set_value("BUILD_STATUS",str(0)) case.set_value("BUILD_COMPLETE","FALSE") case.flush()
def _build_model_thread(config_dir, compclass, compname, caseroot, libroot, bldroot, incroot, file_build, thread_bad_results, smp, compiler): ############################################################################### logger.info("Building {} with output to {}".format(compclass, file_build)) t1 = time.time() cmd = os.path.join(caseroot, "SourceMods", "src." + compname, "buildlib") if os.path.isfile(cmd): logger.warning("WARNING: using local buildlib script for {}".format(compname)) else: cmd = os.path.join(config_dir, "buildlib") expect(os.path.isfile(cmd), "Could not find buildlib for {}".format(compname)) with open(file_build, "w") as fd: stat = run_cmd("MODEL={} SMP={} {} {} {} {} " .format(compclass, stringify_bool(smp), cmd, caseroot, libroot, bldroot), from_dir=bldroot, arg_stdout=fd, arg_stderr=subprocess.STDOUT)[0] analyze_build_log(compclass, file_build, compiler) if (stat != 0): thread_bad_results.append("BUILD FAIL: {}.buildlib failed, cat {}".format(compname, file_build)) for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")): safe_copy(mod_file, incroot) t2 = time.time() logger.info("{} built in {:f} seconds".format(compname, (t2 - t1)))
def _build_model_thread( config_dir, compclass, compname, caseroot, libroot, bldroot, incroot, file_build, thread_bad_results, smp, compiler, ): ############################################################################### logger.info("Building {} with output to {}".format(compclass, file_build)) t1 = time.time() cmd = os.path.join(caseroot, "SourceMods", "src." + compname, "buildlib") if os.path.isfile(cmd): logger.warning( "WARNING: using local buildlib script for {}".format(compname)) else: cmd = os.path.join(config_dir, "buildlib") expect(os.path.isfile(cmd), "Could not find buildlib for {}".format(compname)) compile_cmd = "COMP_CLASS={compclass} COMP_NAME={compname} {cmd} {caseroot} {libroot} {bldroot} ".format( compclass=compclass, compname=compname, cmd=cmd, caseroot=caseroot, libroot=libroot, bldroot=bldroot, ) if get_model() != "ufs": compile_cmd = "SMP={} {}".format(stringify_bool(smp), compile_cmd) if is_python_executable(cmd): logging_options = get_logging_options() if logging_options != "": compile_cmd = compile_cmd + logging_options with open(file_build, "w") as fd: stat = run_cmd(compile_cmd, from_dir=bldroot, arg_stdout=fd, arg_stderr=subprocess.STDOUT)[0] if stat != 0: thread_bad_results.append( "BUILD FAIL: {}.buildlib failed, cat {}".format( compname, file_build)) analyze_build_log(compclass, file_build, compiler) for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")): safe_copy(mod_file, incroot) t2 = time.time() logger.info("{} built in {:f} seconds".format(compname, (t2 - t1)))
def _build_model_thread(config_dir, compclass, caseroot, libroot, bldroot, incroot, file_build, thread_bad_results, smp, compiler): ############################################################################### logger.info("Building {} with output to {}".format(compclass, file_build)) t1 = time.time() with open(file_build, "w") as fd: stat = run_cmd("MODEL={} SMP={} {}/buildlib {} {} {} ".format( compclass, stringify_bool(smp), config_dir, caseroot, libroot, bldroot), from_dir=bldroot, arg_stdout=fd, arg_stderr=subprocess.STDOUT)[0] analyze_build_log(compclass, file_build, compiler) if (stat != 0): thread_bad_results.append( "BUILD FAIL: {}.buildlib failed, cat {}".format( compclass, file_build)) for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")): shutil.copy(mod_file, incroot) t2 = time.time() logger.info("{} built in {:f} seconds".format(compclass, (t2 - t1)))
def _build_model_thread(config_dir, compclass, compname, caseroot, libroot, bldroot, incroot, file_build, thread_bad_results, smp, compiler, case): ############################################################################### logger.info("Building {} with output to {}".format(compclass, file_build)) t1 = time.time() cmd = os.path.join(caseroot, "SourceMods", "src." + compname, "buildlib") if os.path.isfile(cmd): logger.warning( "WARNING: using local buildlib script for {}".format(compname)) else: cmd = os.path.join(config_dir, "buildlib") expect(os.path.isfile(cmd), "Could not find buildlib for {}".format(compname)) # Add to this list as components are converted to python/cmake if compname in ["cam"] and get_model() == "e3sm": try: stat = 0 run_sub_or_cmd(cmd, [caseroot, libroot, bldroot], "buildlib", [bldroot, libroot, case], logfile=file_build) except Exception: stat = 1 else: with open(file_build, "w") as fd: stat = run_cmd("MODEL={} SMP={} {} {} {} {} ".format( compclass, stringify_bool(smp), cmd, caseroot, libroot, bldroot), from_dir=bldroot, arg_stdout=fd, arg_stderr=subprocess.STDOUT)[0] analyze_build_log(compclass, file_build, compiler) if stat != 0: thread_bad_results.append( "BUILD FAIL: {}.buildlib failed, cat {}".format( compname, file_build)) analyze_build_log(compclass, file_build, compiler) for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")): safe_copy(mod_file, incroot) t2 = time.time() logger.info("{} built in {:f} seconds".format(compname, (t2 - t1)))
def _main(): output, build_dir, build_optimized, clean,\ cmake_args, compiler, enable_genf90, machine, machines_dir,\ make_j, use_mpi, mpilib, mpirun_command, test_spec_dir, ctest_args,\ use_openmp, xml_test_list, verbose \ = parse_command_line(sys.argv) #================================================= # Find directory and file paths. #================================================= suite_specs = [] # TODO: this violates cime policy of direct access to xml # should be moved to CIME/XML if xml_test_list is not None: test_xml_tree = ElementTree() test_xml_tree.parse(xml_test_list) known_paths = { "here": os.path.abspath(os.path.dirname(xml_test_list)), } suite_specs.extend(suites_from_xml(test_xml_tree, known_paths)) if test_spec_dir is not None: suite_specs.append( TestSuiteSpec("__command_line_test__", ["__command_line_test__"], [os.path.abspath(test_spec_dir)])) if machines_dir is not None: machines_file = os.path.join(machines_dir, "config_machines.xml") machobj = Machines(infile=machines_file, machine=machine) else: machobj = Machines(machine=machine) # Create build directory if necessary. build_dir = os.path.abspath(build_dir) if not os.path.isdir(build_dir): os.mkdir(build_dir) # Switch to the build directory. os.chdir(build_dir) if clean: pwd_contents = os.listdir(os.getcwd()) # Clear CMake cache. for file_ in pwd_contents: if file_ in ("Macros.cmake", "env_mach_specific.xml") \ or file_.startswith('Depends') or file_.startswith(".env_mach_specific"): os.remove(file_) #================================================= # Functions to perform various stages of build. #================================================= if not use_mpi: mpilib = "mpi-serial" elif mpilib is None: mpilib = machobj.get_default_MPIlib() logger.info("Using mpilib: {}".format(mpilib)) if compiler is None: compiler = machobj.get_default_compiler() logger.info("Compiler is {}".format(compiler)) compilerobj = Compilers(machobj, compiler=compiler, mpilib=mpilib) pfunit_path = find_pfunit(compilerobj, mpilib=mpilib, use_openmp=use_openmp) debug = not build_optimized os_ = machobj.get_value("OS") # Create the environment, and the Macros.cmake file # # configure(machobj, build_dir, ["CMake"], compiler, mpilib, debug, os_, unit_testing=True) machspecific = EnvMachSpecific(build_dir, unit_testing=True) fake_case = FakeCase(compiler, mpilib, debug) machspecific.load_env(fake_case) os.environ["OS"] = os_ os.environ["COMPILER"] = compiler os.environ["DEBUG"] = stringify_bool(debug) os.environ["MPILIB"] = mpilib if use_openmp: os.environ["compile_threaded"] = "true" else: os.environ["compile_threaded"] = "false" os.environ["UNIT_TEST_HOST"] = socket.gethostname() if "NETCDF_PATH" in os.environ and not "NETCDF" in os.environ: # The CMake Netcdf find utility that we use (from pio2) seems to key off # of the environment variable NETCDF, but not NETCDF_PATH logger.info("Setting NETCDF environment variable: {}".format( os.environ["NETCDF_PATH"])) os.environ["NETCDF"] = os.environ["NETCDF_PATH"] if not use_mpi: mpirun_command = "" elif mpirun_command is None: mpi_attribs = { "compiler": compiler, "mpilib": mpilib, "threaded": use_openmp, "unit_testing": True } # We can get away with specifying case=None since we're using exe_only=True mpirun_command, _ = machspecific.get_mpirun(None, mpi_attribs, None, exe_only=True) mpirun_command = machspecific.get_resolved_value(mpirun_command) logger.info("mpirun command is '{}'".format(mpirun_command)) #================================================= # Run tests. #================================================= for spec in suite_specs: os.chdir(build_dir) if os.path.isdir(spec.name): if clean: rmtree(spec.name) if not os.path.isdir(spec.name): os.mkdir(spec.name) for label, directory in spec: os.chdir(os.path.join(build_dir, spec.name)) if not os.path.isdir(label): os.mkdir(label) os.chdir(label) name = spec.name + "/" + label if not os.path.islink("Macros.cmake"): os.symlink(os.path.join(build_dir, "Macros.cmake"), "Macros.cmake") use_mpiserial = not use_mpi cmake_stage(name, directory, build_optimized, use_mpiserial, mpirun_command, output, pfunit_path, verbose=verbose, enable_genf90=enable_genf90, cmake_args=cmake_args) make_stage(name, output, make_j, clean=clean, verbose=verbose) for spec in suite_specs: os.chdir(os.path.join(build_dir, spec.name)) for label, directory in spec: name = spec.name + "/" + label output.print_header("Running CTest tests for " + name + ".") ctest_command = ["ctest", "--output-on-failure"] if verbose: ctest_command.append("-VV") if ctest_args is not None: ctest_command.extend(ctest_args.split(" ")) run_cmd_no_fail(" ".join(ctest_command), from_dir=label, arg_stdout=None, arg_stderr=subprocess.STDOUT)
stringify_bool(uses_kokkos(case))) for var in _CMD_ARGS_FOR_BUILD: cmake_args += xml_to_make_variable(case, var, cmake=True) # Disable compiler checks cmake_args += " -DCMAKE_C_COMPILER_WORKS=1 -DCMAKE_CXX_COMPILER_WORKS=1 -DCMAKE_Fortran_COMPILER_WORKS=1" return cmake_args def xml_to_make_variable(case, varname, cmake=False): varvalue = case.get_value(varname) if varvalue is None: return "" if type(varvalue) == type(True): varvalue = stringify_bool(varvalue) return "{}{}=\"{}\" ".format("-D" if cmake else "", varname, varvalue) ############################################################################### def uses_kokkos(case): ############################################################################### cam_target = case.get_value("CAM_TARGET") return get_model() == "e3sm" and cam_target in ("preqx_kokkos", "theta-l") ############################################################################### def _build_model(build_threaded, exeroot, incroot, complist, lid, caseroot, cimeroot, compiler, buildlist, comp_interface, case): ############################################################################### logs = []
sharedlibroot = os.path.abspath(case.get_value("SHAREDLIBROOT")) expect(sharedlibroot is not None,"No SHAREDLIBROOT defined in case") if sharedlibroot != exeroot and os.path.isdir(sharedlibroot): logging.warning("cleaning directory {}".format(sharedlibroot)) shutil.rmtree(sharedlibroot) else: expect((cleanlist is not None and len(cleanlist) > 0) or (clean_depends is not None and len(clean_depends)),"Empty cleanlist not expected") debug = case.get_value("DEBUG") use_esmf_lib = case.get_value("USE_ESMF_LIB") build_threaded = case.get_build_threaded() gmake = case.get_value("GMAKE") caseroot = os.path.abspath(case.get_value("CASEROOT")) casetools = case.get_value("CASETOOLS") os.environ["DEBUG"] = stringify_bool(debug) os.environ["USE_ESMF_LIB"] = stringify_bool(use_esmf_lib) os.environ["BUILD_THREADED"] = stringify_bool(build_threaded) os.environ["CASEROOT"] = caseroot os.environ["COMP_INTERFACE"] = case.get_value("COMP_INTERFACE") os.environ["PIO_VERSION"] = str(case.get_value("PIO_VERSION")) cmd = gmake + " -f " + os.path.join(casetools, "Makefile") if cleanlist is not None: for item in cleanlist: tcmd = cmd + " clean" + item logger.info("calling {} ".format(tcmd)) run_cmd_no_fail(tcmd) else: for item in clean_depends: tcmd = cmd + " clean_depends" + item
def _case_build_impl(caseroot, case, sharedlib_only, model_only, buildlist, save_build_provenance): ############################################################################### t1 = time.time() expect(not (sharedlib_only and model_only), "Contradiction: both sharedlib_only and model_only") logger.info("Building case in directory {}".format(caseroot)) logger.info("sharedlib_only is {}".format(sharedlib_only)) logger.info("model_only is {}".format(model_only)) expect(os.path.isdir(caseroot), "'{}' is not a valid directory".format(caseroot)) os.chdir(caseroot) expect(os.path.exists(get_batch_script_for_job(case.get_primary_job())), "ERROR: must invoke case.setup script before calling build script ") cimeroot = case.get_value("CIMEROOT") comp_classes = case.get_values("COMP_CLASSES") case.check_lockedfiles(skip="env_batch") # Retrieve relevant case data # This environment variable gets set for cesm Make and # needs to be unset before building again. if "MODEL" in os.environ: del os.environ["MODEL"] build_threaded = case.get_build_threaded() casetools = case.get_value("CASETOOLS") exeroot = os.path.abspath(case.get_value("EXEROOT")) incroot = os.path.abspath(case.get_value("INCROOT")) libroot = os.path.abspath(case.get_value("LIBROOT")) sharedlibroot = os.path.abspath(case.get_value("SHAREDLIBROOT")) multi_driver = case.get_value("MULTI_DRIVER") complist = [] ninst = 1 for comp_class in comp_classes: if comp_class == "CPL": config_dir = None if multi_driver: ninst = case.get_value("NINST_MAX") else: config_dir = os.path.dirname(case.get_value("CONFIG_{}_FILE".format(comp_class))) if multi_driver: ninst = 1 else: ninst = case.get_value("NINST_{}".format(comp_class)) comp = case.get_value("COMP_{}".format(comp_class)) thrds = case.get_value("NTHRDS_{}".format(comp_class)) expect(ninst is not None,"Failed to get ninst for comp_class {}".format(comp_class)) complist.append((comp_class.lower(), comp, thrds, ninst, config_dir )) os.environ["COMP_{}".format(comp_class)] = comp ocn_submodel = case.get_value("OCN_SUBMODEL") profile_papi_enable = case.get_value("PROFILE_PAPI_ENABLE") compiler = case.get_value("COMPILER") comp_interface = case.get_value("COMP_INTERFACE") mpilib = case.get_value("MPILIB") use_esmf_lib = case.get_value("USE_ESMF_LIB") debug = case.get_value("DEBUG") ninst_build = case.get_value("NINST_BUILD") smp_value = case.get_value("SMP_VALUE") clm_use_petsc = case.get_value("CLM_USE_PETSC") cism_use_trilinos = case.get_value("CISM_USE_TRILINOS") mali_use_albany = case.get_value("MALI_USE_ALBANY") use_moab = case.get_value("USE_MOAB") clm_config_opts = case.get_value("CLM_CONFIG_OPTS") cam_config_opts = case.get_value("CAM_CONFIG_OPTS") pio_config_opts = case.get_value("PIO_CONFIG_OPTS") ninst_value = case.get_value("NINST_VALUE") mach = case.get_value("MACH") os_ = case.get_value("OS") # Load some params into env os.environ["CIMEROOT"] = cimeroot os.environ["CASETOOLS"] = casetools os.environ["EXEROOT"] = exeroot os.environ["INCROOT"] = incroot os.environ["LIBROOT"] = libroot os.environ["SHAREDLIBROOT"] = sharedlibroot os.environ["CASEROOT"] = caseroot os.environ["COMPILER"] = compiler os.environ["COMP_INTERFACE"] = comp_interface os.environ["NINST_VALUE"] = str(ninst_value) os.environ["BUILD_THREADED"] = stringify_bool(build_threaded) os.environ["MACH"] = mach os.environ["USE_ESMF_LIB"] = stringify_bool(use_esmf_lib) os.environ["MPILIB"] = mpilib os.environ["DEBUG"] = stringify_bool(debug) os.environ["OS"] = os_ os.environ["CLM_CONFIG_OPTS"] = clm_config_opts if clm_config_opts is not None else "" os.environ["CAM_CONFIG_OPTS"] = cam_config_opts if cam_config_opts is not None else "" os.environ["PIO_CONFIG_OPTS"] = pio_config_opts if pio_config_opts is not None else "" os.environ["OCN_SUBMODEL"] = ocn_submodel if ocn_submodel is not None else "" os.environ["PROFILE_PAPI_ENABLE"] = stringify_bool(profile_papi_enable) os.environ["CLM_USE_PETSC"] = stringify_bool(clm_use_petsc) os.environ["CISM_USE_TRILINOS"] = stringify_bool(cism_use_trilinos) os.environ["MALI_USE_ALBANY"] = stringify_bool(mali_use_albany) os.environ["USE_MOAB"] = stringify_bool(use_moab) if get_model() == "e3sm" and mach == "titan" and compiler == "pgiacc": case.set_value("CAM_TARGET", "preqx_acc") # This is a timestamp for the build , not the same as the testid, # and this case may not be a test anyway. For a production # experiment there may be many builds of the same case. lid = get_timestamp("%y%m%d-%H%M%S") os.environ["LID"] = lid # Set the overall USE_PETSC variable to TRUE if any of the # *_USE_PETSC variables are TRUE. # For now, there is just the one CLM_USE_PETSC variable, but in # the future there may be others -- so USE_PETSC will be true if # ANY of those are true. use_petsc = clm_use_petsc case.set_value("USE_PETSC", use_petsc) os.environ["USE_PETSC"] = stringify_bool(use_petsc) # Set the overall USE_TRILINOS variable to TRUE if any of the # *_USE_TRILINOS variables are TRUE. # For now, there is just the one CISM_USE_TRILINOS variable, but in # the future there may be others -- so USE_TRILINOS will be true if # ANY of those are true. use_trilinos = False if cism_use_trilinos is None else cism_use_trilinos case.set_value("USE_TRILINOS", use_trilinos) os.environ["USE_TRILINOS"] = stringify_bool(use_trilinos) # Set the overall USE_ALBANY variable to TRUE if any of the # *_USE_ALBANY variables are TRUE. # For now, there is just the one MALI_USE_ALBANY variable, but in # the future there may be others -- so USE_ALBANY will be true if # ANY of those are true. use_albany = stringify_bool(mali_use_albany) case.set_value("USE_ALBANY", use_albany) os.environ["USE_ALBANY"] = use_albany # Load modules case.load_env() sharedpath = _build_checks(case, build_threaded, comp_interface, use_esmf_lib, debug, compiler, mpilib, complist, ninst_build, smp_value, model_only, buildlist) t2 = time.time() logs = [] if not model_only: logs = _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid, compiler, buildlist, comp_interface) if not sharedlib_only: os.environ["INSTALL_SHAREDPATH"] = os.path.join(exeroot, sharedpath) # for MPAS makefile generators logs.extend(_build_model(build_threaded, exeroot, clm_config_opts, incroot, complist, lid, caseroot, cimeroot, compiler, buildlist, comp_interface)) if not buildlist: # in case component build scripts updated the xml files, update the case object case.read_xml() # Note, doing buildlists will never result in the system thinking the build is complete post_build(case, logs, build_complete=not (buildlist or sharedlib_only), save_build_provenance=save_build_provenance) t3 = time.time() if not sharedlib_only: logger.info("Time spent not building: {:f} sec".format(t2 - t1)) logger.info("Time spent building: {:f} sec".format(t3 - t2)) logger.info("MODEL BUILD HAS FINISHED SUCCESSFULLY") return True
def _main(): output, build_dir, build_optimized, clean,\ cmake_args, compiler, enable_genf90, machine, machines_dir,\ make_j, use_mpi, mpilib, mpirun_command, test_spec_dir, ctest_args,\ use_openmp, xml_test_list, verbose \ = parse_command_line(sys.argv) #================================================= # Find directory and file paths. #================================================= suite_specs = [] # TODO: this violates cime policy of direct access to xml # should be moved to CIME/XML if xml_test_list is not None: test_xml_tree = ElementTree() test_xml_tree.parse(xml_test_list) known_paths = { "here": os.path.abspath(os.path.dirname(xml_test_list)), } suite_specs.extend(suites_from_xml(test_xml_tree, known_paths)) if test_spec_dir is not None: suite_specs.append( TestSuiteSpec("__command_line_test__", ["__command_line_test__"], [os.path.abspath(test_spec_dir)]) ) if machines_dir is not None: machines_file = os.path.join(machines_dir, "config_machines.xml") machobj = Machines(infile=machines_file, machine=machine) else: machobj = Machines(machine=machine) # Create build directory if necessary. build_dir = os.path.abspath(build_dir) if not os.path.isdir(build_dir): os.mkdir(build_dir) # Switch to the build directory. os.chdir(build_dir) #================================================= # Functions to perform various stages of build. #================================================= if not use_mpi: mpilib = "mpi-serial" elif mpilib is None: mpilib = machobj.get_default_MPIlib() logger.info("Using mpilib: {}".format(mpilib)) if compiler is None: compiler = machobj.get_default_compiler() logger.info("Compiler is {}".format(compiler)) compilerobj = Compilers(machobj, compiler=compiler, mpilib=mpilib) pfunit_path = find_pfunit(compilerobj, mpilib=mpilib, use_openmp=use_openmp) debug = not build_optimized os_ = machobj.get_value("OS") # Create the environment, and the Macros.cmake file # # configure(machobj, build_dir, ["CMake"], compiler, mpilib, debug, os_, unit_testing=True) machspecific = EnvMachSpecific(build_dir, unit_testing=True) fake_case = FakeCase(compiler, mpilib, debug) machspecific.load_env(fake_case) os.environ["OS"] = os_ os.environ["COMPILER"] = compiler os.environ["DEBUG"] = stringify_bool(debug) os.environ["MPILIB"] = mpilib if use_openmp: os.environ["compile_threaded"] = "true" else: os.environ["compile_threaded"] = "false" os.environ["UNIT_TEST_HOST"] = socket.gethostname() if "NETCDF_PATH" in os.environ and not "NETCDF" in os.environ: # The CMake Netcdf find utility that we use (from pio2) seems to key off # of the environment variable NETCDF, but not NETCDF_PATH logger.info("Setting NETCDF environment variable: {}".format(os.environ["NETCDF_PATH"])) os.environ["NETCDF"] = os.environ["NETCDF_PATH"] if not use_mpi: mpirun_command = "" elif mpirun_command is None: mpi_attribs = { "compiler" : compiler, "mpilib" : mpilib, "threaded" : use_openmp, "unit_testing" : True } # We can get away with specifying case=None since we're using exe_only=True mpirun_command, _ = machspecific.get_mpirun(case=None, attribs=mpi_attribs, exe_only=True) mpirun_command = machspecific.get_resolved_value(mpirun_command) logger.info("mpirun command is '{}'".format(mpirun_command)) #================================================= # Run tests. #================================================= for spec in suite_specs: os.chdir(build_dir) if os.path.isdir(spec.name): if clean: rmtree(spec.name) if not os.path.isdir(spec.name): os.mkdir(spec.name) for label, directory in spec: os.chdir(os.path.join(build_dir,spec.name)) if not os.path.isdir(label): os.mkdir(label) os.chdir(label) name = spec.name+"/"+label if not os.path.islink("Macros.cmake"): os.symlink(os.path.join(build_dir,"Macros.cmake"), "Macros.cmake") use_mpiserial = not use_mpi cmake_stage(name, directory, build_optimized, use_mpiserial, mpirun_command, output, pfunit_path, verbose=verbose, enable_genf90=enable_genf90, cmake_args=cmake_args) make_stage(name, output, make_j, clean=clean, verbose=verbose) for spec in suite_specs: os.chdir(os.path.join(build_dir,spec.name)) for label, directory in spec: name = spec.name+"/"+label output.print_header("Running CTest tests for "+name+".") ctest_command = ["ctest", "--output-on-failure"] if verbose: ctest_command.append("-VV") if ctest_args is not None: ctest_command.extend(ctest_args.split(" ")) run_cmd_no_fail(" ".join(ctest_command), from_dir=label, arg_stdout=None, arg_stderr=subprocess.STDOUT)
def configure( machobj, output_dir, macros_format, compiler, mpilib, debug, comp_interface, sysos, unit_testing=False, noenv=False, threaded=False, extra_machines_dir=None, ): """Add Macros, Depends, and env_mach_specific files to a directory. Arguments: machobj - Machines argument for this machine. output_dir - Directory in which to place output. macros_format - Container containing the string 'Makefile' to produce Makefile Macros output, and/or 'CMake' for CMake output. compiler - String containing the compiler vendor to configure for. mpilib - String containing the MPI implementation to configure for. debug - Boolean specifying whether debugging options are enabled. unit_testing - Boolean specifying whether we're running unit tests (as opposed to a system run) extra_machines_dir - String giving path to an additional directory that will be searched for a config_compilers.xml file. """ # Macros generation. suffixes = {"Makefile": "make", "CMake": "cmake"} new_cmake_macros_dir = Files( comp_interface=comp_interface).get_value("CMAKE_MACROS_DIR") macro_maker = None for form in macros_format: if (new_cmake_macros_dir is not None and os.path.exists(new_cmake_macros_dir) and not "CIME_NO_CMAKE_MACRO" in os.environ): if not os.path.isfile(os.path.join(output_dir, "Macros.cmake")): safe_copy(os.path.join(new_cmake_macros_dir, "Macros.cmake"), output_dir) if not os.path.exists(os.path.join(output_dir, "cmake_macros")): shutil.copytree(new_cmake_macros_dir, os.path.join(output_dir, "cmake_macros")) # Grab macros from extra machine dir if it was provided if extra_machines_dir: extra_cmake_macros = glob.glob( "{}/cmake_macros/*.cmake".format(extra_machines_dir)) for extra_cmake_macro in extra_cmake_macros: safe_copy(extra_cmake_macro, new_cmake_macros_dir) if form == "Makefile": # Use the cmake macros to generate the make macros cmake_args = " -DOS={} -DMACH={} -DCOMPILER={} -DDEBUG={} -DMPILIB={} -Dcompile_threaded={} -DCASEROOT={}".format( sysos, machobj.get_machine_name(), compiler, stringify_bool(debug), mpilib, stringify_bool(threaded), output_dir, ) with CmakeTmpBuildDir(macroloc=output_dir) as cmaketmp: output = cmaketmp.get_makefile_vars(cmake_args=cmake_args) with open(os.path.join(output_dir, "Macros.make"), "w") as fd: fd.write(output) else: logger.warning("Using deprecated CIME makefile generators") if macro_maker is None: macro_maker = Compilers( machobj, compiler=compiler, mpilib=mpilib, extra_machines_dir=extra_machines_dir, ) out_file_name = os.path.join(output_dir, "Macros." + suffixes[form]) macro_maker.write_macros_file(macros_file=out_file_name, output_format=suffixes[form]) copy_depends_files(machobj.get_machine_name(), machobj.machines_dir, output_dir, compiler) generate_env_mach_specific( output_dir, machobj, compiler, mpilib, debug, comp_interface, sysos, unit_testing, threaded, noenv=noenv, )