def _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug, sysos, unit_testing): """ env_mach_specific generation. """ ems_path = os.path.join(output_dir, "env_mach_specific.xml") if os.path.exists(ems_path): logger.warning("{} already exists, delete to replace".format(ems_path)) return ems_file = EnvMachSpecific(output_dir, unit_testing=unit_testing) ems_file.populate(machobj) ems_file.write() fake_case = FakeCase(compiler, mpilib, debug) ems_file.load_env(fake_case) for shell in ('sh', 'csh'): ems_file.make_env_mach_specific_file(shell, fake_case) shell_path = os.path.join(output_dir, ".env_mach_specific." + shell) with open(shell_path, 'a') as shell_file: if shell == 'sh': shell_file.write("\nexport COMPILER={}\n".format(compiler)) shell_file.write("export MPILIB={}\n".format(mpilib)) shell_file.write("export DEBUG={}\n".format( repr(debug).upper())) shell_file.write("export OS={}\n".format(sysos)) else: shell_file.write("\nsetenv COMPILER {}\n".format(compiler)) shell_file.write("setenv MPILIB {}\n".format(mpilib)) shell_file.write("setenv DEBUG {}\n".format( repr(debug).upper())) shell_file.write("setenv OS {}\n".format(sysos))
def generate_env_mach_specific( output_dir, machobj, compiler, mpilib, debug, comp_interface, sysos, unit_testing, threaded, noenv=False, ): """ env_mach_specific generation. """ ems_path = os.path.join(output_dir, "env_mach_specific.xml") if os.path.exists(ems_path): logger.warning("{} already exists, delete to replace".format(ems_path)) return ems_file = EnvMachSpecific(output_dir, unit_testing=unit_testing, standalone_configure=True) ems_file.populate( machobj, attributes={ "mpilib": mpilib, "compiler": compiler, "threaded": threaded }, ) ems_file.write() if noenv: return fake_case = FakeCase(compiler, mpilib, debug, comp_interface) ems_file.load_env(fake_case) for shell in ("sh", "csh"): ems_file.make_env_mach_specific_file(shell, fake_case, output_dir=output_dir) shell_path = os.path.join(output_dir, ".env_mach_specific." + shell) with open(shell_path, "a") as shell_file: if shell == "sh": shell_file.write("\nexport COMPILER={}\n".format(compiler)) shell_file.write("export MPILIB={}\n".format(mpilib)) shell_file.write("export DEBUG={}\n".format( repr(debug).upper())) shell_file.write("export OS={}\n".format(sysos)) else: shell_file.write("\nsetenv COMPILER {}\n".format(compiler)) shell_file.write("setenv MPILIB {}\n".format(mpilib)) shell_file.write("setenv DEBUG {}\n".format( repr(debug).upper())) shell_file.write("setenv OS {}\n".format(sysos))
def _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug, sysos, unit_testing): """ env_mach_specific generation. """ ems_path = os.path.join(output_dir, "env_mach_specific.xml") if os.path.exists(ems_path): logger.warning("{} already exists, delete to replace".format(ems_path)) return ems_file = EnvMachSpecific(output_dir, unit_testing=unit_testing) ems_file.populate(machobj) ems_file.write() ems_file.load_env(compiler, debug, mpilib) for shell in ('sh', 'csh'): ems_file.make_env_mach_specific_file(compiler, debug, mpilib, shell) shell_path = os.path.join(output_dir, ".env_mach_specific." + shell) with open(shell_path, 'a') as shell_file: if shell == 'sh': shell_file.write("\nexport COMPILER={}\n".format(compiler)) shell_file.write("export MPILIB={}\n".format(mpilib)) shell_file.write("export DEBUG={}\n".format(repr(debug).upper())) shell_file.write("export OS={}\n".format(sysos)) else: shell_file.write("\nsetenv COMPILER {}\n".format(compiler)) shell_file.write("setenv MPILIB {}\n".format(mpilib)) shell_file.write("setenv DEBUG {}\n".format(repr(debug).upper())) shell_file.write("setenv OS {}\n".format(sysos))
def load_env_for_case(self): mach_specific = EnvMachSpecific(caseroot=self._caseroot) module_nodes = mach_specific.get_node("modules") env_nodes = mach_specific.get_node("environment_variables") if (module_nodes is not None): modules_to_load = self._compute_module_actions(module_nodes) self.load_modules(modules_to_load) if (env_nodes is not None): envs_to_set = self._compute_env_actions(env_nodes) self.load_envs(envs_to_set)
def _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug, sysos, unit_testing): """ env_mach_specific generation. """ ems_path = os.path.join(output_dir, "env_mach_specific.xml") if os.path.exists(ems_path): logger.warn("%s already exists, delete to replace" % ems_path) return ems_file = EnvMachSpecific(output_dir, unit_testing=unit_testing) ems_file.populate(machobj) ems_file.write() for shell in ('sh', 'csh'): ems_file.make_env_mach_specific_file(compiler, debug, mpilib, shell) shell_path = os.path.join(output_dir, ".env_mach_specific." + shell) with open(shell_path, 'a') as shell_file: if shell == 'sh': shell_file.write("\nexport COMPILER=%s\n" % compiler) shell_file.write("export MPILIB=%s\n" % mpilib) shell_file.write("export DEBUG=%s\n" % repr(debug).upper()) shell_file.write("export OS=%s\n" % sysos) else: shell_file.write("\nsetenv COMPILER %s\n" % compiler) shell_file.write("setenv MPILIB %s\n" % mpilib) shell_file.write("setenv DEBUG %s\n" % repr(debug).upper()) shell_file.write("setenv OS %s\n" % sysos)
def read_xml(self): if (len(self._env_files_that_need_rewrite) > 0): files = "" for env_file in self._env_files_that_need_rewrite: files += " " + env_file.filename expect( False, "Object(s) %s seem to have newer data than the corresponding case file" % files) self._env_entryid_files = [] self._env_entryid_files.append(EnvCase(self._caseroot, components=None)) components = self._env_entryid_files[0].get_values("COMP_CLASSES") self._env_entryid_files.append( EnvRun(self._caseroot, components=components)) self._env_entryid_files.append( EnvBuild(self._caseroot, components=components)) self._env_entryid_files.append( EnvMachPes(self._caseroot, components=components)) if os.path.isfile(os.path.join(self._caseroot, "env_test.xml")): self._env_entryid_files.append( EnvTest(self._caseroot, components=components)) self._env_generic_files = [] self._env_generic_files.append(EnvBatch(self._caseroot)) self._env_generic_files.append(EnvMachSpecific(self._caseroot)) self._env_generic_files.append(EnvArchive(self._caseroot)) self._files = self._env_entryid_files + self._env_generic_files
def _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug, sysos): """ env_mach_specific generation. """ ems_path = os.path.join(output_dir, "env_mach_specific.xml") if os.path.exists(ems_path): logger.warn("%s already exists, delete to replace"%ems_path) return ems_file = EnvMachSpecific(output_dir) ems_file.populate(machobj) ems_file.write() for shell in ('sh', 'csh'): ems_file.make_env_mach_specific_file(compiler, debug, mpilib, shell) shell_path = os.path.join(output_dir, ".env_mach_specific." + shell) with open(shell_path, 'a') as shell_file: if shell == 'sh': shell_file.write("\nexport COMPILER=%s\n" % compiler) shell_file.write("export MPILIB=%s\n" % mpilib) shell_file.write("export DEBUG=%s\n" % repr(debug).upper()) shell_file.write("export OS=%s\n" % sysos) else: shell_file.write("\nsetenv COMPILER %s\n" % compiler) shell_file.write("setenv MPILIB %s\n" % mpilib) shell_file.write("setenv DEBUG %s\n" % repr(debug).upper()) shell_file.write("setenv OS %s\n" % sysos)
def _main(): output, build_dir, build_optimized, clean,\ cmake_args, compiler, enable_genf90, machine, machines_dir,\ make_j, use_mpi, mpilib, mpirun_command, test_spec_dir, ctest_args,\ use_openmp, xml_test_list, verbose \ = parse_command_line(sys.argv) #================================================= # Find directory and file paths. #================================================= suite_specs = [] # TODO: this violates cime policy of direct access to xml # should be moved to CIME/XML if xml_test_list is not None: test_xml_tree = ElementTree() test_xml_tree.parse(xml_test_list) known_paths = { "here": os.path.abspath(os.path.dirname(xml_test_list)), } suite_specs.extend(suites_from_xml(test_xml_tree, known_paths)) if test_spec_dir is not None: suite_specs.append( TestSuiteSpec("__command_line_test__", ["__command_line_test__"], [os.path.abspath(test_spec_dir)])) if machines_dir is not None: machines_file = os.path.join(machines_dir, "config_machines.xml") machobj = Machines(infile=machines_file, machine=machine) else: machobj = Machines(machine=machine) # Create build directory if necessary. build_dir = os.path.abspath(build_dir) if not os.path.isdir(build_dir): os.mkdir(build_dir) # Switch to the build directory. os.chdir(build_dir) if clean: pwd_contents = os.listdir(os.getcwd()) # Clear CMake cache. for file_ in pwd_contents: if file_ in ("Macros.cmake", "env_mach_specific.xml") \ or file_.startswith('Depends') or file_.startswith(".env_mach_specific"): os.remove(file_) #================================================= # Functions to perform various stages of build. #================================================= if not use_mpi: mpilib = "mpi-serial" elif mpilib is None: mpilib = machobj.get_default_MPIlib() logger.info("Using mpilib: {}".format(mpilib)) if compiler is None: compiler = machobj.get_default_compiler() logger.info("Compiler is {}".format(compiler)) compilerobj = Compilers(machobj, compiler=compiler, mpilib=mpilib) pfunit_path = find_pfunit(compilerobj, mpilib=mpilib, use_openmp=use_openmp) debug = not build_optimized os_ = machobj.get_value("OS") # Create the environment, and the Macros.cmake file # # configure(machobj, build_dir, ["CMake"], compiler, mpilib, debug, os_, unit_testing=True) machspecific = EnvMachSpecific(build_dir, unit_testing=True) fake_case = FakeCase(compiler, mpilib, debug) machspecific.load_env(fake_case) os.environ["OS"] = os_ os.environ["COMPILER"] = compiler os.environ["DEBUG"] = stringify_bool(debug) os.environ["MPILIB"] = mpilib if use_openmp: os.environ["compile_threaded"] = "true" else: os.environ["compile_threaded"] = "false" os.environ["UNIT_TEST_HOST"] = socket.gethostname() if "NETCDF_PATH" in os.environ and not "NETCDF" in os.environ: # The CMake Netcdf find utility that we use (from pio2) seems to key off # of the environment variable NETCDF, but not NETCDF_PATH logger.info("Setting NETCDF environment variable: {}".format( os.environ["NETCDF_PATH"])) os.environ["NETCDF"] = os.environ["NETCDF_PATH"] if not use_mpi: mpirun_command = "" elif mpirun_command is None: mpi_attribs = { "compiler": compiler, "mpilib": mpilib, "threaded": use_openmp, "unit_testing": True } # We can get away with specifying case=None since we're using exe_only=True mpirun_command, _ = machspecific.get_mpirun(None, mpi_attribs, None, exe_only=True) mpirun_command = machspecific.get_resolved_value(mpirun_command) logger.info("mpirun command is '{}'".format(mpirun_command)) #================================================= # Run tests. #================================================= for spec in suite_specs: os.chdir(build_dir) if os.path.isdir(spec.name): if clean: rmtree(spec.name) if not os.path.isdir(spec.name): os.mkdir(spec.name) for label, directory in spec: os.chdir(os.path.join(build_dir, spec.name)) if not os.path.isdir(label): os.mkdir(label) os.chdir(label) name = spec.name + "/" + label if not os.path.islink("Macros.cmake"): os.symlink(os.path.join(build_dir, "Macros.cmake"), "Macros.cmake") use_mpiserial = not use_mpi cmake_stage(name, directory, build_optimized, use_mpiserial, mpirun_command, output, pfunit_path, verbose=verbose, enable_genf90=enable_genf90, cmake_args=cmake_args) make_stage(name, output, make_j, clean=clean, verbose=verbose) for spec in suite_specs: os.chdir(os.path.join(build_dir, spec.name)) for label, directory in spec: name = spec.name + "/" + label output.print_header("Running CTest tests for " + name + ".") ctest_command = ["ctest", "--output-on-failure"] if verbose: ctest_command.append("-VV") if ctest_args is not None: ctest_command.extend(ctest_args.split(" ")) run_cmd_no_fail(" ".join(ctest_command), from_dir=label, arg_stdout=None, arg_stderr=subprocess.STDOUT)
def _main(): output, build_dir, build_optimized, clean,\ cmake_args, compiler, enable_genf90, machine, machines_dir,\ make_j, use_mpi, mpilib, mpirun_command, test_spec_dir, ctest_args,\ use_openmp, xml_test_list, verbose \ = parse_command_line(sys.argv) #================================================= # Find directory and file paths. #================================================= suite_specs = [] # TODO: this violates cime policy of direct access to xml # should be moved to CIME/XML if xml_test_list is not None: test_xml_tree = ElementTree() test_xml_tree.parse(xml_test_list) known_paths = { "here": os.path.abspath(os.path.dirname(xml_test_list)), } suite_specs.extend(suites_from_xml(test_xml_tree, known_paths)) if test_spec_dir is not None: suite_specs.append( TestSuiteSpec("__command_line_test__", ["__command_line_test__"], [os.path.abspath(test_spec_dir)]) ) if machines_dir is not None: machines_file = os.path.join(machines_dir, "config_machines.xml") machobj = Machines(infile=machines_file, machine=machine) else: machobj = Machines(machine=machine) # Create build directory if necessary. build_dir = os.path.abspath(build_dir) if not os.path.isdir(build_dir): os.mkdir(build_dir) # Switch to the build directory. os.chdir(build_dir) #================================================= # Functions to perform various stages of build. #================================================= if not use_mpi: mpilib = "mpi-serial" elif mpilib is None: mpilib = machobj.get_default_MPIlib() logger.info("Using mpilib: {}".format(mpilib)) if compiler is None: compiler = machobj.get_default_compiler() logger.info("Compiler is {}".format(compiler)) compilerobj = Compilers(machobj, compiler=compiler, mpilib=mpilib) pfunit_path = find_pfunit(compilerobj, mpilib=mpilib, use_openmp=use_openmp) debug = not build_optimized os_ = machobj.get_value("OS") # Create the environment, and the Macros.cmake file # # configure(machobj, build_dir, ["CMake"], compiler, mpilib, debug, os_, unit_testing=True) machspecific = EnvMachSpecific(build_dir, unit_testing=True) fake_case = FakeCase(compiler, mpilib, debug) machspecific.load_env(fake_case) os.environ["OS"] = os_ os.environ["COMPILER"] = compiler os.environ["DEBUG"] = stringify_bool(debug) os.environ["MPILIB"] = mpilib if use_openmp: os.environ["compile_threaded"] = "true" else: os.environ["compile_threaded"] = "false" os.environ["UNIT_TEST_HOST"] = socket.gethostname() if "NETCDF_PATH" in os.environ and not "NETCDF" in os.environ: # The CMake Netcdf find utility that we use (from pio2) seems to key off # of the environment variable NETCDF, but not NETCDF_PATH logger.info("Setting NETCDF environment variable: {}".format(os.environ["NETCDF_PATH"])) os.environ["NETCDF"] = os.environ["NETCDF_PATH"] if not use_mpi: mpirun_command = "" elif mpirun_command is None: mpi_attribs = { "compiler" : compiler, "mpilib" : mpilib, "threaded" : use_openmp, "unit_testing" : True } # We can get away with specifying case=None since we're using exe_only=True mpirun_command, _ = machspecific.get_mpirun(case=None, attribs=mpi_attribs, exe_only=True) mpirun_command = machspecific.get_resolved_value(mpirun_command) logger.info("mpirun command is '{}'".format(mpirun_command)) #================================================= # Run tests. #================================================= for spec in suite_specs: os.chdir(build_dir) if os.path.isdir(spec.name): if clean: rmtree(spec.name) if not os.path.isdir(spec.name): os.mkdir(spec.name) for label, directory in spec: os.chdir(os.path.join(build_dir,spec.name)) if not os.path.isdir(label): os.mkdir(label) os.chdir(label) name = spec.name+"/"+label if not os.path.islink("Macros.cmake"): os.symlink(os.path.join(build_dir,"Macros.cmake"), "Macros.cmake") use_mpiserial = not use_mpi cmake_stage(name, directory, build_optimized, use_mpiserial, mpirun_command, output, pfunit_path, verbose=verbose, enable_genf90=enable_genf90, cmake_args=cmake_args) make_stage(name, output, make_j, clean=clean, verbose=verbose) for spec in suite_specs: os.chdir(os.path.join(build_dir,spec.name)) for label, directory in spec: name = spec.name+"/"+label output.print_header("Running CTest tests for "+name+".") ctest_command = ["ctest", "--output-on-failure"] if verbose: ctest_command.append("-VV") if ctest_args is not None: ctest_command.extend(ctest_args.split(" ")) run_cmd_no_fail(" ".join(ctest_command), from_dir=label, arg_stdout=None, arg_stderr=subprocess.STDOUT)