def configure_module(srcdir, builddir, prefix_dirs, cmake_args=None): """configure a given module by running CMake Args: srcdir: source directory of module builddir: build directory for module (may equal srcdir for in-source builds) prefix_dirs: dictionary mapping dependent modules to their prefix cmake_args (optional): list of additional CMake flags Returns: Output of CMake command """ args = [ get_cmake_command() ] args += cmakeArguments(cmake_args) args += ['-D' + module + '_DIR=' + dir for module, dir in prefix_dirs.items()] args.append(srcdir) if not os.path.isdir(builddir): os.makedirs(builddir) logger.debug('Calling "' + ' '.join(args) + '"') cmake = subprocess.Popen(args, cwd=builddir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = cmake.communicate() logging.debug(buffer_to_str(stdout)) if cmake.returncode != 0: raise RuntimeError(buffer_to_str(stderr)) return buffer_to_str(stdout)
def build_module(builddir, build_args=None): if build_args is None: build_args = get_default_build_args() cmake_args = [get_cmake_command(), '--build', '.'] if build_args is not None: cmake_args += ['--'] + build_args cmake = subprocess.Popen(cmake_args, cwd=builddir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = cmake.communicate() if cmake.returncode != 0: raise RuntimeError(buffer_to_str(stderr)) return buffer_to_str(stdout)
def preprocessorAssert(tests): '''perform preprocessore checks. A list of checks can be provided each should contain a pair, the first being the preprocessor check to perform (e.g. #if or #ifdef) and the second being the message to print if the check fails. The generated code is of the form: tests[i][0] #else test failed #endif so the first argument of each test has to make this piece of code valid C++ code assuming config.h is included. ''' source = "#include <config.h>\nint main() {\n" i = 0 for t in tests: source = source + t[0]+"\n#else\nreturn "+str(i+1)+";\n#endif\n" i += 1 source = source + "return 0;\n}\n" with open(os.path.join(builder.generated_dir, "generated_module.hh"), 'w') as out: out.write(source) builder.compile("generated_test") test_args = ["./generated_test"] test = subprocess.Popen(test_args, cwd=builder.generated_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = test.communicate() logger.debug(buffer_to_str(stdout)) returncode = test.returncode if returncode > 0: logger.debug("failed testing:\n"+source) logger.critical("checking "+tests[returncode-1][0]+" failed: "+tests[returncode-1][1]) raise ConfigurationError(tests[returncode-1][1])
def compile(self, target='all'): cmake_command = dune.common.module.get_cmake_command() cmake_args = [ cmake_command, "--build", self.dune_py_dir, "--target", target ] make_args = [] if self.build_args is not None: make_args += self.build_args if cxxFlags is not None: make_args += ['CXXFLAGS=' + cxxFlags] if noDepCheck: make_args += ["-B"] if cmake_args != []: cmake_args += ["--"] + make_args cmake = subprocess.Popen(cmake_args, cwd=self.generated_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = cmake.communicate() logger.debug("Compiler output: " + buffer_to_str(stdout)) if cmake.returncode > 0: raise CompileError(buffer_to_str(stderr)) if self.savedOutput is not None: out = buffer_to_str(stdout) nlines = out.count('\n') if nlines > 1: self.savedOutput[0].write("###############################\n") self.savedOutput[0].write("###" + " ".join(cmake_args) + "\n") if nlines > 0: self.savedOutput[0].write(out) if nlines > 1: self.savedOutput[0].write( "\n###############################\n") err = buffer_to_str(stderr) nlines = err.count('\n') if nlines > 1: self.savedOutput[1].write("###############################\n") self.savedOutput[1].write("###" + " ".join(cmake_args) + "\n") if nlines > 0: self.savedOutput[1].write(err) if nlines > 1: self.savedOutput[1].write( "\n###############################\n")
def pkg_config(pkg, var=None, paths=[]): args = ['pkg-config', pkg] if var is not None: args += ['--variable=' + var] env = dict(os.environ) env.update({'PKG_CONFIG_PATH': ':'.join(paths)}) pkgconfig = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) pkgconfig.wait() prefix = pkgconfig.stdout.read() if pkgconfig.returncode != 0: raise KeyError('package ' + pkg + ' not found.') return buffer_to_str(prefix).strip()
def dunepy_from_template(dunepy_dir,force=False): # Extract the raw data dictionary metaData = getBuildMetaData() modules = metaData.combine_across_modules("MODULENAME") builddirs = metaData.zip_across_modules("DEPS", "DEPBUILDDIRS") # add dune modules which where available during the build of a # python module but don't provide their own python module for k,v in builddirs.items(): if k not in modules and not v.endswith("NOTFOUND"): modules += [k] modules.sort() logger.debug("Found the following Dune module dependencies for dune-py: " + str(modules)) # for an existing dune-py compare module dependencies to check if rebuild required if os.path.exists(os.path.join(dunepy_dir,"dune.module")): description = Description(os.path.join(dunepy_dir,"dune.module")) deps = description.depends force = not modules == [d[0] for d in deps] # ignore version number in dependency for now # todo also check if location of modules have changed to find # modules moved from build to install or vice versa - or tag that some other way # For now we simply always force a rebuild of the dune-py # module which might lead to more work than required force = True else: force = True if force: # Remove any remnants of an old dune-py installation # if os.path.exists(dunepy_dir): # shutil.rmtree(dunepy_dir) # only remove cache try: os.remove(os.path.join(dunepy_dir, 'CMakeCache.txt')) except FileNotFoundError: pass # Gather and reorganize meta data context that is used to write dune-py context = {} context["modules"] = modules context["builddirs"] = builddirs context["install_prefix"] = metaData.unique_value_across_modules("INSTALL_PREFIX") context["cmake_flags"] = getCMakeFlags() # Find the correct template path path, _ = os.path.split(__file__) template_path = os.path.join(path, "template") # Run the template through Jinja2 env = jinja2.Environment( loader=jinja2.FileSystemLoader(template_path), keep_trailing_newline=True, ) for root, _, files in os.walk(template_path): # fix issues due to template taken from build dir if root.endswith("CMakeFiles") or root.endswith("src_dir"): continue for template_file in files: full_template_file = os.path.join(root, template_file) relative_template_file = os.path.relpath(full_template_file, start=template_path) gen_file = os.path.join(dunepy_dir, relative_template_file.split(".template")[0]) # we don't want to remove CMakeLists.txt in the generated folder if it already exists if root.endswith('generated') and os.path.exists(gen_file): continue if gen_file.endswith(".cmake") or gen_file.endswith("Makefile"): continue # issues due to template taken from build dir os.makedirs(os.path.split(gen_file)[0], exist_ok=True) with open(gen_file, "w") as outfile: outfile.write(env.get_template(relative_template_file).render(**context)) # configure dune-py logger.debug("Configuring dune-py with CMake") cmake = subprocess.Popen( ["cmake", "."], cwd=dunepy_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdout, stderr = cmake.communicate() if cmake.returncode > 0: raise CompileError(buffer_to_str(stderr)) else: logger.debug("CMake configuration output: "+buffer_to_str(stdout))
def load(self, moduleName, source, pythonName, extraCMake=None): # use with-statement to log info if compiling takes some time class PrintCompiling: def handle_signal(self, signum, frame): logger.info("Compiling "+pythonName) def __enter__(self): signal.signal(signal.SIGALRM, self.handle_signal) signal.alarm(1) def __exit__(self, type, value, traceback): signal.alarm(0) class EmptyPrintCompiling: def __enter__(self): pass def __exit__(self, type, value, traceback): pass ## TODO replace if rank with something better ## and remove barrier further down # check if we need to initialize dune-py either because # this is the first call to load or because an external module with metadata has been registered if not self.initialized or not self.externalPythonModules == getExternalPythonModules(): self.initialize() if comm.rank == 0: module = sys.modules.get("dune.generated." + moduleName) if module is None: logger.debug("Module {} not loaded".format(moduleName)) # make sure nothing (compilation, generating and building) is taking place with Lock(os.path.join(self.dune_py_dir, '..', 'lock-module.lock'), flags=LOCK_EX): # module must be generated so lock the source file with Lock(os.path.join(self.dune_py_dir, 'lock-'+moduleName+'.lock'), flags=LOCK_EX): sourceFileName = os.path.join(self.generated_dir, moduleName + ".cc") line = "dune_add_pybind11_module(NAME " + moduleName + " EXCLUDE_FROM_ALL)" # first check if this line is already present in the CMakeLists file # (possible if a previous script was stopped by user before module was compiled) with open(os.path.join(self.generated_dir, "CMakeLists.txt"), 'r') as out: found = line in out.read() if not os.path.isfile(sourceFileName) or not found: logger.info("Generating " + pythonName) # don't print 'Compiling' twice PrintCompiling = EmptyPrintCompiling code = str(source) with open(os.path.join(sourceFileName), 'w') as out: out.write(code) assert os.path.isfile(sourceFileName), "Error in writing module .cc file" if not found: origPos = -1 with open(os.path.join(self.generated_dir, "CMakeLists.txt"), 'a') as out: # store original file size origPos = out.tell() out.write(line+"\n") if extraCMake is not None: for x in extraCMake: out.write(x.replace("TARGET",moduleName)+"\n") # update build system logger.debug( "Configuring module {} ({}) with CMake".format( pythonName, moduleName ) ) try: # self.compile() cmake = subprocess.Popen( "cmake .".split(), cwd=self.dune_py_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = cmake.communicate() #logger.debug("CMake configuration output: "+buffer_to_str(stdout)) if cmake.returncode > 0: raise CompileError(buffer_to_str(stderr)) except: # all exceptions will cause a problem here os.remove(os.path.join(sourceFileName)) # remove line from CMakeLists to avoid problems with open(os.path.join(self.generated_dir, "CMakeLists.txt"), 'a') as out: out.truncate(origPos) raise elif isString(source) and not source == open(os.path.join(sourceFileName), 'r').read(): logger.info("Compiling " + pythonName + " (updated)") # don't print 'Compiling' twice PrintCompiling = EmptyPrintCompiling code = str(source) with open(os.path.join(sourceFileName), 'w') as out: out.write(code) else: # we only can directly load the module # sanity check line = "dune_add_pybind11_module(NAME " + moduleName + " EXCLUDE_FROM_ALL)" # the CMakeLists file should already include this line with open(os.path.join(self.generated_dir, "CMakeLists.txt"), 'r') as out: found = line in out.read() assert found, "CMakeLists.txt file does not contain an entry to build"+moduleName # end of exclusive dune-py lock # we always compile even if the module is always compiled since it can happen # that dune-py was updated in the mean time. # This step is quite fast but there is room for optimization. # for compilation a shared lock is enough with Lock(os.path.join(self.dune_py_dir, '..', 'lock-module.lock'), flags=LOCK_SH): # lock generated module with Lock(os.path.join(self.dune_py_dir, 'lock-'+moduleName+'.lock'), flags=LOCK_EX): with PrintCompiling(): self.compile(target=moduleName) ## TODO remove barrier here comm.barrier() logger.debug("Loading " + moduleName) module = importlib.import_module("dune.generated." + moduleName) if self.force: logger.info("Reloading " + pythonName) module = reload_module(module) return module