def configure_module(srcdir, builddir, prefix_dirs, cmake_args=None):
"""configure a given module by running CMake
Args:
srcdir: source directory of module
builddir: build directory for module (may equal srcdir for in-source builds)
prefix_dirs: dictionary mapping dependent modules to their prefix
cmake_args (optional): list of additional CMake flags
Returns:
Output of CMake command
"""
args = [ get_cmake_command() ]
args += cmakeArguments(cmake_args)
args += ['-D' + module + '_DIR=' + dir for module, dir in prefix_dirs.items()]
args.append(srcdir)
if not os.path.isdir(builddir):
os.makedirs(builddir)
logger.debug('Calling "' + ' '.join(args) + '"')
cmake = subprocess.Popen(args, cwd=builddir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = cmake.communicate()
logging.debug(buffer_to_str(stdout))
if cmake.returncode != 0:
raise RuntimeError(buffer_to_str(stderr))
return buffer_to_str(stdout)
def build_module(builddir, build_args=None):
if build_args is None:
build_args = get_default_build_args()
cmake_args = [get_cmake_command(), '--build', '.']
if build_args is not None:
cmake_args += ['--'] + build_args
cmake = subprocess.Popen(cmake_args, cwd=builddir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = cmake.communicate()
if cmake.returncode != 0:
raise RuntimeError(buffer_to_str(stderr))
return buffer_to_str(stdout)
def preprocessorAssert(tests):
'''perform preprocessore checks.
A list of checks can be provided each should contain a pair, the
first being the preprocessor check to perform (e.g. #if or #ifdef)
and the second being the message to print if the check fails. The
generated code is of the form:
tests[i][0]
#else
test failed
#endif
so the first argument of each test has to make this piece of code
valid C++ code assuming config.h is included.
'''
source = "#include <config.h>\nint main() {\n"
i = 0
for t in tests:
source = source + t[0]+"\n#else\nreturn "+str(i+1)+";\n#endif\n"
i += 1
source = source + "return 0;\n}\n"
with open(os.path.join(builder.generated_dir, "generated_module.hh"), 'w') as out:
out.write(source)
builder.compile("generated_test")
test_args = ["./generated_test"]
test = subprocess.Popen(test_args, cwd=builder.generated_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = test.communicate()
logger.debug(buffer_to_str(stdout))
returncode = test.returncode
if returncode > 0:
logger.debug("failed testing:\n"+source)
logger.critical("checking "+tests[returncode-1][0]+" failed: "+tests[returncode-1][1])
raise ConfigurationError(tests[returncode-1][1])
def compile(self, target='all'):
cmake_command = dune.common.module.get_cmake_command()
cmake_args = [
cmake_command, "--build", self.dune_py_dir, "--target", target
]
make_args = []
if self.build_args is not None:
make_args += self.build_args
if cxxFlags is not None:
make_args += ['CXXFLAGS=' + cxxFlags]
if noDepCheck:
make_args += ["-B"]
if cmake_args != []:
cmake_args += ["--"] + make_args
cmake = subprocess.Popen(cmake_args,
cwd=self.generated_dir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = cmake.communicate()
logger.debug("Compiler output: " + buffer_to_str(stdout))
if cmake.returncode > 0:
raise CompileError(buffer_to_str(stderr))
if self.savedOutput is not None:
out = buffer_to_str(stdout)
nlines = out.count('\n')
if nlines > 1:
self.savedOutput[0].write("###############################\n")
self.savedOutput[0].write("###" + " ".join(cmake_args) + "\n")
if nlines > 0:
self.savedOutput[0].write(out)
if nlines > 1:
self.savedOutput[0].write(
"\n###############################\n")
err = buffer_to_str(stderr)
nlines = err.count('\n')
if nlines > 1:
self.savedOutput[1].write("###############################\n")
self.savedOutput[1].write("###" + " ".join(cmake_args) + "\n")
if nlines > 0:
self.savedOutput[1].write(err)
if nlines > 1:
self.savedOutput[1].write(
"\n###############################\n")
def pkg_config(pkg, var=None, paths=[]):
args = ['pkg-config', pkg]
if var is not None:
args += ['--variable=' + var]
env = dict(os.environ)
env.update({'PKG_CONFIG_PATH': ':'.join(paths)})
pkgconfig = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
pkgconfig.wait()
prefix = pkgconfig.stdout.read()
if pkgconfig.returncode != 0:
raise KeyError('package ' + pkg + ' not found.')
return buffer_to_str(prefix).strip()
def dunepy_from_template(dunepy_dir,force=False):
# Extract the raw data dictionary
metaData = getBuildMetaData()
modules = metaData.combine_across_modules("MODULENAME")
builddirs = metaData.zip_across_modules("DEPS", "DEPBUILDDIRS")
# add dune modules which where available during the build of a
# python module but don't provide their own python module
for k,v in builddirs.items():
if k not in modules and not v.endswith("NOTFOUND"):
modules += [k]
modules.sort()
logger.debug("Found the following Dune module dependencies for dune-py: " + str(modules))
# for an existing dune-py compare module dependencies to check if rebuild required
if os.path.exists(os.path.join(dunepy_dir,"dune.module")):
description = Description(os.path.join(dunepy_dir,"dune.module"))
deps = description.depends
force = not modules == [d[0] for d in deps] # ignore version number in dependency for now
# todo also check if location of modules have changed to find
# modules moved from build to install or vice versa - or tag that some other way
# For now we simply always force a rebuild of the dune-py
# module which might lead to more work than required
force = True
else:
force = True
if force:
# Remove any remnants of an old dune-py installation
# if os.path.exists(dunepy_dir):
# shutil.rmtree(dunepy_dir)
# only remove cache
try:
os.remove(os.path.join(dunepy_dir, 'CMakeCache.txt'))
except FileNotFoundError:
pass
# Gather and reorganize meta data context that is used to write dune-py
context = {}
context["modules"] = modules
context["builddirs"] = builddirs
context["install_prefix"] = metaData.unique_value_across_modules("INSTALL_PREFIX")
context["cmake_flags"] = getCMakeFlags()
# Find the correct template path
path, _ = os.path.split(__file__)
template_path = os.path.join(path, "template")
# Run the template through Jinja2
env = jinja2.Environment(
loader=jinja2.FileSystemLoader(template_path),
keep_trailing_newline=True,
)
for root, _, files in os.walk(template_path):
# fix issues due to template taken from build dir
if root.endswith("CMakeFiles") or root.endswith("src_dir"): continue
for template_file in files:
full_template_file = os.path.join(root, template_file)
relative_template_file = os.path.relpath(full_template_file, start=template_path)
gen_file = os.path.join(dunepy_dir, relative_template_file.split(".template")[0])
# we don't want to remove CMakeLists.txt in the generated folder if it already exists
if root.endswith('generated') and os.path.exists(gen_file): continue
if gen_file.endswith(".cmake") or gen_file.endswith("Makefile"): continue # issues due to template taken from build dir
os.makedirs(os.path.split(gen_file)[0], exist_ok=True)
with open(gen_file, "w") as outfile:
outfile.write(env.get_template(relative_template_file).render(**context))
# configure dune-py
logger.debug("Configuring dune-py with CMake")
cmake = subprocess.Popen(
["cmake", "."],
cwd=dunepy_dir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
stdout, stderr = cmake.communicate()
if cmake.returncode > 0:
raise CompileError(buffer_to_str(stderr))
else:
logger.debug("CMake configuration output: "+buffer_to_str(stdout))
def load(self, moduleName, source, pythonName, extraCMake=None):
# use with-statement to log info if compiling takes some time
class PrintCompiling:
def handle_signal(self, signum, frame):
logger.info("Compiling "+pythonName)
def __enter__(self):
signal.signal(signal.SIGALRM, self.handle_signal)
signal.alarm(1)
def __exit__(self, type, value, traceback):
signal.alarm(0)
class EmptyPrintCompiling:
def __enter__(self): pass
def __exit__(self, type, value, traceback): pass
## TODO replace if rank with something better
## and remove barrier further down
# check if we need to initialize dune-py either because
# this is the first call to load or because an external module with metadata has been registered
if not self.initialized or not self.externalPythonModules == getExternalPythonModules():
self.initialize()
if comm.rank == 0:
module = sys.modules.get("dune.generated." + moduleName)
if module is None:
logger.debug("Module {} not loaded".format(moduleName))
# make sure nothing (compilation, generating and building) is taking place
with Lock(os.path.join(self.dune_py_dir, '..', 'lock-module.lock'), flags=LOCK_EX):
# module must be generated so lock the source file
with Lock(os.path.join(self.dune_py_dir, 'lock-'+moduleName+'.lock'), flags=LOCK_EX):
sourceFileName = os.path.join(self.generated_dir, moduleName + ".cc")
line = "dune_add_pybind11_module(NAME " + moduleName + " EXCLUDE_FROM_ALL)"
# first check if this line is already present in the CMakeLists file
# (possible if a previous script was stopped by user before module was compiled)
with open(os.path.join(self.generated_dir, "CMakeLists.txt"), 'r') as out:
found = line in out.read()
if not os.path.isfile(sourceFileName) or not found:
logger.info("Generating " + pythonName)
# don't print 'Compiling' twice
PrintCompiling = EmptyPrintCompiling
code = str(source)
with open(os.path.join(sourceFileName), 'w') as out:
out.write(code)
assert os.path.isfile(sourceFileName), "Error in writing module .cc file"
if not found:
origPos = -1
with open(os.path.join(self.generated_dir, "CMakeLists.txt"), 'a') as out:
# store original file size
origPos = out.tell()
out.write(line+"\n")
if extraCMake is not None:
for x in extraCMake:
out.write(x.replace("TARGET",moduleName)+"\n")
# update build system
logger.debug(
"Configuring module {} ({}) with CMake".format(
pythonName, moduleName
)
)
try:
# self.compile()
cmake = subprocess.Popen( "cmake .".split(),
cwd=self.dune_py_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = cmake.communicate()
#logger.debug("CMake configuration output: "+buffer_to_str(stdout))
if cmake.returncode > 0:
raise CompileError(buffer_to_str(stderr))
except: # all exceptions will cause a problem here
os.remove(os.path.join(sourceFileName))
# remove line from CMakeLists to avoid problems
with open(os.path.join(self.generated_dir, "CMakeLists.txt"), 'a') as out:
out.truncate(origPos)
raise
elif isString(source) and not source == open(os.path.join(sourceFileName), 'r').read():
logger.info("Compiling " + pythonName + " (updated)")
# don't print 'Compiling' twice
PrintCompiling = EmptyPrintCompiling
code = str(source)
with open(os.path.join(sourceFileName), 'w') as out:
out.write(code)
else:
# we only can directly load the module
# sanity check
line = "dune_add_pybind11_module(NAME " + moduleName + " EXCLUDE_FROM_ALL)"
# the CMakeLists file should already include this line
with open(os.path.join(self.generated_dir, "CMakeLists.txt"), 'r') as out:
found = line in out.read()
assert found, "CMakeLists.txt file does not contain an entry to build"+moduleName
# end of exclusive dune-py lock
# we always compile even if the module is always compiled since it can happen
# that dune-py was updated in the mean time.
# This step is quite fast but there is room for optimization.
# for compilation a shared lock is enough
with Lock(os.path.join(self.dune_py_dir, '..', 'lock-module.lock'), flags=LOCK_SH):
# lock generated module
with Lock(os.path.join(self.dune_py_dir, 'lock-'+moduleName+'.lock'), flags=LOCK_EX):
with PrintCompiling():
self.compile(target=moduleName)
## TODO remove barrier here
comm.barrier()
logger.debug("Loading " + moduleName)
module = importlib.import_module("dune.generated." + moduleName)
if self.force:
logger.info("Reloading " + pythonName)
module = reload_module(module)
return module
