def extract_files(signature,
cache_params,
prefix="",
path=os.curdir,
categories=("inc", "src", "lib", "log")):
"""Make a copy of files stored under this signature.
Target filenames are '<path>/<prefix>-<signature>.*'
"""
path = os.path.join(path, prefix + signature)
make_dirs(path)
if "inc" in categories:
inc_filename = create_inc_filename(signature, cache_params)
try_copy_file(inc_filename, path)
if "src" in categories:
src_filename = create_src_filename(signature, cache_params)
if not os.path.exists(src_filename):
src_filename = src_filename + ".gz"
if os.path.exists(src_filename):
try_copy_file(src_filename, path)
if src_filename.endswith(".gz"):
gunzip_file(os.path.join(path, os.path.basename(src_filename)))
if "lib" in categories:
lib_filename = create_lib_filename(signature, cache_params)
try_copy_file(lib_filename, path)
if "log" in categories:
log_filename = create_log_filename(signature, cache_params)
try_copy_file(log_filename, path)
return path
def make_log_dir(cache_params):
d = os.path.join(cache_params["cache_dir"], cache_params["log_dir"])
make_dirs(d)
return d
def build_shared_library(signature, header, source, dependencies, params):
"""Build shared library from a source file and store library in
cache.
"""
cache_params = params["cache"]
build_params = params["build"]
# Create basenames
inc_basename = create_inc_basename(signature, cache_params)
src_basename = create_src_basename(signature, cache_params)
lib_basename = create_lib_basename(signature, cache_params)
# Create a temp directory and filenames within it
tmpdir = temp_dir(cache_params)
temp_inc_filename = os.path.join(tmpdir, inc_basename)
temp_src_filename = os.path.join(tmpdir, src_basename)
temp_lib_filename = os.path.join(tmpdir, lib_basename)
# Store source and header in temp dir
if header:
store_textfile(temp_inc_filename, header)
store_textfile(temp_src_filename, source)
# Build final command as list of arguments
cmd = make_compile_command(temp_src_filename, temp_lib_filename,
dependencies, build_params, cache_params)
# Execute command to compile generated source code to dynamic
# library
status, output = get_status_output(cmd)
# Move files to cache on success or a local dir on failure,
# using safe lockfree move
if status == 0:
# Ensure dirnames exist in cache dirs
ensure_dirs(cache_params)
# Move library first
lib_filename = create_lib_filename(signature, cache_params)
assert os.path.exists(os.path.dirname(lib_filename))
lockfree_move_file(temp_lib_filename, lib_filename)
# Write header only if there is one
if header:
inc_filename = create_inc_filename(signature, cache_params)
assert os.path.exists(os.path.dirname(inc_filename))
lockfree_move_file(temp_inc_filename, inc_filename)
else:
inc_filename = None
# Compress or delete source code based on params
temp_src_filename = compress_source_code(temp_src_filename,
cache_params)
if temp_src_filename:
src_filename = create_src_filename(signature, cache_params)
if temp_src_filename.endswith(".gz"):
src_filename = src_filename + ".gz"
assert os.path.exists(os.path.dirname(src_filename))
lockfree_move_file(temp_src_filename, src_filename)
else:
src_filename = None
# Write compiler command and output to log file
if cache_params["enable_build_log"]:
# Recreate compiler command without the tempdir
cmd = make_compile_command(src_basename, lib_basename,
dependencies, build_params,
cache_params)
log_contents = "%s\n\n%s" % (" ".join(cmd), output)
log_filename = create_log_filename(signature, cache_params)
assert os.path.exists(os.path.dirname(log_filename))
store_textfile(log_filename, log_contents)
else:
log_filename = None
files = set((inc_filename, src_filename, lib_filename, log_filename))
files = files - set((None, ))
files = sorted(files)
debug("Compilation succeeded. Files written to cache:\n" +
"\n".join(files))
err_info = None
else:
# Create filenames in a local directory to store files for
# reproducing failure
fail_dir = create_fail_dir_path(signature, cache_params)
make_dirs(fail_dir)
# Library name is returned below
lib_filename = None
# Write header only if there is one
if header:
inc_filename = os.path.join(fail_dir, inc_basename)
lockfree_move_file(temp_inc_filename, inc_filename)
# Always write source for inspection after compilation failure
src_filename = os.path.join(fail_dir, src_basename)
lockfree_move_file(temp_src_filename, src_filename)
# Write compile command to failure dir, adjusted to use local
# source file name so it can be rerun
cmd = make_compile_command(src_basename, lib_basename, dependencies,
build_params, cache_params)
cmds = " ".join(cmd)
script = "#!/bin/bash\n# Execute this file to recompile locally\n" + cmds
cmd_filename = os.path.join(fail_dir, "recompile.sh")
store_textfile(cmd_filename, script)
make_executable(cmd_filename)
# Write readme file with instructions
readme = "Run or source recompile.sh to compile locally and reproduce the build failure.\n"
readme_filename = os.path.join(fail_dir, "README")
store_textfile(readme_filename, readme)
# Write compiler output to failure dir (will refer to temp paths)
log_filename = os.path.join(fail_dir, "error.log")
store_textfile(log_filename, output)
info(
"------------------- Start compiler output ------------------------"
)
info(output)
info(
"------------------- End compiler output ------------------------"
)
warning("Compilation failed! Sources, command, and "
"errors have been written to: %s" % (fail_dir, ))
err_info = {
'src_filename': src_filename,
'cmd_filename': cmd_filename,
'readme_filename': readme_filename,
'fail_dir': fail_dir,
'log_filename': log_filename
}
return status, output, lib_filename, err_info