def _transfer_output_files(
self, tool_output_dir, working_dir, output_dir, patterns
):
"""Transfer files created by the tool in the container to the output directory.
@param tool_output_dir:
The directory under which all tool output files are created.
@param working_dir: The absolute working directory of the tool in the container.
@param output_dir: the directory where to write result files
@param patterns: a list of patterns of files to retrieve as result files
"""
assert output_dir and patterns
if any(os.path.isabs(pattern) for pattern in patterns):
base_dir = tool_output_dir
else:
base_dir = tool_output_dir + working_dir
def transfer_file(abs_file):
assert abs_file.startswith(base_dir)
# We ignore (empty) directories, because we create them for hidden dirs etc.
# We ignore device nodes, because overlayfs creates them.
# We also ignore all other files (symlinks, fifos etc.),
# because they are probably irrelevant, and just handle regular files.
file = os.path.join("/", os.path.relpath(abs_file, base_dir))
if (
os.path.isfile(abs_file)
and not os.path.islink(abs_file)
and not container.is_container_system_config_file(file)
):
target = output_dir + file
logging.debug("Transferring output file %s to %s", abs_file, target)
try:
os.makedirs(os.path.dirname(target))
except EnvironmentError:
pass # exist_ok=True not supported on Python 2
try:
# move is more efficient than copy in case both abs_file and target
# are on the same filesystem, and it avoids matching the file again
# with the next pattern.
shutil.move(abs_file, target)
except EnvironmentError as e:
logging.warning("Could not retrieve output file '%s': %s", file, e)
for pattern in patterns:
if os.path.isabs(pattern):
pattern = tool_output_dir + pattern
else:
pattern = tool_output_dir + os.path.join(working_dir, pattern)
# normalize pattern for preventing directory traversal attacks:
for abs_file in util.maybe_recursive_iglob(
os.path.normpath(pattern), recursive=True
):
# Recursive matching is only supported starting with Python 3.5, so we
# allow the user to match directories and transfer them recursively.
if os.path.isdir(abs_file):
for root, unused_dirs, files in os.walk(abs_file):
for file in files:
transfer_file(os.path.join(root, file))
else:
transfer_file(abs_file)
def _transfer_output_files(self, temp_dir, working_dir, output_dir, patterns):
"""Transfer files created by the tool in the container to the output directory.
@param temp_dir: The base directory under which all our directories are created.
@param working_dir: The absolute working directory of the tool in the container.
@param output_dir: the directory where to write result files
@param patterns: a list of patterns of files to retrieve as result files
"""
assert output_dir and patterns
tool_output_dir = os.path.join(temp_dir, "temp")
if any(os.path.isabs(pattern) for pattern in patterns):
base_dir = tool_output_dir
else:
base_dir = tool_output_dir + working_dir
def transfer_file(abs_file):
assert abs_file.startswith(base_dir)
# We ignore (empty) directories, because we create them for hidden dirs etc.
# We ignore device nodes, because overlayfs creates them.
# We also ignore all other files (symlinks, fifos etc.),
# because they are probably irrelevant, and just handle regular files.
file = os.path.join("/", os.path.relpath(abs_file, base_dir))
if (
os.path.isfile(abs_file)
and not os.path.islink(abs_file)
and not container.is_container_system_config_file(file)
):
target = output_dir + file
try:
os.makedirs(os.path.dirname(target))
except EnvironmentError:
pass # exist_ok=True not supported on Python 2
try:
# move is more efficient than copy in case both abs_file and target
# are on the same filesystem, and it avoids matching the file again
# with the next pattern.
shutil.move(abs_file, target)
except EnvironmentError as e:
logging.warning("Could not retrieve output file '%s': %s", file, e)
for pattern in patterns:
if os.path.isabs(pattern):
pattern = tool_output_dir + pattern
else:
pattern = tool_output_dir + os.path.join(working_dir, pattern)
# normalize pattern for preventing directory traversal attacks:
for abs_file in util.maybe_recursive_iglob(os.path.normpath(pattern)):
# Recursive matching is only supported starting with Python 3.5,
# so we allow the user to match directories and transfer them recursively.
if os.path.isdir(abs_file):
for root, unused_dirs, files in os.walk(abs_file):
for file in files:
transfer_file(os.path.join(root, file))
else:
transfer_file(abs_file)