def transfer_file(abs_file):
assert abs_file.startswith(base_dir)
# We ignore (empty) directories, because we create them for hidden dirs etc.
# We ignore device nodes, because overlayfs creates them.
# We also ignore all other files (symlinks, fifos etc.),
# because they are probably irrelevant, and just handle regular files.
file = os.path.join("/", os.path.relpath(abs_file, base_dir))
if (
os.path.isfile(abs_file)
and not os.path.islink(abs_file)
and not container.is_container_system_config_file(file)
):
target = output_dir + file
nonlocal file_count
file_count += 1
if file_count <= _MAX_RESULT_FILE_LOG_COUNT:
logging.debug("Transferring output file %s to %s", abs_file, target)
if file_count == _MAX_RESULT_FILE_LOG_COUNT:
logging.debug(
"%s output files transferred, "
"further files will not be logged.",
file_count,
)
os.makedirs(os.path.dirname(target), exist_ok=True)
try:
# move is more efficient than copy in case both abs_file and target
# are on the same filesystem, and it avoids matching the file again
# with the next pattern.
shutil.move(abs_file, target)
except OSError as e:
logging.warning("Could not retrieve output file '%s': %s", file, e)
def transfer_file(abs_file):
assert abs_file.startswith(base_dir)
# We ignore (empty) directories, because we create them for hidden dirs etc.
# We ignore device nodes, because overlayfs creates them.
# We also ignore all other files (symlinks, fifos etc.),
# because they are probably irrelevant, and just handle regular files.
file = os.path.join("/", os.path.relpath(abs_file, base_dir))
if (
os.path.isfile(abs_file)
and not os.path.islink(abs_file)
and not container.is_container_system_config_file(file)
):
target = output_dir + file
try:
os.makedirs(os.path.dirname(target))
except EnvironmentError:
pass # exist_ok=True not supported on Python 2
try:
# move is more efficient than copy in case both abs_file and target
# are on the same filesystem, and it avoids matching the file again
# with the next pattern.
shutil.move(abs_file, target)
except EnvironmentError as e:
logging.warning("Could not retrieve output file '%s': %s", file, e)
def transfer_file(abs_file):
assert abs_file.startswith(base_dir)
# We ignore (empty) directories, because we create them for hidden dirs etc.
# We ignore device nodes, because overlayfs creates them.
# We also ignore all other files (symlinks, fifos etc.),
# because they are probably irrelevant, and just handle regular files.
file = os.path.join("/", os.path.relpath(abs_file, base_dir))
if (os.path.isfile(abs_file) and not os.path.islink(abs_file)
and not container.is_container_system_config_file(file)):
target = output_dir + file
logging.debug("Transferring output file %s to %s", abs_file,
target)
try:
os.makedirs(os.path.dirname(target))
except EnvironmentError:
pass # exist_ok=True not supported on Python 2
try:
# move is more efficient than copy in case both abs_file and target
# are on the same filesystem, and it avoids matching the file again
# with the next pattern.
shutil.move(abs_file, target)
except EnvironmentError as e:
logging.warning("Could not retrieve output file '%s': %s",
file, e)
def run(self):
while not self._finished.is_set():
self._finished.wait(_CHECK_INTERVAL_SECONDS)
files_count = 0
files_size = 0
start_time = util.read_monotonic_time()
for current_dir, dirs, files in os.walk(self._path):
for file in files:
abs_file = os.path.join(current_dir, file)
file = '/' + os.path.relpath(file, self._path) # as visible for tool
if (not container.is_container_system_config_file(file) and
os.path.isfile(abs_file) and not os.path.islink(abs_file)):
files_count += 1
if self._files_size_limit:
try:
files_size += os.path.getsize(abs_file)
except OSError:
# possibly just deleted
pass
if self._check_limit(files_count, files_size):
return
duration = util.read_monotonic_time() - start_time
logging.debug(
"FileHierarchyLimitThread for process %d: "
"files count: %d, files size: %d, scan duration %fs",
self._pid_to_kill, files_count, files_size, duration)
if duration > _DURATION_WARNING_THRESHOLD:
logging.warning(
"Scanning file hierarchy for enforcement of limits took %ds.", duration)