def transfer_file(abs_file): assert abs_file.startswith(base_dir) # We ignore (empty) directories, because we create them for hidden dirs etc. # We ignore device nodes, because overlayfs creates them. # We also ignore all other files (symlinks, fifos etc.), # because they are probably irrelevant, and just handle regular files. file = os.path.join("/", os.path.relpath(abs_file, base_dir)) if ( os.path.isfile(abs_file) and not os.path.islink(abs_file) and not container.is_container_system_config_file(file) ): target = output_dir + file nonlocal file_count file_count += 1 if file_count <= _MAX_RESULT_FILE_LOG_COUNT: logging.debug("Transferring output file %s to %s", abs_file, target) if file_count == _MAX_RESULT_FILE_LOG_COUNT: logging.debug( "%s output files transferred, " "further files will not be logged.", file_count, ) os.makedirs(os.path.dirname(target), exist_ok=True) try: # move is more efficient than copy in case both abs_file and target # are on the same filesystem, and it avoids matching the file again # with the next pattern. shutil.move(abs_file, target) except OSError as e: logging.warning("Could not retrieve output file '%s': %s", file, e)
def transfer_file(abs_file): assert abs_file.startswith(base_dir) # We ignore (empty) directories, because we create them for hidden dirs etc. # We ignore device nodes, because overlayfs creates them. # We also ignore all other files (symlinks, fifos etc.), # because they are probably irrelevant, and just handle regular files. file = os.path.join("/", os.path.relpath(abs_file, base_dir)) if ( os.path.isfile(abs_file) and not os.path.islink(abs_file) and not container.is_container_system_config_file(file) ): target = output_dir + file try: os.makedirs(os.path.dirname(target)) except EnvironmentError: pass # exist_ok=True not supported on Python 2 try: # move is more efficient than copy in case both abs_file and target # are on the same filesystem, and it avoids matching the file again # with the next pattern. shutil.move(abs_file, target) except EnvironmentError as e: logging.warning("Could not retrieve output file '%s': %s", file, e)
def transfer_file(abs_file): assert abs_file.startswith(base_dir) # We ignore (empty) directories, because we create them for hidden dirs etc. # We ignore device nodes, because overlayfs creates them. # We also ignore all other files (symlinks, fifos etc.), # because they are probably irrelevant, and just handle regular files. file = os.path.join("/", os.path.relpath(abs_file, base_dir)) if (os.path.isfile(abs_file) and not os.path.islink(abs_file) and not container.is_container_system_config_file(file)): target = output_dir + file logging.debug("Transferring output file %s to %s", abs_file, target) try: os.makedirs(os.path.dirname(target)) except EnvironmentError: pass # exist_ok=True not supported on Python 2 try: # move is more efficient than copy in case both abs_file and target # are on the same filesystem, and it avoids matching the file again # with the next pattern. shutil.move(abs_file, target) except EnvironmentError as e: logging.warning("Could not retrieve output file '%s': %s", file, e)
def run(self): while not self._finished.is_set(): self._finished.wait(_CHECK_INTERVAL_SECONDS) files_count = 0 files_size = 0 start_time = util.read_monotonic_time() for current_dir, dirs, files in os.walk(self._path): for file in files: abs_file = os.path.join(current_dir, file) file = '/' + os.path.relpath(file, self._path) # as visible for tool if (not container.is_container_system_config_file(file) and os.path.isfile(abs_file) and not os.path.islink(abs_file)): files_count += 1 if self._files_size_limit: try: files_size += os.path.getsize(abs_file) except OSError: # possibly just deleted pass if self._check_limit(files_count, files_size): return duration = util.read_monotonic_time() - start_time logging.debug( "FileHierarchyLimitThread for process %d: " "files count: %d, files size: %d, scan duration %fs", self._pid_to_kill, files_count, files_size, duration) if duration > _DURATION_WARNING_THRESHOLD: logging.warning( "Scanning file hierarchy for enforcement of limits took %ds.", duration)