def delete_disconnected_fs_submissions(args):
"""
Delete files without a corresponding Submission record in the database.
"""
with app_context():
disconnected_files = find_disconnected_fs_submissions(args.store_dir)
bytes_deleted = 0
time_elapsed = 0.0
rate = 1.0
filecount = len(disconnected_files)
eta = 1.0
eta_msg = ""
for i, f in enumerate(disconnected_files, 1):
remove = args.force
if not args.force:
remove = input("Enter 'y' to delete {}: ".format(f)) == "y"
if remove:
filesize = os.stat(f).st_size
if i > 1:
eta = filesize / rate
eta_msg = " (ETA to remove {:d} bytes: {:.0f}s )".format(filesize, eta)
print("Securely removing file {}/{} {}{}...".format(i, filecount, f, eta_msg))
start = time.time()
secure_delete(f)
file_elapsed = time.time() - start
bytes_deleted += filesize
time_elapsed += file_elapsed
rate = bytes_deleted / time_elapsed
print(
"elapsed: {:.2f}s rate: {:.1f} MB/s overall rate: {:.1f} MB/s".format(
file_elapsed, filesize / 1048576 / file_elapsed, rate / 1048576
)
)
else:
print("Not removing {}.".format(f))
def clear_shredder(self):
current_app.logger.info("Clearing shredder")
directories = []
targets = []
for directory, subdirs, files in os.walk(self.shredder_path):
for subdir in subdirs:
real_subdir = os.path.realpath(os.path.join(directory, subdir))
if self.shredder_contains(real_subdir):
directories.append(real_subdir)
for f in files:
abs_file = os.path.abspath(os.path.join(directory, f))
if os.path.islink(abs_file):
# Somehow, a symbolic link was created in the
# store. This shouldn't happen in normal
# operations. Just remove the link; don't try to
# shred its target. Note that we only have special
# handling for symlinks. Hard links -- which
# again, shouldn't occur in the store -- will
# result in the file data being shredded once for
# each link.
current_app.logger.info("Deleting link {} to {}".format(
abs_file, os.readlink(abs_file)))
os.unlink(abs_file)
continue
if self.shredder_contains(abs_file):
targets.append(abs_file)
target_count = len(targets)
current_app.logger.info("Files to delete: {}".format(target_count))
for i, t in enumerate(targets, 1):
current_app.logger.info("Securely deleting file {}/{}: {}".format(
i, target_count, t))
rm.secure_delete(t)
current_app.logger.info("Securely deleted file {}/{}: {}".format(
i, target_count, t))
directories_to_remove = set(directories)
dir_count = len(directories_to_remove)
for i, d in enumerate(reversed(sorted(directories_to_remove)), 1):
current_app.logger.debug("Removing directory {}/{}: {}".format(
i, dir_count, d))
os.rmdir(d)
current_app.logger.debug("Removed directory {}/{}: {}".format(
i, dir_count, d))
def test_secure_delete(config):
content = "abc123\n"
testfile = "test_shred.txt"
# Shred a file
testfile1 = os.path.abspath(os.path.join(config.TEMP_DIR, testfile))
with open(testfile1, "w") as f:
f.write(content)
assert os.path.exists(testfile1)
rm.secure_delete(testfile1)
assert os.path.exists(testfile1) is False
# Shred a directory
testdir = os.path.abspath(os.path.join(config.TEMP_DIR, "shredtest1"))
testsubdir1 = os.path.abspath(os.path.join(testdir, "shredtest1.1"))
testsubdir2 = os.path.abspath(os.path.join(testdir, "shredtest1.2"))
os.makedirs(testsubdir1)
os.makedirs(testsubdir2)
testfile1 = os.path.abspath(os.path.join(testdir, testfile))
with open(testfile1, "w") as f:
f.write(content)
testfile2 = os.path.abspath(os.path.join(testsubdir1, testfile))
with open(testfile2, "w") as f:
f.write(content)
assert os.path.exists(testfile1)
assert os.path.exists(testfile2)
rm.secure_delete(testdir)
assert os.path.exists(testfile1) is False
assert os.path.exists(testfile2) is False
assert os.path.exists(testsubdir1) is False
assert os.path.exists(testsubdir2) is False
assert os.path.exists(testdir) is False