def test_remove_empty_dirs(tmpdir):
"""Test that remove_empty_directories does so."""
# expect an exception if a non-list gets passed in
stats = dd.DedupStats()
with pytest.raises(RuntimeError):
dd.remove_empty_directories('x', stats, False)
assert len(stats.dir_stats) == 0
# expect silent failure if passing in a non-existent directory
dd.remove_empty_directories([str(tmpdir.join('x'))], stats, False)
assert tmpdir.exists()
assert len(stats.dir_stats) == 1
assert str(tmpdir.join('x')) in stats.dir_stats
assert stats.dir_stats[str(tmpdir.join('x'))].empty_dir_count == 0
# try removing two empty dirs side by side
DirectoryBuilder("""
d1:
d2:""", tmpdir)
d1 = tmpdir.join('d1')
d2 = tmpdir.join('d2')
stats = dd.DedupStats()
dd.remove_empty_directories([str(d1), str(d2)], stats, False)
expect_exists([d1, d2], False)
assert tmpdir.exists()
assert len(stats.dir_stats) == 2
assert stats.dir_stats[str(d1)].empty_dir_count == 1
assert stats.dir_stats[str(d2)].empty_dir_count == 1
# try two nested dirs with a file in the deepest
DirectoryBuilder("""
d1:
d2:
f1""", tmpdir)
d1 = tmpdir.join('d1')
d2 = d1.join('d2')
f1 = d2.join('f1')
stats = dd.DedupStats()
dd.remove_empty_directories([str(d1)], stats, False)
expect_exists([d1, d2, f1], True)
assert len(stats.dir_stats) == 1
assert stats.dir_stats[str(d1)].empty_dir_count == 0
# now get rid of the file and expect both directories to be removed
f1.remove()
dd.remove_empty_directories([str(d1)], stats, False)
assert len(stats.dir_stats) == 1
expect_exists([d1, d2], False)
assert tmpdir.exists()
assert stats.dir_stats[str(d1)].empty_dir_count == 2
def test_empty_dirs(tmpdir):
DirectoryBuilder("""
d1:
d2:
""", tmpdir)
assert tmpdir.join('d1').check(dir=1)
assert tmpdir.join('d2').check(dir=1)
def test_structure(tmpdir):
DirectoryBuilder(
"""
d1:
d2:
f1*/1
f2/1
f3**
d3:
f4**
f5*""", tmpdir)
d1 = tmpdir.join('d1')
d2 = tmpdir.join('d1', 'd2')
d3 = tmpdir.join('d3')
f1 = d2.join('f1')
f2 = d2.join('f2')
f3 = d2.join('f3')
f4 = d3.join('f4')
f5 = tmpdir.join('f5')
assert d1.check(dir=1)
assert d2.check(dir=1)
assert d3.check(dir=1)
assert f1.size() == f2.size()
assert identical(f1, f5)
assert identical(f3, f4)
def test_overlapping_target_and_ro(tmpdir):
"""Test that an error is raised if a read-only directory is a parent of a target
directory."""
DirectoryBuilder(
"""
readonly_dir_base:
ro1:
rf1*
ro2:
rf2**
rf3**
ro3:
target_dir_base:
td1:
tf1
td2:
tf2***
tf3***
tf4
""", tmpdir)
ro1 = tmpdir.join('readonly_dir_base', 'ro1')
ro2 = tmpdir.join('readonly_dir_base', 'ro2')
ro3 = tmpdir.join('readonly_dir_base', 'ro3')
td1 = ro3.join('target_dir_base', 'td1')
td2 = ro3.join('target_dir_base', 'td2')
run_dedup([td1, td2], [ro1, ro2]) # should be ok
with pytest.raises(SystemExit):
run_dedup([td1, td2], [ro1, ro2, ro3]) # should exit
def enlarge(self):
"""
If my parentbuilder has any subdirectories, see if they
contain a Confix2.dir file. If any, wrap DirectoryBuilder
objects around them and add them to the parentbuilder.
"""
super(SubdirectoryRecognizer, self).enlarge()
errors = []
for name, entry in self.parentbuilder().directory().entries():
if not isinstance(entry, VFSDirectory):
continue
if entry in self.__recognized_directories:
continue
confix2_dir_file = entry.get(const.CONFIX2_DIR)
if confix2_dir_file is None:
continue
if not isinstance(confix2_dir_file, VFSFile):
errors.append(
Error(
os.sep.join(
confix2_dir_file.relpath(
self.package().rootdirectory())) +
' is not a file'))
continue
self.parentbuilder().add_builder(DirectoryBuilder(directory=entry))
self.__recognized_directories.add(entry)
pass
if len(errors):
raise Error('There were errors in directory '+\
os.sep.join(self.parentbuilder().directory().relpath(self.package().rootdirectory())), errors)
pass
def test_readonly(tmpdir, remove_empty_dirs):
"""Ensure files are not deleted from readonly directories.
Args:
tmpdir: pytest temporary directory fixture
"""
DirectoryBuilder(
"""
target_dir_base:
td1:
tf1*
td2:
tf2***
tf3***
tf4
readonly_dir_base:
ro1:
rf1*
ro2:
rf2**
rf3**
ro3:
""", tmpdir)
td1 = tmpdir.join('target_dir_base', 'td1')
td2 = tmpdir.join('target_dir_base', 'td2')
tf1 = td1.join('tf1')
tf2 = td2.join('tf2')
tf3 = td2.join('tf3')
tf4 = td2.join('tf4')
all_target_dirs = [td1, td2]
ro1 = tmpdir.join('readonly_dir_base', 'ro1')
ro2 = tmpdir.join('readonly_dir_base', 'ro2')
ro3 = tmpdir.join('readonly_dir_base', 'ro3')
rf1 = ro1.join('rf1')
rf2 = ro2.join('rf2')
rf3 = ro2.join('rf3')
all_ro_dirs = [ro1, ro2, ro3]
all_ro_files = [rf1, rf2, rf3]
run_dedup(all_target_dirs, all_ro_dirs, remove_empty_dirs)
# files in readonly directories must never be deleted
expect_exists(all_ro_files, True)
# readonly directories, even empty ones, must never be deleted
expect_exists(all_ro_dirs, True)
# the unique target file should always still exist
assert tf4.exists()
# now check results that differ based on remove_empty_dirs
if not remove_empty_dirs:
# no target dirs should have been removed
expect_exists(all_target_dirs, True)
else:
# empty target dirs (namely td1) should have been removed
assert not td1.exists()
# but td2 should not be removed
assert td2.exists()
# tf1 should have been deleted because it is made a duplicate by read-only file rf1
assert not tf1.exists()
# exactly one of tf2 and tf3 should exist. Exactly which was deleted depends on
# the order that os.walk reported them to dedup.walk_dirs().
assert tf2.exists() ^ tf3.exists()
def DIRECTORY(self, path):
if type(path) not in (list, tuple):
raise Error('DIRECTORY(' + str(path) +
'): path argument must be list or tuple')
directory = self.__dirbuilder.directory().find(path=path)
if directory is None:
raise Error('DIRECTORY(): could not find directory ' + str(path))
dirbuilder = DirectoryBuilder(directory=directory)
self.__dirbuilder.add_builder(dirbuilder)
return dirbuilder
def test_is_same_or_subdir(tmpdir):
"""Test that `is_same_or_subdir` correctly identifies directories that are the same
or that it identifies one directory as a subdirectory of another."""
DirectoryBuilder("""
t1:
t1_1:
t2:
""", tmpdir)
t1 = tmpdir.join('t1')
t1_1 = t1.join('t1_1')
t2 = tmpdir.join('t2')
# Passing pathlike objects would work, but the live code will be passing in strings
assert dd.is_same_or_subdir(str(t1), str(t1))
assert dd.is_same_or_subdir(str(t1_1), str(t1))
assert dd.is_same_or_subdir(str(t1_1), str(tmpdir))
assert not dd.is_same_or_subdir(str(t1), str(t2))
def test_same_target_dirs(tmpdir):
"""Expect that an error is raised if the same target directory is specified more
than once."""
DirectoryBuilder(
"""
t1:
foo*
t2:
bar*
ro:
baz
""", tmpdir)
t1 = tmpdir.join('t1')
ro = tmpdir.join('ro')
with pytest.raises(SystemExit):
run_dedup([t1, t1], [ro], remove_empty=True)
expect_exists([t1, ro], True)
def test_size_and_content(tmpdir):
# This usage is allowed because no conflict arises:
# f1 is created and its contents are saved as content group * (as before)
# f2 is created and its contents are copied from content group *. Size
# group 1 is also created, and set to the size of f2.
# f3 is created and made to be the saved size of size group 1.
DirectoryBuilder(
"""
f1*
f2*/1
f3/1
""", tmpdir)
f1 = tmpdir.join('f1')
f2 = tmpdir.join('f2')
f3 = tmpdir.join('f3')
assert f2.size() == f3.size()
assert identical(f1, f2)
def add_confix_admin(package):
"""
If necessary, add a directory <packageroot>/confix-admin and the
associated directory builder.
@return: DirectoryBuilder instance representing the confix-admin
directory
"""
admin_builder = package.rootbuilder().find_entry_builder([const.ADMIN_DIR])
if admin_builder:
return admin_builder
admin_dir = package.rootdirectory().get(const.ADMIN_DIR)
if admin_dir is None:
admin_dir = package.rootdirectory().add(name=const.ADMIN_DIR,
entry=Directory())
pass
return package.rootbuilder().add_builder(
DirectoryBuilder(directory=admin_dir))
def test_size_conflict(tmpdir):
# If a file is in both a size group and a content group, the size group must have
# been created by a file that is also in the content group.
# This should throw an exception because:
# f1 is created and its contents are saved as content group *
# f2 is created and its size saved in size group 1
# f3 must now have the same content as f1 because it is in content group
# *, however it is also in size group 1. Size group 1 has some random size
# that could differ from the size of content group *, therefore this situation
# is syntactically valid but semantically invalid.
with pytest.raises(ValueError):
DirectoryBuilder(
"""
f1*
f2/1
f3*/1
""", tmpdir)
def test_skip(tmpdir, monkeypatch):
"""Test user choice of skipping the current directory."""
def mock_input(*args, **kwargs):
"""On the nth invocation (0 based) return the nth string in parent scope
`input_values` list."""
if 'invocation_count' not in mock_input.__dict__:
mock_input.invocation_count = 0
result = input_values[mock_input.invocation_count]
mock_input.invocation_count += 1
return result
DirectoryBuilder(
"""
td1:
tf1*
tf2**
td2:
tf3*
tf4**
td2_1:
tf6*
td3:
tf5**
""", tmpdir)
td1 = tmpdir.join('td1')
td2 = tmpdir.join('td2')
td2_1 = td2.join('td2_1')
td3 = tmpdir.join('td3')
tf1 = td1.join('tf1')
tf2 = td1.join('tf2')
tf3 = td2.join('tf3')
tf4 = td2.join('tf4')
tf5 = td3.join('tf5')
tf6 = td2_1.join('tf6')
# monkey patch the 'input' builtin function to use our input function
monkeypatch.setattr('builtins.input', mock_input)
input_values = ['skip', 'yes']
# expect confirm to ask to delete tf3, and answer skip
run_dedup([td1, td2, td3], confirm=True)
# expect tf4 and td2_1 to not be examined because of skip
expect_exists([tf1, tf2, tf3, tf4, tf6], True)
# expect confirmation for tf5 to be deleted and supply 'yes'
assert not tf5.exists()
def test_dedup(tmpdir, remove_empty_dirs):
"""Test that duplicate files are removed, but that unique files that happen to have
the same size are not."""
# Build a directory structure for testing
DirectoryBuilder(
"""
d1:
d2:
f1*/1
f2/1
f3**
d3:
f4**
f5*
ro1:
f6
""", tmpdir)
# For readability, get a handle to all the directories and files just created
d1 = tmpdir.join('d1')
d2 = d1.join('d2')
d3 = tmpdir.join('d3')
ro1 = tmpdir.join('ro1')
f1 = d2.join('f1')
f2 = d2.join('f2')
f4 = d3.join('f4')
f3 = d2.join('f3')
f5 = d3.join('f5')
f6 = ro1.join('f6')
# Make sets of files and directories for comparison
all_dirs = {d1, d2, d3, ro1}
all_files = {f1, f2, f3, f4, f5, f6}
expected_unique_files = {f1, f2, f3, f6}
expected_duplicate_files = all_files - expected_unique_files
# Run the deduplifier using parametrized flags
run_dedup([d1, d3], [ro1], remove_empty_dirs)
# expect duplicate files deleted and empty directories removed according to flag,
# however none of the directories other than d3 should ever be removed
expect_exists(all_dirs - {d3}, True)
assert not d3.exists() == remove_empty_dirs
expect_exists(expected_unique_files, True)
expect_exists(expected_duplicate_files, False)
def test_ro_duplicates_multiple_targets(tmpdir):
"""Test that target files matching read-only files are deleted."""
DirectoryBuilder(
"""
t1:
f1*
t2:
f2**
f22****
t3:
f3***
ro1:
rof1*
rof2**
rof3***
rof4****
""", tmpdir)
t1 = tmpdir.join('t1')
t2 = tmpdir.join('t2')
t3 = tmpdir.join('t3')
f1 = t1.join('f1')
f2 = t2.join('f2')
f22 = t2.join('f22')
f3 = t3.join('f3')
ro1 = tmpdir.join('ro1')
rof1 = ro1.join('rof1')
rof2 = ro1.join('rof2')
rof3 = ro1.join('rof3')
rof4 = ro1.join('rof4')
all_target_files = [f1, f2, f22, f3]
all_target_dirs = [t1, t2, t3]
all_ro_files = [rof1, rof2, rof3, rof4]
run_dedup([t1, t2, t3], [ro1], remove_empty=True)
expect_exists(all_target_files, False)
expect_exists(all_target_dirs, False)
expect_exists(all_ro_files, True)
def test_size_group(tmpdir):
DirectoryBuilder("""
f1/1
f2/1
""", tmpdir)
assert tmpdir.join('f1').size() == tmpdir.join('f2').size()
def test_f1(tmpdir):
DirectoryBuilder("f1", tmpdir)
assert tmpdir.join('f1').exists()
def test_syntax(tmpdir):
with pytest.raises(RuntimeError):
DirectoryBuilder(':', tmpdir)
with pytest.raises(RuntimeError):
DirectoryBuilder('$', tmpdir)