def test_compare_with_changed_symlink(self):
# Make reference directory
os.mkdir("test1")
self._populate_dir("test1")
# Make directory to compare
self._copy_dir("test1", "test2")
# Change link target (changed_link)
# Small delay to ensure time is different
time.sleep(.01)
os.remove("test2/test.lnk")
os.symlink("test1.dir", "test2/test.lnk")
# Build indexes
indx1 = FilesystemObjectIndex("test1")
indx2 = FilesystemObjectIndex("test2")
# Do comparison
diff = compare(indx1, indx2)
self.assertEqual(diff.missing, [])
self.assertEqual(diff.extra, [])
self.assertEqual(diff.restricted_source, [])
self.assertEqual(diff.restricted_target, [])
self.assertEqual(diff.changed_type, [])
self.assertEqual(diff.changed_size, [])
self.assertEqual(diff.changed_md5, [])
self.assertEqual(diff.changed_link, ["test.lnk"])
self.assertEqual(diff.changed_time, ["test.lnk"])
def test_compare_with_restricted_objects(self):
# Make reference directory
os.mkdir("test1")
self._populate_dir("test1")
# Make directory to compare
self._copy_dir("test1", "test2")
# Remove permissions on source subdir (restricted_source)
os.chmod("test1/test1.dir", 0o055)
# Remove permissions on target subdir (restricted_target)
os.chmod("test2/test2.dir", 0o055)
# Build indexes
indx1 = FilesystemObjectIndex("test1")
indx2 = FilesystemObjectIndex("test2")
# Do comparison
diff = compare(indx1, indx2)
self.assertEqual(diff.missing, ["test2.dir/test.txt"])
self.assertEqual(diff.extra, [
"test1.dir/sub.dir", "test1.dir/sub.dir/test.lnk",
"test1.dir/test.txt"
])
self.assertEqual(diff.restricted_source, ["test1.dir"])
self.assertEqual(diff.restricted_target, ["test2.dir"])
self.assertEqual(diff.changed_type, [])
self.assertEqual(diff.changed_size, [])
self.assertEqual(diff.changed_md5, [])
self.assertEqual(diff.changed_link, [])
self.assertEqual(diff.changed_time, [])
def test_compare_with_differences(self):
# Make reference directory
os.mkdir("test1")
self._populate_dir("test1")
# Make directory to compare
time.sleep(.01)
self._copy_dir("test1", "test2")
# Remove a file (missing)
os.remove("test2/test.txt")
# Add a new file (extra)
with open("test2/test2.dir/new.txt", "wt") as fp:
fp.write("blah")
# Change content (changed_size/changed_md5/changed_time)
time.sleep(.01)
with open("test2/test2.dir/test.txt", "wt") as fp:
fp.write("blah")
# Build indexes
indx1 = FilesystemObjectIndex("test1")
indx2 = FilesystemObjectIndex("test2")
# Do comparison
diff = compare(indx1, indx2)
self.assertEqual(diff.missing, ["test.txt"])
self.assertEqual(diff.extra, ["test2.dir/new.txt"])
self.assertEqual(diff.restricted_source, [])
self.assertEqual(diff.restricted_target, [])
self.assertEqual(diff.changed_type, [])
self.assertEqual(diff.changed_size, ["test2.dir/test.txt"])
self.assertEqual(diff.changed_md5, ["test2.dir/test.txt"])
self.assertEqual(diff.changed_link, [])
self.assertEqual(diff.changed_time,
["test2.dir", "test2.dir/test.txt"])
def test_compare_empty(self):
os.mkdir("test1")
indx1 = FilesystemObjectIndex("test1")
os.mkdir("test2")
indx2 = FilesystemObjectIndex("test2")
diff = compare(indx1, indx2)
self.assertEqual(diff.missing, [])
self.assertEqual(diff.extra, [])
self.assertEqual(diff.restricted_source, [])
self.assertEqual(diff.restricted_target, [])
self.assertEqual(diff.changed_type, [])
self.assertEqual(diff.changed_size, [])
self.assertEqual(diff.changed_md5, [])
self.assertEqual(diff.changed_link, [])
self.assertEqual(diff.changed_time, [])
def test_find_owned_by_users(self):
# Make reference directory
os.mkdir("test")
self._populate_dir("test")
# Build index
indx = FilesystemObjectIndex("test")
# Find files owned by current user
current_username = getpass.getuser()
self.assertEqual(find(indx, exts="fastq", users=current_username), [
"analysis/fastqs/PJB_R1.fastq", "analysis/fastqs/PJB_R2.fastq",
"analysis/trimming/PJB_R1.trimmed.fastq",
"analysis/trimming/PJB_R2.trimmed.fastq",
"fastqs/PJB_S1_R1_001.fastq.gz", "fastqs/PJB_S1_R2_001.fastq.gz"
])
# Find files owned by different user
self.assertEqual(
find(indx, exts="fastq", users="non_existent_user0123"), [])
# Find files owned by multiple users
self.assertEqual(
find(indx,
exts="fastq",
users=",".join((current_username, "non_existent_user0123"))),
[
"analysis/fastqs/PJB_R1.fastq", "analysis/fastqs/PJB_R2.fastq",
"analysis/trimming/PJB_R1.trimmed.fastq",
"analysis/trimming/PJB_R2.trimmed.fastq",
"fastqs/PJB_S1_R1_001.fastq.gz",
"fastqs/PJB_S1_R2_001.fastq.gz"
])
def test_find_no_conditions(self):
# Make reference directory
os.mkdir("test")
self._populate_dir("test")
# Build index
indx = FilesystemObjectIndex("test")
self.assertEqual(len(find(indx)), 0)
def test_check_accessibility_without_restricted_objects(self):
# Make reference directory
os.mkdir("test")
self._populate_dir("test")
# Build index
indx = FilesystemObjectIndex("test")
self.assertEqual(check_accessibility(indx), [])
def test_compare_with_changed_types(self):
# Make reference directory
os.mkdir("test1")
self._populate_dir("test1")
# Make directory to compare
time.sleep(0.01)
self._copy_dir("test1", "test2")
# Change a file to a directory
os.remove("test2/test.txt")
os.mkdir("test2/test.txt")
# Change a file to a link
os.remove("test2/test2.dir/test.txt")
os.symlink(".", "test2/test2.dir/test.txt")
# Change a directory to a file
shutil.rmtree("test2/test1.dir")
with open("test2/test1.dir", "wt") as fp:
fp.write("blah")
# Change a directory to a link
shutil.rmtree("test2/test3.dir")
os.symlink(".", "test2/test3.dir")
# Change a link to a file
os.remove("test2/test1.lnk")
with open("test2/test1.lnk", "wt") as fp:
fp.write("blah")
# Change a link to a directory
os.remove("test2/test2.lnk")
os.mkdir("test2/test2.lnk")
# Build indexes
indx1 = FilesystemObjectIndex("test1")
indx2 = FilesystemObjectIndex("test2")
# Do comparison
diff = compare(indx1, indx2)
self.assertEqual(diff.missing, [
"test1.dir/sub.dir", "test1.dir/sub.dir/test.lnk",
"test1.dir/test.txt"
])
self.assertEqual(diff.extra, [])
self.assertEqual(diff.restricted_source, [])
self.assertEqual(diff.restricted_target, [])
self.assertEqual(diff.changed_type, [
"test.txt", "test1.dir", "test1.lnk", "test2.dir/test.txt",
"test2.lnk", "test3.dir"
])
self.assertEqual(diff.changed_size, [])
self.assertEqual(diff.changed_md5, [])
self.assertEqual(diff.changed_link, [])
self.assertEqual(diff.changed_time, ["test2.dir"])
def test_find_multiple_extensions(self):
# Make reference directory
os.mkdir("test")
self._populate_dir("test")
# Build index
indx = FilesystemObjectIndex("test")
self.assertEqual(find(indx, exts="sam,bam"), [
"analysis/mapping/PJB.trimmed.bam",
"analysis/mapping/PJB.trimmed.sam"
])
def test_find_no_compressed_files(self):
# Make reference directory
os.mkdir("test")
self._populate_dir("test")
# Build index
indx = FilesystemObjectIndex("test")
self.assertEqual(find(indx, exts="fastq", nocompressed=True), [
"analysis/fastqs/PJB_R1.fastq", "analysis/fastqs/PJB_R2.fastq",
"analysis/trimming/PJB_R1.trimmed.fastq",
"analysis/trimming/PJB_R2.trimmed.fastq"
])
def test_find_one_extension(self):
# Make reference directory
os.mkdir("test")
self._populate_dir("test")
# Build index
indx = FilesystemObjectIndex("test")
self.assertEqual(find(indx, exts="fastq"), [
"analysis/fastqs/PJB_R1.fastq", "analysis/fastqs/PJB_R2.fastq",
"analysis/trimming/PJB_R1.trimmed.fastq",
"analysis/trimming/PJB_R2.trimmed.fastq",
"fastqs/PJB_S1_R1_001.fastq.gz", "fastqs/PJB_S1_R2_001.fastq.gz"
])
def test_check_accessibility_with_restricted_objects(self):
# Make reference directory
os.mkdir("test")
self._populate_dir("test")
# Remove permissions on subdir
os.chmod("test/test1.dir", 0o055)
# Remove permissions on file
os.chmod("test/test2.dir/test.txt", 0o044)
# Build index
indx = FilesystemObjectIndex("test")
self.assertEqual(check_accessibility(indx),
["test1.dir", "test2.dir/test.txt"])
def test_populated_objectindex(self):
# Add some objects to current dir
dirs = (
"test1.dir",
"test2.dir",
"test1.dir/sub.dir",
)
files = (
"test.txt",
"test1.dir/test.txt",
"test2.dir/test.txt",
)
symlinks = {
"test.lnk": "test.txt",
"test1.dir/sub.dir/test.lnk": "../test.txt"
}
for d in dirs:
os.mkdir(d)
for f in files:
with open(f, "wt") as fp:
fp.write("test\n")
for s in symlinks:
os.symlink(symlinks[s], s)
# Build the index
indx = FilesystemObjectIndex(self.wd)
self.assertEqual(len(indx), 8)
# Check __contains__
for d in dirs:
self.assertTrue(d in indx)
for d in files:
self.assertTrue(f in indx)
for s in symlinks:
self.assertTrue(s in indx)
self.assertFalse("missing" in indx)
# Check names
for n in indx.names:
self.assertTrue((n in dirs) or (n in files) or (n in symlinks))
# Check __getitem__
for n in indx.names:
obj = indx[n]
self.assertTrue(isinstance(obj, FilesystemObject))
self.assertEqual(obj.path, os.path.join(self.wd, n))
# Don't like this way of invoking __getitem__
# but seems to work for testing
self.assertRaises(KeyError, indx.__getitem__, "missing")
def test_find_minimum_size(self):
# Make reference directory
os.mkdir("test")
self._populate_dir("test")
# Build index
indx = FilesystemObjectIndex("test")
# Find files owned by current user
current_username = getpass.getuser()
self.assertEqual(find(indx, exts="fastq", users=current_username), [
"analysis/fastqs/PJB_R1.fastq", "analysis/fastqs/PJB_R2.fastq",
"analysis/trimming/PJB_R1.trimmed.fastq",
"analysis/trimming/PJB_R2.trimmed.fastq",
"fastqs/PJB_S1_R1_001.fastq.gz", "fastqs/PJB_S1_R2_001.fastq.gz"
])
# Find files larger than specified sizes
self.assertEqual(find(indx, size=100), [
"big_file",
])
self.assertEqual(find(indx, size=1000000), [])
def test_empty_objectindex(self):
indx = FilesystemObjectIndex(self.wd)
self.assertEqual(indx.names, [])
self.assertEqual(len(indx), 0)