def setUp(self):
"""Build directory with test data
"""
self.dir1 = ExampleDirLanguages()
self.dir1.create_directory()
self.dir2 = ExampleDirLanguages()
self.dir2.create_directory()
class TestMd5CheckerComputeMd5sms(unittest.TestCase):
"""Tests for the 'compute_md5sums' method of the Md5Checker class
"""
def setUp(self):
self.example_dir = ExampleDirLanguages()
self.example_dir.create_directory()
def tearDown(self):
self.example_dir.delete_directory()
def test_compute_md5dums(self):
"""Md5Checker.compute_md5sums returns correct md5sums
"""
files = self.example_dir.filelist(include_links=True,full_path=False)
for f,md5 in Md5Checker.compute_md5sums(self.example_dir.dirn,
links=Md5Checker.FOLLOW_LINKS):
self.assertTrue(f in files)
self.assertEqual(md5,self.example_dir.checksum_for_file(f))
def test_compute_md5dums_ignore_links(self):
"""Md5Checker.compute_md5sums ignores links
"""
files = self.example_dir.filelist(include_links=False,full_path=False)
for f,md5 in Md5Checker.compute_md5sums(self.example_dir.dirn,
links=Md5Checker.IGNORE_LINKS):
self.assertTrue(f in files)
self.assertEqual(md5,self.example_dir.checksum_for_file(f))
def test_archive_multiple_data_dirs(self):
"""DataArchiver copies and verifies multiple data directories
"""
# Make additional data dir
self.example_dir2 = ExampleDirLanguages()
self.data_dir2 = self.example_dir2.create_directory()
# Initial checks
dest_dir = os.path.join(self.archive_dir,os.path.basename(self.data_dir))
self.assertFalse(os.path.exists(dest_dir))
dest_dir2 = os.path.join(self.archive_dir,os.path.basename(self.data_dir2))
self.assertFalse(os.path.exists(dest_dir2))
# Run the archiver
archiver = DataArchiver(self.archive_dir,log_dir=self.log_dir)
archiver.add_data_dir(self.data_dir)
archiver.add_data_dir(self.data_dir2)
status = archiver.archive_dirs()
# Check the copies
self.assertTrue(os.path.exists(dest_dir))
self.check_directory_contents(self.data_dir,dest_dir)
self.assertTrue(os.path.exists(dest_dir2))
self.check_directory_contents(self.data_dir,dest_dir2)
# Check the status of each operation
archiver.report()
self.assertEqual(archiver.result(self.data_dir).completed,0)
self.assertEqual(archiver.result(self.data_dir).copy_status,0)
self.assertEqual(archiver.result(self.data_dir).group_status,None)
self.assertEqual(archiver.result(self.data_dir).verify_status,0)
self.assertEqual(archiver.result(self.data_dir2).completed,0)
self.assertEqual(archiver.result(self.data_dir2).copy_status,0)
self.assertEqual(archiver.result(self.data_dir2).group_status,None)
self.assertEqual(archiver.result(self.data_dir2).verify_status,0)
# Check the overall status
self.assertEqual(status,0)
self.assertEqual(archiver.status,0)
class TestMd5CheckerWalk(unittest.TestCase):
"""Tests for the 'walk' method of the Md5Checker class
"""
def setUp(self):
"""Build directory with test data
"""
self.example_dir = ExampleDirLanguages()
self.dirn = self.example_dir.create_directory()
def tearDown(self):
"""Remove directory with test data
"""
self.example_dir.delete_directory()
def test_walk(self):
"""Md5Checker.walk yields all files
"""
# Walk the example directory and check all yielded files
# are in the list of created files
file_list = self.example_dir.filelist(include_links=True)
print str(file_list)
for f in Md5Checker.walk(self.dirn):
print "Check for %s" % f
self.assertTrue(f in file_list,"%s not in files or links?" % f)
file_list.remove(f)
# Check that no files were missed
self.assertTrue(len(file_list) == 0,
"Some files not yielded: %s" % file_list)
def test_walk_ignore_links(self):
"""Md5Checker.walk ignores links
"""
# Walk the example directory and check all yielded files
# are in the list of created files
file_list = self.example_dir.filelist(include_links=False)
for f in Md5Checker.walk(self.dirn,links=Md5Checker.IGNORE_LINKS):
self.assertTrue(f in file_list,"%s not in files?" % f)
file_list.remove(f)
# Check that no files were missed
self.assertTrue(len(file_list) == 0,
"Some files not yielded: %s" % file_list)
def test_walk_yields_broken_links(self):
"""Md5Checker.walk yields broken links
"""
# Add broken link
self.example_dir.add_link("broken.txt","missing.txt")
# Walk the example directory and check all yielded files
# are in the list of created files
file_list = self.example_dir.filelist(include_links=False)
for f in Md5Checker.walk(self.dirn,links=Md5Checker.IGNORE_LINKS):
self.assertTrue(f in file_list,"%s not in files?" % f)
file_list.remove(f)
# Check that no files were missed
self.assertTrue(len(file_list) == 0,
"Some files not yielded: %s" % file_list)
def setUp(self):
# Make a test data directory structure
self.example_dir = ExampleDirLanguages()
self.data_dir = self.example_dir.create_directory()
self.archive_dir = TestUtils.make_dir()
self.log_dir = TestUtils.make_dir()
# Placeholders for additional data structures
# These should be set in the tests where multiple
# directories are required
self.example_dir2 = None
self.data_dir2 = None
class TestMd5CheckerVerifyMd5sms(unittest.TestCase):
"""Tests for the 'verify_md5sums' method of the Md5Checker class
"""
def setUp(self):
self.example_dir = ExampleDirLanguages()
self.example_dir.create_directory()
def tearDown(self):
self.example_dir.delete_directory()
def test_verify_md5sums(self):
"""Md5Checker.verify_md5sums checks 'md5sum'-format file
"""
# Create MD5sum 'file'
md5sums = []
for f in self.example_dir.filelist(full_path=True):
md5sums.append("%s %s" % (md5sum(f),f))
md5sums = '\n'.join(md5sums)
fp = cStringIO.StringIO(md5sums)
# Run verification
files = self.example_dir.filelist(full_path=True)
self.assertNotEqual(len(files),0)
for f,status in Md5Checker.verify_md5sums(fp=fp):
self.assertTrue(f in files,"%s not in %s" % (f,files))
self.assertEqual(status,Md5Checker.MD5_OK)
files.remove(f)
# Check no files were missed
self.assertEqual(len(files),0)
def test_check_single_data_dir(self):
"""DataArchiver verifies single data directory (no copy)
"""
# Make additional data dir
dest_dir = os.path.join(self.archive_dir,os.path.basename(self.data_dir))
self.example_dir2 = ExampleDirLanguages()
self.data_dir2 = self.example_dir2.create_directory(dest_dir)
# Initial checks
self.assertTrue(os.path.exists(dest_dir))
# Run the archiver checks
archiver = DataArchiver(self.archive_dir,log_dir=self.log_dir)
archiver.add_data_dir(self.data_dir)
status = archiver.check_dirs()
# Check the status of each operation
self.assertEqual(archiver.result(self.data_dir).completed,0)
self.assertEqual(archiver.result(self.data_dir).copy_status,None)
self.assertEqual(archiver.result(self.data_dir).group_status,None)
self.assertEqual(archiver.result(self.data_dir).verify_status,0)
# Check the overall status
self.assertEqual(status,0)
self.assertEqual(archiver.status,0)
class TestDataArchiver(unittest.TestCase):
"""Tests for DataArchiver class
"""
def setUp(self):
# Make a test data directory structure
self.example_dir = ExampleDirLanguages()
self.data_dir = self.example_dir.create_directory()
self.archive_dir = TestUtils.make_dir()
self.log_dir = TestUtils.make_dir()
# Placeholders for additional data structures
# These should be set in the tests where multiple
# directories are required
self.example_dir2 = None
self.data_dir2 = None
def tearDown(self):
# Remove the test data directory (and copy)
self.example_dir.delete_directory()
if self.example_dir2 is not None:
self.example_dir2.delete_directory()
TestUtils.remove_dir(self.archive_dir)
TestUtils.remove_dir(self.log_dir)
def check_directory_contents(self,dir1,dir2):
# Check that contents of dir1 are also in dir2
# Checks that file and directory names match
for dirpath,dirnames,filenames in os.walk(dir1):
for d in dirnames:
d2 = os.path.join(dir2,os.path.relpath(os.path.join(dirpath,d),dir1))
self.assertTrue(os.path.isdir(d2))
for f in filenames:
f2 = os.path.join(dir2,os.path.relpath(os.path.join(dirpath,f),dir1))
self.assertTrue(os.path.isfile(f2))
def check_directory_group(self,dirn,group):
# Check that contents of dirn belong to specified group
gid = bcf_utils.get_gid_from_group(group)
##print "Checking: group %s GID %s" % (group,gid)
for dirpath,dirnames,filenames in os.walk(dirn):
for d in dirnames:
d2 = os.path.join(dirpath,d)
if os.path.islink(d2):
# Ignore links
continue
##print "%s/ %s" % (d2,bcf_utils.PathInfo(d2).gid)
self.assertEqual(bcf_utils.PathInfo(d2).gid,gid)
for f in filenames:
f2 = os.path.join(dirpath,f)
if os.path.islink(f2):
# Ignore links
continue
##print "%s %s" % (f2,bcf_utils.PathInfo(f2).gid)
self.assertEqual(bcf_utils.PathInfo(f2).gid,gid)
def test_archive_single_data_dir(self):
"""DataArchiver copies and verifies single data directory
"""
# Initial checks
dest_dir = os.path.join(self.archive_dir,os.path.basename(self.data_dir))
self.assertFalse(os.path.exists(dest_dir))
# Run the archiver
archiver = DataArchiver(self.archive_dir,log_dir=self.log_dir)
archiver.add_data_dir(self.data_dir)
status = archiver.archive_dirs()
# Check the copy
self.assertTrue(os.path.exists(dest_dir))
self.check_directory_contents(self.data_dir,dest_dir)
# Check the status of each operation
self.assertEqual(archiver.result(self.data_dir).completed,0)
self.assertEqual(archiver.result(self.data_dir).copy_status,0)
self.assertEqual(archiver.result(self.data_dir).group_status,None)
self.assertEqual(archiver.result(self.data_dir).verify_status,0)
# Check the overall status
self.assertEqual(status,0)
self.assertEqual(archiver.status,0)
def test_archive_multiple_data_dirs(self):
"""DataArchiver copies and verifies multiple data directories
"""
# Make additional data dir
self.example_dir2 = ExampleDirLanguages()
self.data_dir2 = self.example_dir2.create_directory()
# Initial checks
dest_dir = os.path.join(self.archive_dir,os.path.basename(self.data_dir))
self.assertFalse(os.path.exists(dest_dir))
dest_dir2 = os.path.join(self.archive_dir,os.path.basename(self.data_dir2))
self.assertFalse(os.path.exists(dest_dir2))
# Run the archiver
archiver = DataArchiver(self.archive_dir,log_dir=self.log_dir)
archiver.add_data_dir(self.data_dir)
archiver.add_data_dir(self.data_dir2)
status = archiver.archive_dirs()
# Check the copies
self.assertTrue(os.path.exists(dest_dir))
self.check_directory_contents(self.data_dir,dest_dir)
self.assertTrue(os.path.exists(dest_dir2))
self.check_directory_contents(self.data_dir,dest_dir2)
# Check the status of each operation
archiver.report()
self.assertEqual(archiver.result(self.data_dir).completed,0)
self.assertEqual(archiver.result(self.data_dir).copy_status,0)
self.assertEqual(archiver.result(self.data_dir).group_status,None)
self.assertEqual(archiver.result(self.data_dir).verify_status,0)
self.assertEqual(archiver.result(self.data_dir2).completed,0)
self.assertEqual(archiver.result(self.data_dir2).copy_status,0)
self.assertEqual(archiver.result(self.data_dir2).group_status,None)
self.assertEqual(archiver.result(self.data_dir2).verify_status,0)
# Check the overall status
self.assertEqual(status,0)
self.assertEqual(archiver.status,0)
def test_archive_data_dir_set_group(self):
"""DataArchiver sets group when copying data directory
"""
# Get a list of groups
current_user = pwd.getpwuid(os.getuid()).pw_name
groups = [g.gr_gid for g in grp.getgrall() if current_user in g.gr_mem]
if len(groups) < 2:
raise unittest.SkipTest("user '%s' must be in at least two groups" % current_user)
# Get a second group
gid = bcf_utils.PathInfo(self.archive_dir).gid
new_gid = None
for group in groups:
if group != gid:
new_gid = group
break
self.assertNotEqual(new_gid,gid)
new_group = bcf_utils.get_group_from_gid(new_gid)
##print "Group %s GID %s" % (new_group,new_gid)
# Initial checks
dest_dir = os.path.join(self.archive_dir,os.path.basename(self.data_dir))
self.assertFalse(os.path.exists(dest_dir))
# Run the archiver
archiver = DataArchiver(self.archive_dir,new_group=new_group,log_dir=self.log_dir)
archiver.add_data_dir(self.data_dir)
status = archiver.archive_dirs()
# Check the copy
self.assertTrue(os.path.exists(dest_dir))
self.check_directory_contents(self.data_dir,dest_dir)
self.check_directory_group(dest_dir,new_group)
# Check the status of each operation
self.assertEqual(archiver.result(self.data_dir).completed,0)
self.assertEqual(archiver.result(self.data_dir).copy_status,0)
self.assertEqual(archiver.result(self.data_dir).group_status,0)
self.assertEqual(archiver.result(self.data_dir).verify_status,0)
# Check the overall status
self.assertEqual(status,0)
self.assertEqual(archiver.status,0)
def test_archive_single_data_dir_with_unreadable_file(self):
"""DataArchiver reports failure when copying directory with unreadable file
"""
# Remove read permission from a file in source dir
os.chmod(self.example_dir.path('hello'),0244)
# Initial checks
dest_dir = os.path.join(self.archive_dir,os.path.basename(self.data_dir))
self.assertFalse(os.path.exists(dest_dir))
# Run the archiver
archiver = DataArchiver(self.archive_dir,log_dir=self.log_dir)
archiver.add_data_dir(self.data_dir)
status = archiver.archive_dirs()
# Check the copy
self.assertTrue(os.path.exists(dest_dir))
##self.check_directory_contents(self.data_dir,dest_dir)
# Check the status of each operation
self.assertEqual(archiver.result(self.data_dir).completed,0)
self.assertEqual(archiver.result(self.data_dir).copy_status,23)
self.assertEqual(archiver.result(self.data_dir).group_status,None)
self.assertEqual(archiver.result(self.data_dir).verify_status,1)
# Check the overall status
self.assertEqual(status,1)
self.assertEqual(archiver.status,1)
def test_check_single_data_dir(self):
"""DataArchiver verifies single data directory (no copy)
"""
# Make additional data dir
dest_dir = os.path.join(self.archive_dir,os.path.basename(self.data_dir))
self.example_dir2 = ExampleDirLanguages()
self.data_dir2 = self.example_dir2.create_directory(dest_dir)
# Initial checks
self.assertTrue(os.path.exists(dest_dir))
# Run the archiver checks
archiver = DataArchiver(self.archive_dir,log_dir=self.log_dir)
archiver.add_data_dir(self.data_dir)
status = archiver.check_dirs()
# Check the status of each operation
self.assertEqual(archiver.result(self.data_dir).completed,0)
self.assertEqual(archiver.result(self.data_dir).copy_status,None)
self.assertEqual(archiver.result(self.data_dir).group_status,None)
self.assertEqual(archiver.result(self.data_dir).verify_status,0)
# Check the overall status
self.assertEqual(status,0)
self.assertEqual(archiver.status,0)
def setUp(self):
self.example_dir = ExampleDirLanguages()
self.example_dir.create_directory()
class TestMd5CheckerMd5cmpDirs(unittest.TestCase):
"""Tests for the 'md5cmp_dirs' method of the Md5Checker class
"""
def setUp(self):
"""Build directory with test data
"""
self.dir1 = ExampleDirLanguages()
self.dir1.create_directory()
self.dir2 = ExampleDirLanguages()
self.dir2.create_directory()
def tearDown(self):
"""Remove directory with test data
"""
self.dir1.delete_directory()
self.dir2.delete_directory()
def test_cmp_identical_dirs(self):
"""Md5Checker.md5cmp_dirs with identical directories
"""
for f,status in Md5Checker.md5cmp_dirs(self.dir1.dirn,
self.dir2.dirn):
self.assertEqual(Md5Checker.MD5_OK,status,
"Failed for %s (status %d)" % (f,status))
def test_cmp_identical_dirs_ignore_links(self):
"""Md5Checker.md5cmp_dirs with identical directories ignoring links
"""
for f,status in Md5Checker.md5cmp_dirs(self.dir1.dirn,
self.dir2.dirn,
links=Md5Checker.IGNORE_LINKS):
self.assertFalse(os.path.islink(os.path.join(self.dir1.dirn,f)))
self.assertEqual(Md5Checker.MD5_OK,status)
def test_cmp_different_dirs_missing_file(self):
"""Md5Checker.md5cmp_dirs with different directories (missing file)
"""
# Add an additional file in reference
self.dir1.add_file("portuguese/ola","Hello!")
found_missing = False
for f,status in Md5Checker.md5cmp_dirs(self.dir1.dirn,
self.dir2.dirn):
if f == "portuguese/ola":
self.assertEqual(Md5Checker.MISSING_TARGET,status)
found_missing = True
else:
self.assertEqual(Md5Checker.MD5_OK,status)
self.assertTrue(found_missing,"Didn't pick up extra file in reference?")
def test_cmp_different_dirs_extra_file(self):
"""Md5Checker.md5cmp_dirs with different directories (extra file)
"""
# Add an additional file in target
self.dir2.add_file("portuguese/ola","Hello!")
for f,status in Md5Checker.md5cmp_dirs(self.dir1.dirn,
self.dir2.dirn):
if os.path.basename(f) == "portuguese/ola":
self.assertFaile("Target file 'portuguese/ola' should not be found")
else:
self.assertEqual(Md5Checker.MD5_OK,status)
def test_cmp_different_dirs_different_file(self):
"""Md5Checker.md5cmp_dirs with different directories (different file)
"""
# Replace file in target with different content
self.dir2.add_file("goodbye","Goooooodbyeeee!")
for f,status in Md5Checker.md5cmp_dirs(self.dir1.dirn,
self.dir2.dirn,
links=Md5Checker.IGNORE_LINKS):
if os.path.basename(f) == "goodbye":
self.assertEqual(Md5Checker.MD5_FAILED,status)
else:
self.assertEqual(Md5Checker.MD5_OK,status)
def test_cmp_different_dirs_file_is_dir(self):
"""Md5Checker.md5cmp_dirs with different directories ('file' is dir)
"""
# Make a file in reference which is same as dir in target
self.dir1.add_file("portuguese","This is gonna be trouble")
self.dir2.add_dir("portuguese")
for f,status in Md5Checker.md5cmp_dirs(self.dir1.dirn,
self.dir2.dirn):
print "%s: %d" % (f,status)
if os.path.basename(f) == "portuguese":
self.assertEqual(Md5Checker.MD5_ERROR,status)
else:
self.assertEqual(Md5Checker.MD5_OK,status)
def setUp(self):
"""Build directory with test data
"""
self.example_dir = ExampleDirLanguages()
self.dirn = self.example_dir.create_directory()
