def test_parse_fails_invalid_value(self):
self._create_raw_classify('', [
'volume=small', 'protection=super_dooper_spicy_secret',
'recurse=true', 'compress=true'
])
with self.assertRaises(Exception):
classifydir.ClassifiedDir(self.test_dir.name, fetch_info=True)
def test_single_tree_without_fetching(self):
self._create_directories(('a', 'd'))
self._create_classify('',
'large',
'secret',
recurse='true',
name='root')
self._create_files('a', (100, 200, 300))
self._create_files('d', (20000, ))
cd = classifydir.ClassifiedDir(self.test_dir.name, fetch_info=False)
# If we don't fetch into many properties should be None, but the filenames still work.
self.assertEqual([d.name for d in cd.descendants()],
['root', 'a', 'd'])
self.assertEqual([d.name for d in cd.descendant_members()],
['root', 'a', 'd'])
self.assertEqual([d.name for d in cd.descendant_roots()], ['root'])
self.assertIsNone(cd.total_size())
self.assertIsNone(cd.total_file_count())
self.assertEqual(cd.volume, 'large')
self.assertEqual(cd.protection, 'secret')
self.assertIsNone(cd.archive_size())
self.assertIsNone(cd.archive_file_count())
self.assertIsNone(cd.archive_hash())
self.assertEqual(list(cd.archive_filenames()), [
os.path.join(self.test_dir.name, '.classify'),
os.path.join(self.test_dir.name, 'a', '1'),
os.path.join(self.test_dir.name, 'a', '2'),
os.path.join(self.test_dir.name, 'a', '3'),
os.path.join(self.test_dir.name, 'd', '1')
])
def test_parse_fails_unknown_parameter(self):
self._create_raw_classify('', [
'volume=small', 'protection=restricted', 'mystery=what_am_i',
'recurse=true', 'compress=true'
])
with self.assertRaises(Exception):
classifydir.ClassifiedDir(self.test_dir.name, fetch_info=True)
def test_parse_with_comments(self):
self._create_raw_classify('', [
'# Test file with some comments', ' # Some not at the start',
'volume=small', 'protection=none # Comments allowed after text',
'recurse=true', 'compress=true'
])
classifydir.ClassifiedDir(self.test_dir.name, fetch_info=True)
def freshen_archive_sets(search_paths, output_path, sizes, key_file):
"""Updates all tar files for archives of the specified sizes located on any of the supplied
search paths. tars are created or freshened in size based directories in the output_path, and
any unrecognized files in output_path will be deleted on completion. Where encryption is
required, the contents of key_file will be used as the key."""
if not path.exists(output_path):
_write_error('Archive output directory does not exist: ' + output_path)
return
if not path.exists(key_file):
_write_error('Key file does not exist: ' + key_file)
return
cds = [
classifydir.ClassifiedDir(s, True) for s in search_paths
if path.exists(s)
]
for size in sizes:
size_output_path = path.join(output_path, size)
if not path.exists(size_output_path):
_write_status('Creating archive size output directory: ' +
size_output_path)
os.makedirs(size_output_path)
freshen_archives([
a for cd in cds for a in cd.descendantRoots() if a.volume == size
], size_output_path, key_file)
def test_single_recursive_tree(self):
self._create_directories(('a', 'ab', 'abc', 'd'))
self._create_classify('',
'huge',
'secret',
recurse='true',
name='root')
self._create_files('ab', (100, 200, 300))
self._create_files('abc', (1000, ))
self._create_files('d', (20000, ))
cd = classifydir.ClassifiedDir(self.test_dir.name, fetch_info=True)
# Test directory is the only root.
self.assertTrue(cd.is_archive_root())
self.assertEqual([d.name for d in cd.descendants()],
['root', 'a', 'b', 'c', 'd'])
self.assertEqual([d.name for d in cd.descendant_members()],
['root', 'a', 'b', 'c', 'd'])
self.assertEqual([d.name for d in cd.descendant_roots()], ['root'])
self.assertEqual([d.name for d in cd.descendant_attenuations()], [])
# With lots of properties.
self.assertEqual(cd.total_size(), 21668) # includes the classify file.
self.assertEqual(cd.total_file_count(),
6) # includes the classify file.
self.assertEqual(cd.volume, 'huge')
self.assertEqual(cd.protection, 'secret')
self.assertEqual(cd.archive_size(), 21668)
self.assertEqual(cd.archive_file_count(), 6)
self.assertEqual(list(cd.archive_filenames()), [
os.path.join(self.test_dir.name, '.classify'),
os.path.join(self.test_dir.name, 'a', 'b', '1'),
os.path.join(self.test_dir.name, 'a', 'b', '2'),
os.path.join(self.test_dir.name, 'a', 'b', '3'),
os.path.join(self.test_dir.name, 'a', 'b', 'c', '1'),
os.path.join(self.test_dir.name, 'd', '1')
])
# Adding a file should change the hash and date
with open(os.path.join(self._rel_path('d'), 'new'), 'w') as f:
f.write("y" * 80)
updated_cd = classifydir.ClassifiedDir(self.test_dir.name,
fetch_info=True)
self.assertEqual(updated_cd.archive_file_count(), 7)
self.assertNotEqual(cd.archive_hash(), updated_cd.archive_hash())
def test_parse_fails_malformed_line(self):
self._create_raw_classify(
'', ['volume='
'protection=none'
'recurse=true'
'compress=true'])
with self.assertRaises(Exception):
classifydir.ClassifiedDir(self.test_dir.name, fetch_info=False)
def test_missing_classify_in_non_recursive(self):
self._create_directories(('a', 'b'))
self._create_classify('',
'small',
'restricted',
recurse='false',
name='root')
self._create_classify('a', 'small', 'confidential', recurse='true')
# Note: no classify in b, this is an error!
with self.assertRaises(Exception):
classifydir.ClassifiedDir(self.test_dir.name, fetch_info=True)
def test_archives_inside_non_recursive(self):
self._create_directories(('a', 'ab', 'c'))
self._create_classify('',
'small',
'restricted',
recurse='false',
name='root')
self._create_classify('a', 'small', 'confidential', recurse='true')
self._create_classify('c', 'none', 'none', recurse='true')
self._create_files('', (100, 200))
self._create_files('ab', (1000, ))
self._create_files('c', (20000, ))
cd = classifydir.ClassifiedDir(self.test_dir.name, fetch_info=True)
# Test directory should have one child root (volume=none gets ignored) but since its not
# recursive these should not be considered members.
self.assertTrue(cd.is_archive_root())
self.assertEqual([d.name for d in cd.descendants()],
['root', 'a', 'b', 'c'])
self.assertEqual([d.name for d in cd.descendant_members()], ['root'])
self.assertEqual([d.name for d in cd.descendant_roots()],
['root', 'a'])
# Root totals include all files but the archive shouldn't include files in subdirectories.
self.assertEqual(cd.total_size(), 21495)
self.assertEqual(cd.total_file_count(), 7)
self.assertEqual(cd.archive_size(), 374)
self.assertEqual(cd.archive_file_count(), 3)
self.assertEqual(list(cd.archive_filenames()), [
os.path.join(self.test_dir.name, '.classify'),
os.path.join(self.test_dir.name, '1'),
os.path.join(self.test_dir.name, '2')
])
# The lower level archive should include its own files.
child = list(cd.descendant_roots())[1]
self.assertTrue(child.is_archive_root())
self.assertEqual(child.total_size(), 1065)
self.assertEqual(child.total_file_count(), 2)
self.assertEqual(child.archive_size(), 1065)
self.assertEqual(child.archive_file_count(), 2)
self.assertEqual(list(child.archive_filenames()), [
os.path.join(self.test_dir.name, 'a', '.classify'),
os.path.join(self.test_dir.name, 'a', 'b', '1')
])
def test_sibling_archives(self):
self._create_directories(('a', 'b', 'c', 'cd'))
self._create_classify('a', 'small', 'restricted', recurse='true')
self._create_classify('b', 'medium', 'confidential', recurse='true')
self._create_classify('c', 'large', 'none', recurse='true')
self._create_files('a', (100, 200))
self._create_files('b', (2000, ))
self._create_files('c', (30000, ))
cd = classifydir.ClassifiedDir(self.test_dir.name, fetch_info=True)
# Test directory is not itself a root.
self.assertFalse(cd.is_archive_root())
self.assertEqual([d.name for d in cd.descendants()],
[self.test_subdir, 'a', 'b', 'c', 'd'])
self.assertEqual(cd.total_size(),
32486) # includes the classify files.
self.assertEqual(cd.total_file_count(),
7) # includes the classify files.
# But should contain three roots.
roots = list(cd.descendant_roots())
self.assertEqual([d.name for d in roots], ['a', 'b', 'c'])
for root in roots:
self.assertTrue(root.is_archive_root())
# With lots of properties.
self.assertEqual(roots[0].archive_size(), 363)
self.assertEqual(roots[0].total_file_count(), 3)
self.assertEqual(roots[0].volume, 'small')
self.assertEqual(roots[0].protection, 'restricted')
self.assertEqual([d.name for d in roots[0].descendants()], ['a'])
self.assertEqual(roots[1].archive_size(), 2066)
self.assertEqual(roots[1].total_file_count(), 2)
self.assertEqual(roots[1].volume, 'medium')
self.assertEqual(roots[1].protection, 'confidential')
self.assertEqual([d.name for d in roots[1].descendants()], ['b'])
self.assertEqual(roots[2].archive_size(), 30057)
self.assertEqual(roots[2].total_file_count(), 2)
self.assertEqual(roots[2].volume, 'large')
self.assertEqual(roots[2].protection, 'none')
self.assertEqual([d.name for d in roots[2].descendants()], ['c', 'd'])
def test_parse_fails_missing_parameter(self):
self._create_raw_classify(
'', ['volume=small', 'recurse=true', 'compress=true'])
with self.assertRaises(Exception):
classifydir.ClassifiedDir(self.test_dir.name, fetch_info=True)
def test_override_archives_inside_recursive_archive(self):
self._create_directories(('a', 'ab', 'c', 'cd', 'e'))
self._create_classify('',
'small',
'restricted',
recurse='true',
name='root')
# Note this overrides a directory inside the recursive parent.
self._create_classify('a', 'small', 'confidential', recurse='true')
# Note this ceases archiving inside the recursive parent.
self._create_classify('c', 'none', 'none', recurse='true')
# Note this tries to restart archiving in a subdirectory with a different name.
self._create_classify('cd', 'medium', 'restricted', recurse='true')
self._create_files('', (100, 200))
self._create_files('a', (10000, ))
self._create_files('ab', (20000, ))
self._create_files('c', (1000, ))
self._create_files('cd', (2000, ))
self._create_files('e', (5000, ))
cd = classifydir.ClassifiedDir(self.test_dir.name, fetch_info=True)
self.assertTrue(cd.is_archive_root())
self.assertEqual([d.name for d in cd.descendants()],
['root', 'a', 'b', 'c', 'd', 'e'])
# Members should not include the overridden directory or the attenuated directory.
self.assertEqual([d.name for d in cd.descendant_members()],
['root', 'e'])
self.assertEqual([d.name for d in cd.descendant_roots()],
['root', 'a', 'd'])
self.assertEqual([d.name for d in cd.descendant_attenuations()], ['c'])
# Root totals include all files but the archive shouldn't include files in the overridden
# directory.
self.assertEqual(cd.total_size(), 38558)
self.assertEqual(cd.total_file_count(), 11)
self.assertEqual(cd.archive_size(), 5373)
self.assertEqual(cd.archive_file_count(), 4)
self.assertEqual(list(cd.archive_filenames()), [
os.path.join(self.test_dir.name, '.classify'),
os.path.join(self.test_dir.name, '1'),
os.path.join(self.test_dir.name, '2'),
os.path.join(self.test_dir.name, 'e', '1')
])
# The lower level archives should include thier own files.
child = list(cd.descendant_roots())[1]
self.assertTrue(child.is_archive_root())
self.assertEqual(child.total_size(), 30065)
self.assertEqual(child.total_file_count(), 3)
self.assertEqual(child.archive_size(), 30065)
self.assertEqual(child.archive_file_count(), 3)
self.assertEqual(list(child.archive_filenames()), [
os.path.join(self.test_dir.name, 'a', '.classify'),
os.path.join(self.test_dir.name, 'a', '1'),
os.path.join(self.test_dir.name, 'a', 'b', '1')
])
# The lower level archives should include thier own files.
child = list(cd.descendant_roots())[2]
self.assertTrue(child.is_archive_root())
self.assertEqual(child.archive_size(), 2064)
self.assertEqual(child.archive_file_count(), 2)
self.assertEqual(list(child.archive_filenames()), [
os.path.join(self.test_dir.name, 'c', 'd', '.classify'),
os.path.join(self.test_dir.name, 'c', 'd', '1')
])
