def setUp(self):
self.remote = datatypes.DirectoryInfo('/store')
self.remote.add_file_list(self.new_files)
self.invent = datatypes.DirectoryInfo('/store')
self.invent.add_file_list(self.inventory_files)
self.remote.setup_hash()
self.invent.setup_hash()
def test_missing_dir(self):
remote = datatypes.DirectoryInfo('/store')
remote.add_file_list([('/store/yo/0000/file_exists.root', 100)])
invent = datatypes.DirectoryInfo('/store')
invent.add_file_list([('/store/yo/0000/file_exists.root', 100),
('/store/yo/0000/file_not_exists.root', 100),
('/store/hmm/other/directory.root', 300)])
remote.setup_hash()
invent.setup_hash()
self.assertEqual(datatypes.compare(invent, remote)[1], 400)
def test_orphan_file(self):
remote = datatypes.DirectoryInfo('/store')
remote.add_file_list([('/store/yo/0000/file_exists.root', 100),
('/store/yo/0000/file_not_exists.root', 200,
time.time())])
invent = datatypes.DirectoryInfo('/store')
invent.add_file_list([('/store/yo/0000/file_exists.root', 100)])
remote.setup_hash()
invent.setup_hash()
self.assertEqual(datatypes.compare(invent, remote), ([], 0, [], 0))
def test_olddir(self):
empty_dir = 'mc/new/empty/0002'
path = os.path.join(TMP_DIR, empty_dir)
os.makedirs(path)
os.utime(path, (1000000000, 1000000000))
listing = datatypes.DirectoryInfo('/store',
to_merge=[
datatypes.create_dirinfo(
'', subdir, my_ls)
for subdir in ['mc', 'data']
])
LOG.info('=' * 40)
LOG.info('Doing the hash')
LOG.info('=' * 40)
listing.setup_hash()
LOG.info('=' * 40)
LOG.info('Doing the comparison')
LOG.info('=' * 40)
LOG.info(listing.displays())
LOG.info('=' * 40)
LOG.info(self.listing.displays())
file_list, dir_list, _ = listing.compare(self.listing)
self.assertEqual(len(dir_list), 1)
self.assertTrue(
os.path.join('/store', empty_dir).startswith(dir_list[0]))
for file_name, _ in self.file_list:
self.assertFalse(file_name.startswith(dir_list[0]))
def do_more_setup(self):
for name, size in self.file_list + self.orphan:
path = os.path.join(TMP_DIR, name[7:])
if not os.path.isdir(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
out = open(path, 'w')
out.write('\0' * size)
out.close()
os.utime(path, (1000000000, 1000000000))
self.tree.add_file_list(self.missing)
for name, size in self.new_file:
path = os.path.join(TMP_DIR, name[7:])
if not os.path.isdir(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
out = open(path, 'w')
out.write('\0' * size)
out.close()
self.listing = datatypes.DirectoryInfo(
'/store', to_merge=[datatypes.create_dirinfo('', subdir, my_ls) \
for subdir in ['mc', 'data']])
self.tree.setup_hash()
self.listing.setup_hash()
def setUp(self):
if os.path.exists(TMP_DIR):
print 'Desired directory location already exists!'
exit(1)
os.makedirs(TMP_DIR)
self.tree = datatypes.DirectoryInfo('/store')
self.tree.add_file_list(self.file_list)
self.do_more_setup()
def test_ls_vs_list(self):
dirinfos = [datatypes.create_dirinfo('', subdir, my_ls) \
for subdir in ['mc', 'data']]
master_dirinfo = datatypes.DirectoryInfo('/store', to_merge=dirinfos)
self.check_equal(self.tree, master_dirinfo)
self.assertEqual(self.tree.count_nodes(), master_dirinfo.count_nodes())
def test_merge_trees(self):
trees = {
'mc': datatypes.DirectoryInfo('mc'),
'data': datatypes.DirectoryInfo('data')
}
for key, tree in trees.iteritems():
tree.add_file_list([('/'.join(name.split('/')[2:]), size) \
for name, size in self.file_list if name.split('/')[2] == key])
one_tree = datatypes.DirectoryInfo('/store',
[trees['data'], trees['mc']])
self.check_equal(self.tree, one_tree)
# Hopefully order doesn't matter
two_tree = datatypes.DirectoryInfo('/store',
[trees['mc'], trees['data']])
self.check_equal(self.tree, two_tree)
self.check_equal(one_tree, two_tree)
def test_cant_compare_subdir(self):
invent = datatypes.DirectoryInfo('/store')
invent.add_file_list([('/store/yo/first/0000/file_exists.root', 100)])
remote = datatypes.DirectoryInfo('/store')
remote.add_file_list([
('/store/yo/first/0000/file_exists.root', 100, 0),
('/store/yo/second/0000/file_new.root', 100, time.time())
])
remote.get_node('yo/second/0001').mtime = 1
invent.setup_hash()
remote.setup_hash()
self.assertTrue(invent.get_node('yo').can_compare)
self.assertFalse(remote.get_node('yo/second/0000').can_compare)
self.assertTrue(remote.get_node('yo/second/0001').can_compare)
self.assertTrue(remote.get_node('yo/second').can_compare)
self.assertFalse(remote.compare(invent)[0])
def test_newdir(self):
empty_dir = 'mc/new/empty/0002'
os.makedirs(os.path.join(TMP_DIR, empty_dir))
dirinfos = [datatypes.create_dirinfo('', subdir, my_ls) \
for subdir in ['mc', 'data']]
master_dirinfo = datatypes.DirectoryInfo('/store', to_merge=dirinfos)
self.check_equal(self.tree, master_dirinfo)
self.assertFalse(
master_dirinfo.get_node(os.path.join(empty_dir, 'not_there'),
False))
self.assertTrue(master_dirinfo.get_node(empty_dir, False))
self.assertFalse(master_dirinfo.get_node(empty_dir, False).can_compare)
def test_xrd_on_t3(self):
remote_tree = getsitecontents.get_site_tree('T3_US_MIT')
local_listing = datatypes.DirectoryInfo(
'/store', to_merge=[
datatypes.create_dirinfo('', subdir, my_ls) for \
subdir in config.config_dict().get('DirectoryList', [])
])
local_listing.setup_hash()
print '=' * 30
local_listing.display()
print '=' * 30
remote_tree.display()
self.assertEqual(local_listing.hash, remote_tree.hash)
def do_more_setup(self):
self.unlisted_tree = datatypes.DirectoryInfo('/store')
self.unlisted_tree.add_file_list(self.unlisted_list)