def test_xlattice_bug3(self):
"""
this test relies on dat2.xlattice.org being locally present
and an internally consistent merkleization
"""
with open('tests/test_data/dat2.xlattice.org', 'rb') as file:
serialization = str(file.read(), 'utf-8')
# create from serialization ---------------------------------
tree1 = MerkleTree.create_from_serialization(serialization,
HashTypes.SHA2)
ser1 = tree1.to_string(0)
self.assertEqual(serialization, ser1)
# create from string array ----------------------------------
string = serialization.split('\n')
string = string[:-1]
self.assertEqual(2511, len(string))
tree2 = MerkleTree.create_from_string_array(string, HashTypes.SHA2)
ser2 = tree2.to_string(0)
self.assertEqual(serialization, ser2)
self.assertTrue(tree1 == tree2)
def test_xlattice_bug1(self):
"""
this test relies on dat.xlattice.org being locally present
and an internally consistent merkleization
"""
with open('tests/test_data/dat.xlattice.org', 'rb') as file:
serialization = str(file.read(), 'utf-8')
# create from serialization ---------------------------------
tree1 = MerkleTree.create_from_serialization(serialization,
HashTypes.SHA1)
# # DEBUG
# print "tree1 has %d nodes" % len(tree1.nodes)
# with open('junk.tree1', 'w') as t:
# t.write( tree1.toString(0) )
# # END
ser1 = tree1.to_string(0)
self.assertEqual(serialization, ser1)
# create from string array ----------------------------------
string = serialization.split('\n')
string = string[:-1]
self.assertEqual(2511, len(string))
tree2 = MerkleTree.create_from_string_array(string, HashTypes.SHA1)
ser2 = tree2.to_string(0)
self.assertEqual(serialization, ser2)
self.assertTrue(tree1 == tree2)
def do_test_bound_needle_dirs(self, hashtype):
"""test directories four deep with one data file at the lowest level"""
(dir_name1, dir_path1, dir_name2, dir_path2) =\
self.make_two_test_directories(FOUR, ONE)
tree1 = MerkleTree.create_from_file_system(dir_path1, hashtype)
self.assertEqual(dir_name1, tree1.name)
nodes1 = tree1.nodes
self.assertTrue(nodes1 is not None)
self.assertEqual(ONE, len(nodes1))
self.verify_tree_sha(tree1, dir_path1, hashtype)
tree2 = MerkleTree.create_from_file_system(dir_path2, hashtype)
self.assertEqual(dir_name2, tree2.name)
nodes2 = tree2.nodes
self.assertTrue(nodes2 is not None)
self.assertEqual(ONE, len(nodes2))
self.verify_tree_sha(tree2, dir_path2, hashtype)
self.assertTrue(tree1 == tree1)
self.assertFalse(tree1 == tree2)
tree1_str = tree1.to_string(0)
tree1_rebuilt = MerkleTree.create_from_serialization(
tree1_str, hashtype)
# # DEBUG
# print "NEEDLEDIR TREE1:\n" + tree1Str
# print "REBUILT TREE1:\n" + tree1Rebuilt.toString("")
# # END
self.assertTrue(tree1 == tree1_rebuilt)
def do_test_bound_needle_dirs(self, hashtype):
"""test directories four deep with one data file at the lowest level"""
(dir_name1, dir_path1, dir_name2, dir_path2) =\
self.make_two_test_directories(FOUR, ONE)
tree1 = MerkleTree.create_from_file_system(dir_path1, hashtype)
self.assertEqual(dir_name1, tree1.name)
nodes1 = tree1.nodes
self.assertTrue(nodes1 is not None)
self.assertEqual(ONE, len(nodes1))
self.verify_tree_sha(tree1, dir_path1, hashtype)
tree2 = MerkleTree.create_from_file_system(dir_path2, hashtype)
self.assertEqual(dir_name2, tree2.name)
nodes2 = tree2.nodes
self.assertTrue(nodes2 is not None)
self.assertEqual(ONE, len(nodes2))
self.verify_tree_sha(tree2, dir_path2, hashtype)
self.assertTrue(tree1 == tree1)
self.assertFalse(tree1 == tree2)
tree1_str = tree1.to_string(0)
tree1_rebuilt = MerkleTree.create_from_serialization(
tree1_str, hashtype)
# # DEBUG
# print "NEEDLEDIR TREE1:\n" + tree1Str
# print "REBUILT TREE1:\n" + tree1Rebuilt.toString("")
# # END
self.assertTrue(tree1 == tree1_rebuilt)
def do_test_bound_flat_dirs(self, hashtype):
"""test directory is single level, with four data files"""
check_hashtype(hashtype)
(dir_name1, dir_path1, dir_name2, dir_path2) =\
self.make_two_test_directories(ONE, FOUR)
tree1 = MerkleTree.create_from_file_system(dir_path1, hashtype)
self.assertEqual(dir_name1, tree1.name)
nodes1 = tree1.nodes
self.assertTrue(nodes1 is not None)
self.assertEqual(FOUR, len(nodes1))
self.verify_tree_sha(tree1, dir_path1, hashtype)
tree2 = MerkleTree.create_from_file_system(dir_path2, hashtype)
self.assertEqual(dir_name2, tree2.name)
nodes2 = tree2.nodes
self.assertTrue(nodes2 is not None)
self.assertEqual(FOUR, len(nodes2))
self.verify_tree_sha(tree2, dir_path2, hashtype)
self.assertFalse(tree1 is None)
self.assertTrue(tree1 == tree1)
self.assertFalse(tree1 == tree2)
tree1_str = tree1.to_string(0)
tree1_rebuilt = MerkleTree.create_from_serialization(
tree1_str, hashtype)
self.assertTrue(tree1 == tree1_rebuilt)
def test_xlattice_bug3(self):
"""
this test relies on dat2.xlattice.org being locally present
and an internally consistent merkleization
"""
with open('tests/test_data/dat2.xlattice.org', 'rb') as file:
serialization = str(file.read(), 'utf-8')
# create from serialization ---------------------------------
tree1 = MerkleTree.create_from_serialization(
serialization, HashTypes.SHA2)
ser1 = tree1.to_string(0)
self.assertEqual(serialization, ser1)
# create from string array ----------------------------------
string = serialization.split('\n')
string = string[:-1]
self.assertEqual(2511, len(string))
tree2 = MerkleTree.create_from_string_array(string, HashTypes.SHA2)
ser2 = tree2.to_string(0)
self.assertEqual(serialization, ser2)
self.assertTrue(tree1 == tree2)
def test_xlattice_bug1(self):
"""
this test relies on dat.xlattice.org being locally present
and an internally consistent merkleization
"""
with open('tests/test_data/dat.xlattice.org', 'rb') as file:
serialization = str(file.read(), 'utf-8')
# create from serialization ---------------------------------
tree1 = MerkleTree.create_from_serialization(
serialization, HashTypes.SHA1)
# # DEBUG
# print "tree1 has %d nodes" % len(tree1.nodes)
# with open('junk.tree1', 'w') as t:
# t.write( tree1.toString(0) )
# # END
ser1 = tree1.to_string(0)
self.assertEqual(serialization, ser1)
# create from string array ----------------------------------
string = serialization.split('\n')
string = string[:-1]
self.assertEqual(2511, len(string))
tree2 = MerkleTree.create_from_string_array(string, HashTypes.SHA1)
ser2 = tree2.to_string(0)
self.assertEqual(serialization, ser2)
self.assertTrue(tree1 == tree2)
def do_test_bound_flat_dirs(self, hashtype):
"""test directory is single level, with four data files"""
check_hashtype(hashtype)
(dir_name1, dir_path1, dir_name2, dir_path2) =\
self.make_two_test_directories(ONE, FOUR)
tree1 = MerkleTree.create_from_file_system(dir_path1, hashtype)
self.assertEqual(dir_name1, tree1.name)
nodes1 = tree1.nodes
self.assertTrue(nodes1 is not None)
self.assertEqual(FOUR, len(nodes1))
self.verify_tree_sha(tree1, dir_path1, hashtype)
tree2 = MerkleTree.create_from_file_system(dir_path2, hashtype)
self.assertEqual(dir_name2, tree2.name)
nodes2 = tree2.nodes
self.assertTrue(nodes2 is not None)
self.assertEqual(FOUR, len(nodes2))
self.verify_tree_sha(tree2, dir_path2, hashtype)
self.assertFalse(tree1 is None)
self.assertTrue(tree1 == tree1)
self.assertFalse(tree1 == tree2)
tree1_str = tree1.to_string(0)
tree1_rebuilt = MerkleTree.create_from_serialization(
tree1_str, hashtype)
self.assertTrue(tree1 == tree1_rebuilt)
def do_test_pathless_unbound(self, hashtype):
"""
Test basic characteristics of very simple MerkleTrees created
using a specific SHA hash type.
"""
(dir_name1, dir_name2) = self.get_two_unique_directory_names()
check_hashtype(hashtype)
tree1 = MerkleTree(dir_name1, hashtype)
self.assertEqual(dir_name1, tree1.name)
if hashtype == HashTypes.SHA1:
self.assertEqual(SHA1_HEX_NONE, tree1.hex_hash)
elif hashtype == HashTypes.SHA2:
self.assertEqual(SHA2_HEX_NONE, tree1.hex_hash)
elif hashtype == HashTypes.SHA3:
self.assertEqual(SHA3_HEX_NONE, tree1.hex_hash)
elif hashtype == HashTypes.BLAKE2B_256:
self.assertEqual(BLAKE2B_256_HEX_NONE, tree1.hex_hash)
else:
raise NotImplementedError
tree2 = MerkleTree(dir_name2, hashtype)
self.assertEqual(dir_name2, tree2.name)
# these tests remain skimpy
self.assertFalse(tree1 is None)
self.assertTrue(tree1 == tree1)
self.assertFalse(tree1 == tree2)
tree1_str = tree1.to_string(0)
# there should be no indent on the first line
self.assertFalse(tree1_str[0] == ' ')
# no extra lines should be added
lines = tree1_str.split('\n')
# this split generates an extra blank line, because the serialization
# ends with CR-LF
if lines[-1] == '':
lines = lines[:-1]
self.assertEqual(1, len(lines))
tree1_rebuilt = MerkleTree.create_from_serialization(
tree1_str, hashtype)
self.assertTrue(tree1 == tree1_rebuilt)
def do_test_pathless_unbound(self, hashtype):
"""
Test basic characteristics of very simple MerkleTrees created
using a specific SHA hash type.
"""
(dir_name1, dir_name2) = self.get_two_unique_directory_names()
check_hashtype(hashtype)
tree1 = MerkleTree(dir_name1, hashtype)
self.assertEqual(dir_name1, tree1.name)
if hashtype == HashTypes.SHA1:
self.assertEqual(SHA1_HEX_NONE, tree1.hex_hash)
elif hashtype == HashTypes.SHA2:
self.assertEqual(SHA2_HEX_NONE, tree1.hex_hash)
elif hashtype == HashTypes.SHA3:
self.assertEqual(SHA3_HEX_NONE, tree1.hex_hash)
elif hashtype == HashTypes.BLAKE2B_256:
self.assertEqual(BLAKE2B_256_HEX_NONE, tree1.hex_hash)
else:
raise NotImplementedError
tree2 = MerkleTree(dir_name2, hashtype)
self.assertEqual(dir_name2, tree2.name)
# these tests remain skimpy
self.assertFalse(tree1 is None)
self.assertTrue(tree1 == tree1)
self.assertFalse(tree1 == tree2)
tree1_str = tree1.to_string(0)
# there should be no indent on the first line
self.assertFalse(tree1_str[0] == ' ')
# no extra lines should be added
lines = tree1_str.split('\n')
# this split generates an extra blank line, because the serialization
# ends with CR-LF
if lines[-1] == '':
lines = lines[:-1]
self.assertEqual(1, len(lines))
tree1_rebuilt = MerkleTree.create_from_serialization(
tree1_str, hashtype)
self.assertTrue(tree1 == tree1_rebuilt)
def do_test_deepish_trees(self, hashtype):
"""
Build a directory of random data, then its MerkleTree, then
round trip to a serialization and back.
"""
tree_top = os.path.join('tmp', self.rng.next_file_name(MAX_NAME_LEN))
while os.path.exists(tree_top):
tree_top = os.path.join('tmp',
self.rng.next_file_name(MAX_NAME_LEN))
# Generate a quasi-random data directory, 7 deep, up to 5 files/dir
self.rng.next_data_dir(tree_top, depth=7, width=5, max_len=4096)
# Build a MerkleTree specifying the directory.
tree = MerkleTree.create_from_file_system(tree_top, hashtype)
# ROUND TRIP 1 ----------------------------------------------
# Serialize it.
ser = tree.__str__()
# Deserialize to make another MerkleTree.
tree2 = MerkleTree.create_from_serialization(ser, hashtype)
self.assertTrue(tree2.__eq__(tree))
self.assertEqual(tree2, tree) # identical test
# ROUND TRIP 2 ----------------------------------------------
strings = ser.split('\n')
strings = strings[:-1]
tree3 = MerkleTree.create_from_string_array(strings, hashtype)
self.assertEqual(tree3, tree)
# ROUND TRIP 3 ----------------------------------------------
filename = os.path.join('tmp', self.rng.next_file_name(8))
while os.path.exists(filename):
filename = os.path.join('tmp', self.rng.next_file_name(8))
with open(filename, 'w') as file:
file.write(ser)
tree4 = MerkleTree.create_from_file(filename, hashtype)
self.assertEqual(tree4, tree)
def do_test_deepish_trees(self, hashtype):
"""
Build a directory of random data, then its MerkleTree, then
round trip to a serialization and back.
"""
tree_top = os.path.join('tmp', self.rng.next_file_name(MAX_NAME_LEN))
while os.path.exists(tree_top):
tree_top = os.path.join(
'tmp', self.rng.next_file_name(MAX_NAME_LEN))
# Generate a quasi-random data directory, 7 deep, up to 5 files/dir
self.rng.next_data_dir(tree_top, depth=7, width=5, max_len=4096)
# Build a MerkleTree specifying the directory.
tree = MerkleTree.create_from_file_system(tree_top, hashtype)
# ROUND TRIP 1 ----------------------------------------------
# Serialize it.
ser = tree.__str__()
# Deserialize to make another MerkleTree.
tree2 = MerkleTree.create_from_serialization(ser, hashtype)
self.assertTrue(tree2.__eq__(tree))
self.assertEqual(tree2, tree) # identical test
# ROUND TRIP 2 ----------------------------------------------
strings = ser.split('\n')
strings = strings[:-1]
tree3 = MerkleTree.create_from_string_array(strings, hashtype)
self.assertEqual(tree3, tree)
# ROUND TRIP 3 ----------------------------------------------
filename = os.path.join('tmp', self.rng.next_file_name(8))
while os.path.exists(filename):
filename = os.path.join('tmp', self.rng.next_file_name(8))
with open(filename, 'w') as file:
file.write(ser)
tree4 = MerkleTree.create_from_file(filename, hashtype)
self.assertEqual(tree4, tree)
def test_gray_boxes_bug3(self):
""" Test solution to bug in handling grayboxes website. """
serialization =\
'088d0e391e1a4872329e0f7ac5d45b2025363e26c199a7' + \
'4ea39901d109afd6ba grayboxes.com/\n' +\
' 24652ddc14687866e6b1251589aee7e1e3079a87f80cd' + \
'7775214f6d837612a90 images/\n' +\
' 1eb774eef9be1e696f69a2f95711be37915aac283bb4' + \
'b34dcbaf7d032233e090 grayboxes.gif\n' +\
' 6eacebda9fd55b59c0d2e48e2ed59ce9fd683379592f8' + \
'e662b1de88e041f53c9 index.html\n'
# create from string array ----------------------------------
string = serialization.split('\n')
string = string[:-1]
self.assertEqual(4, len(string))
tree2 = MerkleTree.create_from_string_array(string, HashTypes.SHA2)
ser2 = tree2.to_string(0)
self.assertEqual(serialization, ser2)
# create from serialization ---------------------------------
tree1 = MerkleTree.create_from_serialization(serialization,
HashTypes.SHA2)
ser1 = tree1.to_string(0)
self.assertEqual(serialization, ser1)
self.assertTrue(tree1 == tree2)
# 2014-06-26 tagged this on here to test firstLineRE_1()
first_line = string[0]
match_ = MerkleTree.first_line_re_2().match(first_line)
self.assertTrue(match_ is not None)
self.assertEqual(match_.group(1), '') # indent
tree_hash = match_.group(2)
dir_name = match_.group(3)
self.assertEqual(tree_hash + ' ' + dir_name, first_line)
def test_gray_boxes_bug3(self):
""" Test solution to bug in handling grayboxes website. """
serialization =\
'088d0e391e1a4872329e0f7ac5d45b2025363e26c199a7' + \
'4ea39901d109afd6ba grayboxes.com/\n' +\
' 24652ddc14687866e6b1251589aee7e1e3079a87f80cd' + \
'7775214f6d837612a90 images/\n' +\
' 1eb774eef9be1e696f69a2f95711be37915aac283bb4' + \
'b34dcbaf7d032233e090 grayboxes.gif\n' +\
' 6eacebda9fd55b59c0d2e48e2ed59ce9fd683379592f8' + \
'e662b1de88e041f53c9 index.html\n'
# create from string array ----------------------------------
string = serialization.split('\n')
string = string[:-1]
self.assertEqual(4, len(string))
tree2 = MerkleTree.create_from_string_array(string, HashTypes.SHA2)
ser2 = tree2.to_string(0)
self.assertEqual(serialization, ser2)
# create from serialization ---------------------------------
tree1 = MerkleTree.create_from_serialization(
serialization, HashTypes.SHA2)
ser1 = tree1.to_string(0)
self.assertEqual(serialization, ser1)
self.assertTrue(tree1 == tree2)
# 2014-06-26 tagged this on here to test firstLineRE_1()
first_line = string[0]
match_ = MerkleTree.first_line_re_2().match(first_line)
self.assertTrue(match_ is not None)
self.assertEqual(match_.group(1), '') # indent
tree_hash = match_.group(2)
dir_name = match_.group(3)
self.assertEqual(tree_hash + ' ' + dir_name, first_line)
def test_gray_boxes_bug1(self):
"""
Verify that bug #1 in handling serialization of grayboxes website
has been corrected.
"""
serialization =\
'721a08022dd26e7be98b723f26131786fd2c0dc3 grayboxes.com/\n' +\
' fcd3973c66230b9078a86a5642b4c359fe72d7da images/\n' +\
' 15e47f4eb55197e1bfffae897e9d5ce4cba49623 grayboxes.gif\n' +\
' 2477b9ea649f3f30c6ed0aebacfa32cb8250f3df index.html\n'
# create from string array ----------------------------------
string = serialization.split('\n')
string = string[:-1]
self.assertEqual(4, len(string))
tree2 = MerkleTree.create_from_string_array(string, HashTypes.SHA1)
ser2 = tree2.to_string(0)
self.assertEqual(serialization, ser2)
# create from serialization ---------------------------------
tree1 = MerkleTree.create_from_serialization(serialization,
HashTypes.SHA1)
ser1 = tree1.to_string(0)
self.assertEqual(serialization, ser1)
self.assertTrue(tree1 == tree2)
# 2014-06-26 tagged this on here to test firstLineRE_1()
first_line = string[0]
match_ = MerkleTree.first_line_re_1().match(first_line)
self.assertTrue(match_ is not None)
self.assertEqual(match_.group(1), '') # indent
tree_hash = match_.group(2)
dir_name = match_.group(3)
self.assertEqual(tree_hash + ' ' + dir_name, first_line)
def test_gray_boxes_bug1(self):
"""
Verify that bug #1 in handling serialization of grayboxes website
has been corrected.
"""
serialization =\
'721a08022dd26e7be98b723f26131786fd2c0dc3 grayboxes.com/\n' +\
' fcd3973c66230b9078a86a5642b4c359fe72d7da images/\n' +\
' 15e47f4eb55197e1bfffae897e9d5ce4cba49623 grayboxes.gif\n' +\
' 2477b9ea649f3f30c6ed0aebacfa32cb8250f3df index.html\n'
# create from string array ----------------------------------
string = serialization.split('\n')
string = string[:-1]
self.assertEqual(4, len(string))
tree2 = MerkleTree.create_from_string_array(string, HashTypes.SHA1)
ser2 = tree2.to_string(0)
self.assertEqual(serialization, ser2)
# create from serialization ---------------------------------
tree1 = MerkleTree.create_from_serialization(
serialization, HashTypes.SHA1)
ser1 = tree1.to_string(0)
self.assertEqual(serialization, ser1)
self.assertTrue(tree1 == tree2)
# 2014-06-26 tagged this on here to test firstLineRE_1()
first_line = string[0]
match_ = MerkleTree.first_line_re_1().match(first_line)
self.assertTrue(match_ is not None)
self.assertEqual(match_.group(1), '') # indent
tree_hash = match_.group(2)
dir_name = match_.group(3)
self.assertEqual(tree_hash + ' ' + dir_name, first_line)