def test_multiple_updates(self):
a = self.make_empty_archive('A')
m = FileManifest()
data0 = ( \
('foo.txt', 'This is the foo file.\n'),
('empty.txt', ''),
('big.txt', '*' * (1 * 128)),
)
print "manifest sha: ", str_sha(m.stored_sha)
m.update(a, entries_from_seq(self.tmps, data0))
print "manifest sha: ", str_sha(m.stored_sha)
dump_archive(a, "AFTER FIRST WRITE:")
verify_manifest(a, m)
data1 = ( \
('foo.txt', 'This is the foo file.\n'),
('empty.txt', ''),
('big.txt', 'hello' + ('*' * (1 * 128))),
)
m.update(a, entries_from_seq(self.tmps, data1))
print "manifest sha: ", str_sha(m.stored_sha)
dump_archive(a)
verify_link_map(a.blocks.link_map)
verify_manifest(a, m)
def link_str(link):
return "(%s, %i, %s, data: %s, %i, %s)" % (str_sha(link[0]),
link[1],
str_sha(link[2]),
bool(link[3]),
link[4],
link[5])
def test_multiple_updates(self):
a = self.make_empty_archive('A')
m = FileManifest()
data0 = ( \
('foo.txt', 'This is the foo file.\n'),
('empty.txt', ''),
('big.txt', '*' * (1 * 128)),
)
print "manifest sha: ", str_sha(m.stored_sha)
m.update(a, entries_from_seq(self.tmps, data0))
print "manifest sha: ", str_sha(m.stored_sha)
dump_archive(a, "AFTER FIRST WRITE:")
verify_manifest(a, m)
data1 = ( \
('foo.txt', 'This is the foo file.\n'),
('empty.txt', ''),
('big.txt', 'hello' + ('*' * (1 * 128))),
)
m.update(a, entries_from_seq(self.tmps, data1))
print "manifest sha: ", str_sha(m.stored_sha)
dump_archive(a)
verify_link_map(a.blocks.link_map)
verify_manifest(a, m)
def verify_manifest(archive, manifest, brief=False):
""" Debugging function to verify the integrity of a manifest. """
failures = 0
for name in manifest.name_map:
tmp = archive.blocks.tmps.make_temp_file()
file_sha, link_sha = manifest.name_map[name]
if not brief:
print "Verifying: %s %s => %s)" % (name,
str_sha(file_sha),
str_sha(link_sha))
archive.get_file(link_sha, tmp)
history = archive.blocks.get_history(link_sha)
if not brief:
print "History: " + " ".join([str_sha(link[0])
for link in history])
retrieved_sha = get_file_sha(tmp)
if retrieved_sha != file_sha:
print "Expected: %s, but got %s." % (str_sha(file_sha),
str_sha(retrieved_sha))
failures += 1
else:
if not brief:
print "Ok. Read %i bytes." % os.path.getsize(tmp)
archive.blocks.tmps.remove_temp_file(tmp)
if failures > 0:
print "%i entries failed to verify!" % failures
assert False
def verify_manifest(archive, manifest, brief=False):
""" Debugging function to verify the integrity of a manifest. """
failures = 0
for name in manifest.name_map:
tmp = archive.blocks.tmps.make_temp_file()
file_sha, link_sha = manifest.name_map[name]
if not brief:
print "Verifying: %s %s => %s)" % (name, str_sha(file_sha),
str_sha(link_sha))
archive.get_file(link_sha, tmp)
history = archive.blocks.get_history(link_sha)
if not brief:
print "History: " + " ".join(
[str_sha(link[0]) for link in history])
retrieved_sha = get_file_sha(tmp)
if retrieved_sha != file_sha:
print "Expected: %s, but got %s." % (str_sha(file_sha),
str_sha(retrieved_sha))
failures += 1
else:
if not brief:
print "Ok. Read %i bytes." % os.path.getsize(tmp)
archive.blocks.tmps.remove_temp_file(tmp)
if failures > 0:
print "%i entries failed to verify!" % failures
assert False
def dump_names_map(names_map, msg=None):
if not msg is None:
print msg
keys = names_map.keys()
keys.sort()
for key in keys:
hashes = names_map[key]
print "%s->(%s, %s)" % (key, str_sha(hashes[0]), str_sha(hashes[1]))
def dump_names_map(names_map, msg=None):
if not msg is None:
print msg
keys = names_map.keys()
keys.sort()
for key in keys:
hashes = names_map[key]
print "%s->(%s, %s)" % (key, str_sha(hashes[0]), str_sha(hashes[1]))
def dump_link_map(link_map, msg=None, brief=False):
if not msg is None:
print msg
print "keys: ", len(link_map)
if brief:
return
keys = link_map.keys()
keys.sort()
for key in keys:
print str_sha(key)
dump_links(link_map[key])
def dump_link_map(link_map, msg=None, brief=False):
if not msg is None:
print msg
print "keys: ", len(link_map)
if brief:
return
keys = link_map.keys()
keys.sort()
for key in keys:
print str_sha(key)
dump_links(link_map[key])
def get_link(self, link_sha, need_data=False):
""" Get a history link by its sha1 hash. """
links = self.get(link_sha, None)
if links is None:
raise IOError("Unresolved link: " + str_sha(link_sha))
assert len(links) > 0
# REDFLAG: Fully think through.
# The same link can exist in multiple files.
link = links[0]
if (not need_data) or (not link[3] is None):
return link
index = link[5]
self.files[index].seek(link[4])
ret = read_link(self.files[index], True)
if ret is None:
raise IOError("Couldn't read blob from disk.")
assert ret[0] == link[0]
assert ret[1] == link[1]
assert ret[2] == link[2]
assert not ret[3] is None
assert ret[0] == link_sha
return ret
def test_archive_write_read(self):
a = self.make_empty_archive('A')
dump_archive(a, "empty")
r0 = self.write_file("OH HAI!")
r1 = self.write_file("OH HAI! AGAIN")
r2 = self.write_file("STILL ME")
t1 = self.tmps.make_temp_file()
try:
a.start_update()
link0 = a.write_new_delta(NULL_SHA, r0)
link1 = a.write_new_delta(NULL_SHA, r1)
link2 = a.write_new_delta(NULL_SHA, r2)
# Write
a.commit_update()
dump_archive(a, "updated")
# Read
print
print str_sha(link0[0]), a.get_data(link0[0])
print str_sha(link1[0]), a.get_data(link1[0])
print str_sha(link2[0]), a.get_data(link2[0])
a.close()
b = self.load_archive('A')
dump_archive(b, "[Reloaded from disk]")
print
# Mix up order.
print str_sha(link1[0]), b.get_data(link1[0])
print str_sha(link0[0]), b.get_data(link0[0])
print str_sha(link2[0]), b.get_data(link2[0])
finally:
self.tmps.remove_temp_file(t1)
self.tmps.remove_temp_file(r0)
self.tmps.remove_temp_file(r1)
self.tmps.remove_temp_file(r2)
def link_str(link):
return "(%s, %i, %s, data: %s, %i, %s)" % (str_sha(
link[0]), link[1], str_sha(link[2]), bool(link[3]), link[4], link[5])
def test_archive_write_read(self):
a = self.make_empty_archive('A')
dump_archive(a, "empty")
r0 = self.write_file("OH HAI!")
r1 = self.write_file("OH HAI! AGAIN")
r2 = self.write_file("STILL ME")
t1 = self.tmps.make_temp_file()
try:
a.start_update()
link0 = a.write_new_delta(NULL_SHA, r0)
link1 = a.write_new_delta(NULL_SHA, r1)
link2 = a.write_new_delta(NULL_SHA, r2)
# Write
a.commit_update()
dump_archive(a, "updated")
# Read
print
print str_sha(link0[0]), a.get_data(link0[0])
print str_sha(link1[0]), a.get_data(link1[0])
print str_sha(link2[0]), a.get_data(link2[0])
a.close()
b = self.load_archive('A')
dump_archive(b, "[Reloaded from disk]")
print
# Mix up order.
print str_sha(link1[0]), b.get_data(link1[0])
print str_sha(link0[0]), b.get_data(link0[0])
print str_sha(link2[0]), b.get_data(link2[0])
finally:
self.tmps.remove_temp_file(t1)
self.tmps.remove_temp_file(r0)
self.tmps.remove_temp_file(r1)
self.tmps.remove_temp_file(r2)
