def verify_manifest(archive, manifest, brief=False):
""" Debugging function to verify the integrity of a manifest. """
failures = 0
for name in manifest.name_map:
tmp = archive.blocks.tmps.make_temp_file()
file_sha, link_sha = manifest.name_map[name]
if not brief:
print "Verifying: %s %s => %s)" % (name,
str_sha(file_sha),
str_sha(link_sha))
archive.get_file(link_sha, tmp)
history = archive.blocks.get_history(link_sha)
if not brief:
print "History: " + " ".join([str_sha(link[0])
for link in history])
retrieved_sha = get_file_sha(tmp)
if retrieved_sha != file_sha:
print "Expected: %s, but got %s." % (str_sha(file_sha),
str_sha(retrieved_sha))
failures += 1
else:
if not brief:
print "Ok. Read %i bytes." % os.path.getsize(tmp)
archive.blocks.tmps.remove_temp_file(tmp)
if failures > 0:
print "%i entries failed to verify!" % failures
assert False
def verify_manifest(archive, manifest, brief=False):
""" Debugging function to verify the integrity of a manifest. """
failures = 0
for name in manifest.name_map:
tmp = archive.blocks.tmps.make_temp_file()
file_sha, link_sha = manifest.name_map[name]
if not brief:
print "Verifying: %s %s => %s)" % (name, str_sha(file_sha),
str_sha(link_sha))
archive.get_file(link_sha, tmp)
history = archive.blocks.get_history(link_sha)
if not brief:
print "History: " + " ".join(
[str_sha(link[0]) for link in history])
retrieved_sha = get_file_sha(tmp)
if retrieved_sha != file_sha:
print "Expected: %s, but got %s." % (str_sha(file_sha),
str_sha(retrieved_sha))
failures += 1
else:
if not brief:
print "Ok. Read %i bytes." % os.path.getsize(tmp)
archive.blocks.tmps.remove_temp_file(tmp)
if failures > 0:
print "%i entries failed to verify!" % failures
assert False
def test_hg_repo_torture_test(self):
if HG_REPO_DIR == '':
print "Set HG_REPO_DIR!"
self.assertTrue(False)
writer = self.make_empty_archive('hgtst')
manifest = FileManifest()
rev = 0
max_rev = 1 # Set below
while rev < max_rev:
target_dir = os.path.join(self.tmp_dir, '__hg_repo__')
if os.path.exists(target_dir):
shutil.rmtree(target_dir) # DANGEROUS
# export the repo
# FIX: Wacky way to set max_rev.
print "Exporting rev: ", rev
max_rev = export_hg_repo(HG_REPO_DIR, target_dir, rev)
if rev >= max_rev:
break
# put the export dir into the archive
# print "Inserting into the archive..."
entries = entries_from_dir(target_dir, True)
manifest.update(writer, entries)
# Will be written into Freenet top key
# along with rest of archive info.
s3kr1t = manifest.stored_sha
dump_blocks(writer.blocks, None, True)
# create a second archive instance from the same block files.
# REDFLAG: Would this work on windoze?
# writer still has files open for reading.
reader = self.load_archive('hgtst')
read_manifest = FileManifest.from_archive(reader, s3kr1t)
# REDFLAG: audit for other places where I could do
# direct dict compares?
assert (read_manifest.name_map == manifest.name_map)
# clean the archive output dir
unarchived_dir = os.path.join(self.tmp_dir, '__unarchived__')
if os.path.exists(unarchived_dir):
shutil.rmtree(unarchived_dir) # DANGEROUS
os.makedirs(unarchived_dir)
# extract the archive to the cleaned files
manifest_to_dir(reader, read_manifest, unarchived_dir)
reader.close()
# diff the directories
# A poor man's diff.
insert_map = {}
for entry in entries_from_dir(target_dir, True):
insert_map[entry.get_name()] = get_file_sha(entry.make_file())
entry.release() # NOP
unarchived_map = {}
for entry in entries_from_dir(unarchived_dir, True):
unarchived_map[entry.get_name()] = (get_file_sha(
entry.make_file()))
entry.release() # NOP
assert len(insert_map) > 0
assert insert_map == unarchived_map
print "%i files compared equal." % len(insert_map)
rev += 1
def test_hg_repo_torture_test(self):
if HG_REPO_DIR == '':
print "Set HG_REPO_DIR!"
self.assertTrue(False)
writer = self.make_empty_archive('hgtst')
manifest = FileManifest()
rev = 0
max_rev = 1 # Set below
while rev < max_rev:
target_dir = os.path.join(self.tmp_dir, '__hg_repo__')
if os.path.exists(target_dir):
shutil.rmtree(target_dir) # DANGEROUS
# export the repo
# FIX: Wacky way to set max_rev.
print "Exporting rev: ", rev
max_rev = export_hg_repo(HG_REPO_DIR, target_dir, rev)
if rev >= max_rev:
break
# put the export dir into the archive
# print "Inserting into the archive..."
entries = entries_from_dir(target_dir, True)
manifest.update(writer, entries)
# Will be written into Freenet top key
# along with rest of archive info.
s3kr1t = manifest.stored_sha
dump_blocks(writer.blocks, None, True)
# create a second archive instance from the same block files.
# REDFLAG: Would this work on windoze?
# writer still has files open for reading.
reader = self.load_archive('hgtst')
read_manifest = FileManifest.from_archive(reader, s3kr1t)
# REDFLAG: audit for other places where I could do
# direct dict compares?
assert (read_manifest.name_map == manifest.name_map)
# clean the archive output dir
unarchived_dir = os.path.join(self.tmp_dir, '__unarchived__')
if os.path.exists(unarchived_dir):
shutil.rmtree(unarchived_dir) # DANGEROUS
os.makedirs(unarchived_dir)
# extract the archive to the cleaned files
manifest_to_dir(reader, read_manifest, unarchived_dir)
reader.close()
# diff the directories
# A poor man's diff.
insert_map = {}
for entry in entries_from_dir(target_dir, True):
insert_map[entry.get_name()] = get_file_sha(entry.make_file())
entry.release() # NOP
unarchived_map = {}
for entry in entries_from_dir(unarchived_dir, True):
unarchived_map[entry.get_name()] = (
get_file_sha(entry.make_file()))
entry.release() # NOP
assert len(insert_map) > 0
assert insert_map == unarchived_map
print "%i files compared equal." % len(insert_map)
rev += 1
def write_changes(self, archive, entry_infos, prev_manifest_sha=NULL_SHA):
""" INTERNAL: Helper function for update().
Writes the changes required to add the IManifestEntries
in entries_infos to an archive.
Raises UpToDateException if there are no changes.
Return an (updated_name_map, manifest_sha) tuple. """
check_shas([prev_manifest_sha, ])
file_sha_map = self.make_file_sha_map()
new_name_map = {}
updated = False
for info in entry_infos:
full_path = info.make_file()
try:
name = info.get_name()
if not is_printable_ascii(name):
raise IOError("Non-ASCII name: %s" % repr(name))
hash_info = self.name_map.get(name, None)
file_sha = get_file_sha(full_path)
if hash_info is None:
updated = True
if file_sha in file_sha_map:
# Renamed
new_name_map[name] = file_sha_map[file_sha]
else:
# REDFLAG: We lose history for files which are renamed
# and modified.
# Created (or renamed and modified)
link = archive.write_new_delta(NULL_SHA, full_path)
new_name_map[name] = (file_sha, link[0])
else:
if self.name_map[name][0] == file_sha:
# Exists in manifest and is unmodified.
new_name_map[name] = self.name_map[name]
continue
# Modified
updated = True
link = archive.write_new_delta(self.name_map[name][1],
full_path)
new_name_map[name] = (file_sha, link[0])
# delete == ophaned history, NOP
finally:
info.release()
if not updated:
if (frozenset(new_name_map.keys()) ==
frozenset(self.name_map.keys())):
raise UpToDateException("The file manifest is up to date.")
# Add updated manifest
link = FileManifest.write_manifest(archive, new_name_map,
prev_manifest_sha)
return (new_name_map, link[0])
def write_changes(self, archive, entry_infos, prev_manifest_sha=NULL_SHA):
""" INTERNAL: Helper function for update().
Writes the changes required to add the IManifestEntries
in entries_infos to an archive.
Raises UpToDateException if there are no changes.
Return an (updated_name_map, manifest_sha) tuple. """
check_shas([
prev_manifest_sha,
])
file_sha_map = self.make_file_sha_map()
new_name_map = {}
updated = False
for info in entry_infos:
full_path = info.make_file()
try:
name = info.get_name()
if not is_printable_ascii(name):
raise IOError("Non-ASCII name: %s" % repr(name))
hash_info = self.name_map.get(name, None)
file_sha = get_file_sha(full_path)
if hash_info is None:
updated = True
if file_sha in file_sha_map:
# Renamed
new_name_map[name] = file_sha_map[file_sha]
else:
# REDFLAG: We lose history for files which are renamed
# and modified.
# Created (or renamed and modified)
link = archive.write_new_delta(NULL_SHA, full_path)
new_name_map[name] = (file_sha, link[0])
else:
if self.name_map[name][0] == file_sha:
# Exists in manifest and is unmodified.
new_name_map[name] = self.name_map[name]
continue
# Modified
updated = True
link = archive.write_new_delta(self.name_map[name][1],
full_path)
new_name_map[name] = (file_sha, link[0])
# delete == ophaned history, NOP
finally:
info.release()
if not updated:
if (frozenset(new_name_map.keys()) == frozenset(
self.name_map.keys())):
raise UpToDateException("The file manifest is up to date.")
# Add updated manifest
link = FileManifest.write_manifest(archive, new_name_map,
prev_manifest_sha)
return (new_name_map, link[0])