def verify_manifest(archive, manifest, brief=False): """ Debugging function to verify the integrity of a manifest. """ failures = 0 for name in manifest.name_map: tmp = archive.blocks.tmps.make_temp_file() file_sha, link_sha = manifest.name_map[name] if not brief: print "Verifying: %s %s => %s)" % (name, str_sha(file_sha), str_sha(link_sha)) archive.get_file(link_sha, tmp) history = archive.blocks.get_history(link_sha) if not brief: print "History: " + " ".join([str_sha(link[0]) for link in history]) retrieved_sha = get_file_sha(tmp) if retrieved_sha != file_sha: print "Expected: %s, but got %s." % (str_sha(file_sha), str_sha(retrieved_sha)) failures += 1 else: if not brief: print "Ok. Read %i bytes." % os.path.getsize(tmp) archive.blocks.tmps.remove_temp_file(tmp) if failures > 0: print "%i entries failed to verify!" % failures assert False
def verify_manifest(archive, manifest, brief=False): """ Debugging function to verify the integrity of a manifest. """ failures = 0 for name in manifest.name_map: tmp = archive.blocks.tmps.make_temp_file() file_sha, link_sha = manifest.name_map[name] if not brief: print "Verifying: %s %s => %s)" % (name, str_sha(file_sha), str_sha(link_sha)) archive.get_file(link_sha, tmp) history = archive.blocks.get_history(link_sha) if not brief: print "History: " + " ".join( [str_sha(link[0]) for link in history]) retrieved_sha = get_file_sha(tmp) if retrieved_sha != file_sha: print "Expected: %s, but got %s." % (str_sha(file_sha), str_sha(retrieved_sha)) failures += 1 else: if not brief: print "Ok. Read %i bytes." % os.path.getsize(tmp) archive.blocks.tmps.remove_temp_file(tmp) if failures > 0: print "%i entries failed to verify!" % failures assert False
def test_hg_repo_torture_test(self): if HG_REPO_DIR == '': print "Set HG_REPO_DIR!" self.assertTrue(False) writer = self.make_empty_archive('hgtst') manifest = FileManifest() rev = 0 max_rev = 1 # Set below while rev < max_rev: target_dir = os.path.join(self.tmp_dir, '__hg_repo__') if os.path.exists(target_dir): shutil.rmtree(target_dir) # DANGEROUS # export the repo # FIX: Wacky way to set max_rev. print "Exporting rev: ", rev max_rev = export_hg_repo(HG_REPO_DIR, target_dir, rev) if rev >= max_rev: break # put the export dir into the archive # print "Inserting into the archive..." entries = entries_from_dir(target_dir, True) manifest.update(writer, entries) # Will be written into Freenet top key # along with rest of archive info. s3kr1t = manifest.stored_sha dump_blocks(writer.blocks, None, True) # create a second archive instance from the same block files. # REDFLAG: Would this work on windoze? # writer still has files open for reading. reader = self.load_archive('hgtst') read_manifest = FileManifest.from_archive(reader, s3kr1t) # REDFLAG: audit for other places where I could do # direct dict compares? assert (read_manifest.name_map == manifest.name_map) # clean the archive output dir unarchived_dir = os.path.join(self.tmp_dir, '__unarchived__') if os.path.exists(unarchived_dir): shutil.rmtree(unarchived_dir) # DANGEROUS os.makedirs(unarchived_dir) # extract the archive to the cleaned files manifest_to_dir(reader, read_manifest, unarchived_dir) reader.close() # diff the directories # A poor man's diff. insert_map = {} for entry in entries_from_dir(target_dir, True): insert_map[entry.get_name()] = get_file_sha(entry.make_file()) entry.release() # NOP unarchived_map = {} for entry in entries_from_dir(unarchived_dir, True): unarchived_map[entry.get_name()] = (get_file_sha( entry.make_file())) entry.release() # NOP assert len(insert_map) > 0 assert insert_map == unarchived_map print "%i files compared equal." % len(insert_map) rev += 1
def test_hg_repo_torture_test(self): if HG_REPO_DIR == '': print "Set HG_REPO_DIR!" self.assertTrue(False) writer = self.make_empty_archive('hgtst') manifest = FileManifest() rev = 0 max_rev = 1 # Set below while rev < max_rev: target_dir = os.path.join(self.tmp_dir, '__hg_repo__') if os.path.exists(target_dir): shutil.rmtree(target_dir) # DANGEROUS # export the repo # FIX: Wacky way to set max_rev. print "Exporting rev: ", rev max_rev = export_hg_repo(HG_REPO_DIR, target_dir, rev) if rev >= max_rev: break # put the export dir into the archive # print "Inserting into the archive..." entries = entries_from_dir(target_dir, True) manifest.update(writer, entries) # Will be written into Freenet top key # along with rest of archive info. s3kr1t = manifest.stored_sha dump_blocks(writer.blocks, None, True) # create a second archive instance from the same block files. # REDFLAG: Would this work on windoze? # writer still has files open for reading. reader = self.load_archive('hgtst') read_manifest = FileManifest.from_archive(reader, s3kr1t) # REDFLAG: audit for other places where I could do # direct dict compares? assert (read_manifest.name_map == manifest.name_map) # clean the archive output dir unarchived_dir = os.path.join(self.tmp_dir, '__unarchived__') if os.path.exists(unarchived_dir): shutil.rmtree(unarchived_dir) # DANGEROUS os.makedirs(unarchived_dir) # extract the archive to the cleaned files manifest_to_dir(reader, read_manifest, unarchived_dir) reader.close() # diff the directories # A poor man's diff. insert_map = {} for entry in entries_from_dir(target_dir, True): insert_map[entry.get_name()] = get_file_sha(entry.make_file()) entry.release() # NOP unarchived_map = {} for entry in entries_from_dir(unarchived_dir, True): unarchived_map[entry.get_name()] = ( get_file_sha(entry.make_file())) entry.release() # NOP assert len(insert_map) > 0 assert insert_map == unarchived_map print "%i files compared equal." % len(insert_map) rev += 1
def write_changes(self, archive, entry_infos, prev_manifest_sha=NULL_SHA): """ INTERNAL: Helper function for update(). Writes the changes required to add the IManifestEntries in entries_infos to an archive. Raises UpToDateException if there are no changes. Return an (updated_name_map, manifest_sha) tuple. """ check_shas([prev_manifest_sha, ]) file_sha_map = self.make_file_sha_map() new_name_map = {} updated = False for info in entry_infos: full_path = info.make_file() try: name = info.get_name() if not is_printable_ascii(name): raise IOError("Non-ASCII name: %s" % repr(name)) hash_info = self.name_map.get(name, None) file_sha = get_file_sha(full_path) if hash_info is None: updated = True if file_sha in file_sha_map: # Renamed new_name_map[name] = file_sha_map[file_sha] else: # REDFLAG: We lose history for files which are renamed # and modified. # Created (or renamed and modified) link = archive.write_new_delta(NULL_SHA, full_path) new_name_map[name] = (file_sha, link[0]) else: if self.name_map[name][0] == file_sha: # Exists in manifest and is unmodified. new_name_map[name] = self.name_map[name] continue # Modified updated = True link = archive.write_new_delta(self.name_map[name][1], full_path) new_name_map[name] = (file_sha, link[0]) # delete == ophaned history, NOP finally: info.release() if not updated: if (frozenset(new_name_map.keys()) == frozenset(self.name_map.keys())): raise UpToDateException("The file manifest is up to date.") # Add updated manifest link = FileManifest.write_manifest(archive, new_name_map, prev_manifest_sha) return (new_name_map, link[0])
def write_changes(self, archive, entry_infos, prev_manifest_sha=NULL_SHA): """ INTERNAL: Helper function for update(). Writes the changes required to add the IManifestEntries in entries_infos to an archive. Raises UpToDateException if there are no changes. Return an (updated_name_map, manifest_sha) tuple. """ check_shas([ prev_manifest_sha, ]) file_sha_map = self.make_file_sha_map() new_name_map = {} updated = False for info in entry_infos: full_path = info.make_file() try: name = info.get_name() if not is_printable_ascii(name): raise IOError("Non-ASCII name: %s" % repr(name)) hash_info = self.name_map.get(name, None) file_sha = get_file_sha(full_path) if hash_info is None: updated = True if file_sha in file_sha_map: # Renamed new_name_map[name] = file_sha_map[file_sha] else: # REDFLAG: We lose history for files which are renamed # and modified. # Created (or renamed and modified) link = archive.write_new_delta(NULL_SHA, full_path) new_name_map[name] = (file_sha, link[0]) else: if self.name_map[name][0] == file_sha: # Exists in manifest and is unmodified. new_name_map[name] = self.name_map[name] continue # Modified updated = True link = archive.write_new_delta(self.name_map[name][1], full_path) new_name_map[name] = (file_sha, link[0]) # delete == ophaned history, NOP finally: info.release() if not updated: if (frozenset(new_name_map.keys()) == frozenset( self.name_map.keys())): raise UpToDateException("The file manifest is up to date.") # Add updated manifest link = FileManifest.write_manifest(archive, new_name_map, prev_manifest_sha) return (new_name_map, link[0])