def __init__(self, input_dir): self.input_dir = input_dir self.root_dir_name = path.basename(input_dir) self.header = FileHeader() self.indices = {} # {id:index, ...} self.cookies = {} # {md5:cookie, ...} self.index_list = [] self.cookie_list = []
class StormPacker(object): def __init__(self, input_dir): self.input_dir = input_dir self.root_dir_name = path.basename(input_dir) self.header = FileHeader() self.indices = {} # {id:index, ...} self.cookies = {} # {md5:cookie, ...} self.index_list = [] self.cookie_list = [] def walk_files(self): index_offset = FileHeader.STRUCT_DEF.size for root, dirs, files in os.walk(self.input_dir): for filename in files: fullpath = path.join(root, filename) pos = fullpath.find(self.root_dir_name) searchpath = fullpath[pos:].replace('\\', '/') # FileIndex fileindex = FileIndex() fileindex.id = hash(searchpath) assert self.indices.has_key(fileindex.id) == False self.indices[fileindex.id] = fileindex f = io.open(fullpath, 'rb') data = f.read() f.close() fileindex.offset = index_offset fileindex.size = len(data) self.index_list.append(fileindex) index_offset += fileindex.size # FileCookie m = hashlib.md5() m.update(data) digest = m.digest() if self.cookies.has_key(digest): cookie = self.cookies[digest] cookie.refcount += 1 else: cookie = FileCookie() cookie.digest = digest cookie.space = fileindex.size self.cookies[digest] = cookie self.cookie_list.append((cookie, data)) self.header.number = len(self.index_list) self.header.offset = index_offset def save_to_file(self): outputfile = self.input_dir + '.sto' f = io.open(outputfile, 'wb') f.write(self.header.pack()) for cookie, data in self.cookie_list: f.write(cookie.pack()) f.write(data) for index in self.index_list: f.write(index.pack()) f.close() def pack(self): print 'packing \'%s\' ...' % (self.input_dir,) self.walk_files() self.save_to_file()