def compile(self): print_progress(self.stats) writeln() self.failed_articles.close() self.empty_articles.close() self.skipped_articles.close() writeln('Compiling .aar files') self.add_metadata("article_count", self.stats.articles) articles = self.article_store.sorted(key=lambda x: collation_key(x).getByteArray()) log.info('Compiling %s', self.output_file_name) metadata = compress(tojson(self.metadata).encode('utf8')) header_meta_len = spec_len(HEADER_SPEC) + len(metadata) create_volume_func = functools.partial(self.create_volume, header_meta_len) for volume in self.make_volumes(create_volume_func, articles): m = "Creating volume %d" % volume.number log.info(m) writeln(m).flush() file_name = self.make_aar(volume) self.file_names.append(file_name) m = "Wrote volume %d" % volume.number log.info(m) writeln(m).flush() self.article_store.close() self.write_volume_count() self.write_sha1sum() rename_files(self.file_names)
def compile(self): print_progress(self.stats) writeln() self.failed_articles.close() self.empty_articles.close() self.skipped_articles.close() writeln('Compiling .aar files') self.add_metadata("article_count", self.stats.articles) articles = self.article_store.sorted( key=lambda x: collation_key(x).getByteArray()) log.info('Compiling %s', self.output_file_name) metadata = compress(tojson(self.metadata).encode('utf8')) header_meta_len = spec_len(HEADER_SPEC) + len(metadata) create_volume_func = functools.partial(self.create_volume, header_meta_len) for volume in self.make_volumes(create_volume_func, articles): m = "Creating volume %d" % volume.number log.info(m) writeln(m).flush() file_name = self.make_aar(volume) self.file_names.append(file_name) m = "Wrote volume %d" % volume.number log.info(m) writeln(m).flush() self.article_store.close() self.write_volume_count() self.write_sha1sum() rename_files(self.file_names)
def write_sha1sum(self): for file_name in self.file_names: msg = "Calculating checksum for %s" % file_name log.info(msg) offset = spec_len(HEADER_SPEC[:2]) st_size = os.stat(file_name).st_size size = float(st_size - offset) for pos, sha1sum in calcsha1(file_name, offset): (display.erase_line().cr() .write(msg).write(': ').write('%.1f%%' % (100*pos/size))) sha1sum = sha1sum.hexdigest() msg = "%s sha1: %s" % (file_name, sha1sum) log.info(msg) display.erase_line().cr().writeln(msg) output_file = open(file_name, "r+b") output_file.seek(spec_len(HEADER_SPEC[:1])) output_file.write(sha1sum) output_file.close()
def write_sha1sum(self): for file_name in self.file_names: msg = "Calculating checksum for %s" % file_name log.info(msg) offset = spec_len(HEADER_SPEC[:2]) st_size = os.stat(file_name).st_size size = float(st_size - offset) for pos, sha1sum in calcsha1(file_name, offset): (display.erase_line().cr().write(msg).write(': ').write( '%.1f%%' % (100 * pos / size))) sha1sum = sha1sum.hexdigest() msg = "%s sha1: %s" % (file_name, sha1sum) log.info(msg) display.erase_line().cr().writeln(msg) output_file = open(file_name, "r+b") output_file.seek(spec_len(HEADER_SPEC[:1])) output_file.write(sha1sum) output_file.close()
def write_volume_count(self): _name, fmt = HEADER_SPEC[5] log.info("Writing volume count %d to all volumes as %s", Volume.number, fmt) log.debug('Writing' ) for file_name in self.file_names: output_file = open(file_name, "r+b") output_file.seek(spec_len(HEADER_SPEC[:5])) output_file.write(struct.pack(fmt, Volume.number)) output_file.close()
def write_volume_count(self): name, fmt = HEADER_SPEC[5] log.info("Writing volume count %d to all volumes as %s", Volume.number, fmt) log.debug('Writing') for file_name in self.file_names: output_file = open(file_name, "r+b") output_file.seek(spec_len(HEADER_SPEC[:5])) output_file.write(struct.pack(fmt, Volume.number)) output_file.close()
def write_header(self, output_file, meta_length, index1Length, index2Length, index_count, volume): article_offset = (spec_len(HEADER_SPEC) + meta_length + index1Length + index2Length) values = dict(signature='aard', sha1sum='0'*40, version=1, uuid=self.uuid.bytes, volume=volume, of=0, total_volumes=0, meta_length=meta_length, index_count=index_count, article_offset=article_offset, index1_item_format=INDEX1_ITEM_FORMAT, key_length_format=KEY_LENGTH_FORMAT, article_length_format=ARTICLE_LENGTH_FORMAT) for name, fmt in HEADER_SPEC: output_file.write(struct.pack(fmt, values[name]))
def write_header(self, output_file, meta_length, index1Length, index2Length, index_count, volume): article_offset = (spec_len(HEADER_SPEC) + meta_length + index1Length + index2Length) values = dict(signature='aard', sha1sum='0' * 40, version=1, uuid=self.uuid.bytes, volume=volume, of=0, total_volumes=0, meta_length=meta_length, index_count=index_count, article_offset=article_offset, index1_item_format=INDEX1_ITEM_FORMAT, key_length_format=KEY_LENGTH_FORMAT, article_length_format=ARTICLE_LENGTH_FORMAT) for name, fmt in HEADER_SPEC: output_file.write(struct.pack(fmt, values[name]))
def create_volume(self): header_meta_len = spec_len(HEADER_SPEC) + len(self.serialized_metadata) return Volume(self.uuid, header_meta_len, self.max_file_size, self.session_dir)