def compile(self):
print_progress(self.stats)
writeln()
self.failed_articles.close()
self.empty_articles.close()
self.skipped_articles.close()
writeln('Compiling .aar files')
self.add_metadata("article_count", self.stats.articles)
articles = self.article_store.sorted(key=lambda x:
collation_key(x).getByteArray())
log.info('Compiling %s', self.output_file_name)
metadata = compress(tojson(self.metadata).encode('utf8'))
header_meta_len = spec_len(HEADER_SPEC) + len(metadata)
create_volume_func = functools.partial(self.create_volume,
header_meta_len)
for volume in self.make_volumes(create_volume_func, articles):
m = "Creating volume %d" % volume.number
log.info(m)
writeln(m).flush()
file_name = self.make_aar(volume)
self.file_names.append(file_name)
m = "Wrote volume %d" % volume.number
log.info(m)
writeln(m).flush()
self.article_store.close()
self.write_volume_count()
self.write_sha1sum()
rename_files(self.file_names)
def compile(self):
print_progress(self.stats)
writeln()
self.failed_articles.close()
self.empty_articles.close()
self.skipped_articles.close()
writeln('Compiling .aar files')
self.add_metadata("article_count", self.stats.articles)
articles = self.article_store.sorted(
key=lambda x: collation_key(x).getByteArray())
log.info('Compiling %s', self.output_file_name)
metadata = compress(tojson(self.metadata).encode('utf8'))
header_meta_len = spec_len(HEADER_SPEC) + len(metadata)
create_volume_func = functools.partial(self.create_volume,
header_meta_len)
for volume in self.make_volumes(create_volume_func, articles):
m = "Creating volume %d" % volume.number
log.info(m)
writeln(m).flush()
file_name = self.make_aar(volume)
self.file_names.append(file_name)
m = "Wrote volume %d" % volume.number
log.info(m)
writeln(m).flush()
self.article_store.close()
self.write_volume_count()
self.write_sha1sum()
rename_files(self.file_names)
def write_sha1sum(self):
for file_name in self.file_names:
msg = "Calculating checksum for %s" % file_name
log.info(msg)
offset = spec_len(HEADER_SPEC[:2])
st_size = os.stat(file_name).st_size
size = float(st_size - offset)
for pos, sha1sum in calcsha1(file_name, offset):
(display.erase_line().cr()
.write(msg).write(': ').write('%.1f%%' % (100*pos/size)))
sha1sum = sha1sum.hexdigest()
msg = "%s sha1: %s" % (file_name, sha1sum)
log.info(msg)
display.erase_line().cr().writeln(msg)
output_file = open(file_name, "r+b")
output_file.seek(spec_len(HEADER_SPEC[:1]))
output_file.write(sha1sum)
output_file.close()
def write_sha1sum(self):
for file_name in self.file_names:
msg = "Calculating checksum for %s" % file_name
log.info(msg)
offset = spec_len(HEADER_SPEC[:2])
st_size = os.stat(file_name).st_size
size = float(st_size - offset)
for pos, sha1sum in calcsha1(file_name, offset):
(display.erase_line().cr().write(msg).write(': ').write(
'%.1f%%' % (100 * pos / size)))
sha1sum = sha1sum.hexdigest()
msg = "%s sha1: %s" % (file_name, sha1sum)
log.info(msg)
display.erase_line().cr().writeln(msg)
output_file = open(file_name, "r+b")
output_file.seek(spec_len(HEADER_SPEC[:1]))
output_file.write(sha1sum)
output_file.close()
def write_volume_count(self):
_name, fmt = HEADER_SPEC[5]
log.info("Writing volume count %d to all volumes as %s",
Volume.number, fmt)
log.debug('Writing' )
for file_name in self.file_names:
output_file = open(file_name, "r+b")
output_file.seek(spec_len(HEADER_SPEC[:5]))
output_file.write(struct.pack(fmt, Volume.number))
output_file.close()
def write_volume_count(self):
name, fmt = HEADER_SPEC[5]
log.info("Writing volume count %d to all volumes as %s", Volume.number,
fmt)
log.debug('Writing')
for file_name in self.file_names:
output_file = open(file_name, "r+b")
output_file.seek(spec_len(HEADER_SPEC[:5]))
output_file.write(struct.pack(fmt, Volume.number))
output_file.close()
def write_header(self, output_file, meta_length, index1Length,
index2Length, index_count, volume):
article_offset = (spec_len(HEADER_SPEC) + meta_length +
index1Length + index2Length)
values = dict(signature='aard',
sha1sum='0'*40,
version=1,
uuid=self.uuid.bytes,
volume=volume,
of=0,
total_volumes=0,
meta_length=meta_length,
index_count=index_count,
article_offset=article_offset,
index1_item_format=INDEX1_ITEM_FORMAT,
key_length_format=KEY_LENGTH_FORMAT,
article_length_format=ARTICLE_LENGTH_FORMAT)
for name, fmt in HEADER_SPEC:
output_file.write(struct.pack(fmt, values[name]))
def write_header(self, output_file, meta_length, index1Length,
index2Length, index_count, volume):
article_offset = (spec_len(HEADER_SPEC) + meta_length + index1Length +
index2Length)
values = dict(signature='aard',
sha1sum='0' * 40,
version=1,
uuid=self.uuid.bytes,
volume=volume,
of=0,
total_volumes=0,
meta_length=meta_length,
index_count=index_count,
article_offset=article_offset,
index1_item_format=INDEX1_ITEM_FORMAT,
key_length_format=KEY_LENGTH_FORMAT,
article_length_format=ARTICLE_LENGTH_FORMAT)
for name, fmt in HEADER_SPEC:
output_file.write(struct.pack(fmt, values[name]))
def create_volume(self):
header_meta_len = spec_len(HEADER_SPEC) + len(self.serialized_metadata)
return Volume(self.uuid,
header_meta_len,
self.max_file_size,
self.session_dir)
def create_volume(self):
header_meta_len = spec_len(HEADER_SPEC) + len(self.serialized_metadata)
return Volume(self.uuid, header_meta_len, self.max_file_size,
self.session_dir)