Esempio n. 1
0
 def compile(self):
     print_progress(self.stats)
     writeln()
     self.failed_articles.close()
     self.empty_articles.close()
     self.skipped_articles.close()
     writeln('Compiling .aar files')
     self.add_metadata("article_count", self.stats.articles)
     articles = self.article_store.sorted(key=lambda x:
                                              collation_key(x).getByteArray())
     log.info('Compiling %s', self.output_file_name)
     metadata = compress(tojson(self.metadata).encode('utf8'))
     header_meta_len = spec_len(HEADER_SPEC) + len(metadata)
     create_volume_func = functools.partial(self.create_volume,
                                            header_meta_len)
     for volume in self.make_volumes(create_volume_func, articles):
         m = "Creating volume %d" % volume.number
         log.info(m)
         writeln(m).flush()
         file_name = self.make_aar(volume)
         self.file_names.append(file_name)
         m = "Wrote volume %d" % volume.number
         log.info(m)
         writeln(m).flush()
     self.article_store.close()
     self.write_volume_count()
     self.write_sha1sum()
     rename_files(self.file_names)
Esempio n. 2
0
 def compile(self):
     print_progress(self.stats)
     writeln()
     self.failed_articles.close()
     self.empty_articles.close()
     self.skipped_articles.close()
     writeln('Compiling .aar files')
     self.add_metadata("article_count", self.stats.articles)
     articles = self.article_store.sorted(
         key=lambda x: collation_key(x).getByteArray())
     log.info('Compiling %s', self.output_file_name)
     metadata = compress(tojson(self.metadata).encode('utf8'))
     header_meta_len = spec_len(HEADER_SPEC) + len(metadata)
     create_volume_func = functools.partial(self.create_volume,
                                            header_meta_len)
     for volume in self.make_volumes(create_volume_func, articles):
         m = "Creating volume %d" % volume.number
         log.info(m)
         writeln(m).flush()
         file_name = self.make_aar(volume)
         self.file_names.append(file_name)
         m = "Wrote volume %d" % volume.number
         log.info(m)
         writeln(m).flush()
     self.article_store.close()
     self.write_volume_count()
     self.write_sha1sum()
     rename_files(self.file_names)
Esempio n. 3
0
 def write_sha1sum(self):
     for file_name in self.file_names:
         msg = "Calculating checksum for %s" % file_name
         log.info(msg)
         offset = spec_len(HEADER_SPEC[:2])
         st_size = os.stat(file_name).st_size
         size = float(st_size - offset)
         for pos, sha1sum in calcsha1(file_name, offset):
             (display.erase_line().cr()
             .write(msg).write(': ').write('%.1f%%' % (100*pos/size)))
         sha1sum = sha1sum.hexdigest()
         msg = "%s sha1: %s" % (file_name, sha1sum)
         log.info(msg)
         display.erase_line().cr().writeln(msg)
         output_file = open(file_name, "r+b")
         output_file.seek(spec_len(HEADER_SPEC[:1]))
         output_file.write(sha1sum)
         output_file.close()
Esempio n. 4
0
 def write_sha1sum(self):
     for file_name in self.file_names:
         msg = "Calculating checksum for %s" % file_name
         log.info(msg)
         offset = spec_len(HEADER_SPEC[:2])
         st_size = os.stat(file_name).st_size
         size = float(st_size - offset)
         for pos, sha1sum in calcsha1(file_name, offset):
             (display.erase_line().cr().write(msg).write(': ').write(
                 '%.1f%%' % (100 * pos / size)))
         sha1sum = sha1sum.hexdigest()
         msg = "%s sha1: %s" % (file_name, sha1sum)
         log.info(msg)
         display.erase_line().cr().writeln(msg)
         output_file = open(file_name, "r+b")
         output_file.seek(spec_len(HEADER_SPEC[:1]))
         output_file.write(sha1sum)
         output_file.close()
Esempio n. 5
0
 def write_volume_count(self):
     _name, fmt = HEADER_SPEC[5]
     log.info("Writing volume count %d to all volumes as %s",
              Volume.number, fmt)
     log.debug('Writing' )
     for file_name in self.file_names:
         output_file = open(file_name, "r+b")
         output_file.seek(spec_len(HEADER_SPEC[:5]))
         output_file.write(struct.pack(fmt, Volume.number))
         output_file.close()
Esempio n. 6
0
 def write_volume_count(self):
     name, fmt = HEADER_SPEC[5]
     log.info("Writing volume count %d to all volumes as %s", Volume.number,
              fmt)
     log.debug('Writing')
     for file_name in self.file_names:
         output_file = open(file_name, "r+b")
         output_file.seek(spec_len(HEADER_SPEC[:5]))
         output_file.write(struct.pack(fmt, Volume.number))
         output_file.close()
Esempio n. 7
0
 def write_header(self, output_file, meta_length, index1Length,
                  index2Length, index_count, volume):
     article_offset = (spec_len(HEADER_SPEC) + meta_length +
                       index1Length + index2Length)
     values = dict(signature='aard',
                   sha1sum='0'*40,
                   version=1,
                   uuid=self.uuid.bytes,
                   volume=volume,
                   of=0,
                   total_volumes=0,
                   meta_length=meta_length,
                   index_count=index_count,
                   article_offset=article_offset,
                   index1_item_format=INDEX1_ITEM_FORMAT,
                   key_length_format=KEY_LENGTH_FORMAT,
                   article_length_format=ARTICLE_LENGTH_FORMAT)
     for name, fmt in HEADER_SPEC:
         output_file.write(struct.pack(fmt, values[name]))
Esempio n. 8
0
 def write_header(self, output_file, meta_length, index1Length,
                  index2Length, index_count, volume):
     article_offset = (spec_len(HEADER_SPEC) + meta_length + index1Length +
                       index2Length)
     values = dict(signature='aard',
                   sha1sum='0' * 40,
                   version=1,
                   uuid=self.uuid.bytes,
                   volume=volume,
                   of=0,
                   total_volumes=0,
                   meta_length=meta_length,
                   index_count=index_count,
                   article_offset=article_offset,
                   index1_item_format=INDEX1_ITEM_FORMAT,
                   key_length_format=KEY_LENGTH_FORMAT,
                   article_length_format=ARTICLE_LENGTH_FORMAT)
     for name, fmt in HEADER_SPEC:
         output_file.write(struct.pack(fmt, values[name]))
Esempio n. 9
0
 def create_volume(self):
     header_meta_len = spec_len(HEADER_SPEC) + len(self.serialized_metadata)
     return Volume(self.uuid,
                   header_meta_len,
                   self.max_file_size,
                   self.session_dir)
Esempio n. 10
0
 def create_volume(self):
     header_meta_len = spec_len(HEADER_SPEC) + len(self.serialized_metadata)
     return Volume(self.uuid, header_meta_len, self.max_file_size,
                   self.session_dir)