def bytestring(self): buf = io.BytesIO() if isinstance(self.index, numbers.Integral): buf.write(encode_number_as_hex(self.index)) else: raw = bytearray(self.index.encode('ascii')) raw.insert(0, len(raw)) buf.write(bytes(raw)) et = self.entry_type buf.write(bytes(bytearray([et]))) if self.control_byte_count == 2: flags = 0 for attr in ('image_index', 'desc_offset', 'author_offset'): val = getattr(self, attr) if val is not None: tag = self.TAG_VALUES[attr] bm = TAGX.BITMASKS[tag] flags |= bm buf.write(bytes(bytearray([flags]))) for tag in self.tag_nums: attr = self.attr_for_tag(tag) val = getattr(self, attr) if isinstance(val, numbers.Integral): val = [val] for x in val: buf.write(encint(x)) if self.control_byte_count == 2: for attr in ('image_index', 'desc_offset', 'author_offset'): val = getattr(self, attr) if val is not None: buf.write(encint(val)) ans = buf.getvalue() return ans
def write_uncrossable_breaks(self): # {{{ ''' Write information about uncrossable breaks (non linear items in the spine. ''' if not WRITE_UNCROSSABLE_BREAKS: return breaks = self.serializer.breaks for i in xrange(1, self.last_text_record_idx+1): offset = i * RECORD_SIZE pbreak = 0 running = offset buf = StringIO() while breaks and (breaks[0] - offset) < RECORD_SIZE: pbreak = (breaks.pop(0) - running) >> 3 encoded = encint(pbreak) buf.write(encoded) running += pbreak << 3 encoded = encode_trailing_data(buf.getvalue()) self.records[i] += encoded
def write_uncrossable_breaks(self): # {{{ ''' Write information about uncrossable breaks (non linear items in the spine. ''' if not WRITE_UNCROSSABLE_BREAKS: return breaks = self.serializer.breaks for i in range(1, self.last_text_record_idx + 1): offset = i * RECORD_SIZE pbreak = 0 running = offset buf = StringIO() while breaks and (breaks[0] - offset) < RECORD_SIZE: pbreak = (breaks.pop(0) - running) >> 3 encoded = encint(pbreak) buf.write(encoded) running += pbreak << 3 encoded = encode_trailing_data(buf.getvalue()) self.records[i] += encoded
def __call__(self): self.control_bytes = self.calculate_control_bytes_for_each_entry( self.entries) index_blocks, idxt_blocks, record_counts, last_indices = [BytesIO()], [ BytesIO() ], [0], [b''] buf = BytesIO() RECORD_LIMIT = 0x10000 - self.HEADER_LENGTH - 1048 # kindlegen uses 1048 (there has to be some margin because of block alignment) for i, (index_num, tags) in enumerate(self.entries): control_bytes = self.control_bytes[i] buf.seek(0), buf.truncate(0) index_num = (index_num.encode('utf-8') if isinstance( index_num, unicode) else index_num) raw = bytearray(index_num) raw.insert(0, len(index_num)) buf.write(bytes(raw)) buf.write(bytes(bytearray(control_bytes))) for tag in self.tag_types: values = tags.get(tag.name, None) if values is None: continue try: len(values) except TypeError: values = [values] if values: for val in values: try: buf.write(encint(val)) except ValueError: raise ValueError('Invalid values for %r: %r' % (tag, values)) raw = buf.getvalue() offset = index_blocks[-1].tell() idxt_pos = idxt_blocks[-1].tell() if offset + idxt_pos + len(raw) + 2 > RECORD_LIMIT: index_blocks.append(BytesIO()) idxt_blocks.append(BytesIO()) record_counts.append(0) offset = idxt_pos = 0 last_indices.append(b'') record_counts[-1] += 1 idxt_blocks[-1].write(pack(b'>H', self.HEADER_LENGTH + offset)) index_blocks[-1].write(raw) last_indices[-1] = index_num index_records = [] for index_block, idxt_block, record_count in zip( index_blocks, idxt_blocks, record_counts): index_block = align_block(index_block.getvalue()) idxt_block = align_block(b'IDXT' + idxt_block.getvalue()) # Create header for this index record header = b'INDX' buf.seek(0), buf.truncate(0) buf.write(pack(b'>I', self.HEADER_LENGTH)) buf.write(b'\0' * 4) # Unknown buf.write( pack(b'>I', 1) ) # Header type (0 for Index header record and 1 for Index records) buf.write(b'\0' * 4) # Unknown # IDXT block offset buf.write(pack(b'>I', self.HEADER_LENGTH + len(index_block))) # Number of index entries in this record buf.write(pack(b'>I', record_count)) buf.write(b'\xff' * 8) # Unknown buf.write(b'\0' * 156) # Unknown header += buf.getvalue() index_records.append(header + index_block + idxt_block) if len(index_records[-1]) > 0x10000: raise ValueError( 'Failed to rollover index blocks for very large index.') # Create the Index Header record tagx = self.generate_tagx() # Geometry of the index records is written as index entries pointed to # by the IDXT records buf.seek(0), buf.truncate() idxt = [b'IDXT'] pos = IndexHeader.HEADER_LENGTH + len(tagx) for last_idx, num in zip(last_indices, record_counts): start = buf.tell() idxt.append(pack(b'>H', pos)) buf.write(bytes(bytearray([len(last_idx)])) + last_idx) buf.write(pack(b'>H', num)) pos += buf.tell() - start header = { 'num_of_entries': sum(r for r in record_counts), 'num_of_records': len(index_records), 'num_of_cncx': len(self.cncx), 'tagx': align_block(tagx), 'geometry': align_block(buf.getvalue()), 'idxt': align_block(b''.join(idxt)), } header = IndexHeader()(**header) self.records = [header] + index_records self.records.extend(self.cncx.records) return self.records
def __call__(self): self.control_bytes = self.calculate_control_bytes_for_each_entry( self.entries) rendered_entries = [] index, idxt, buf = BytesIO(), BytesIO(), BytesIO() IndexEntry = namedtuple('IndexEntry', 'offset length raw') last_lead_text = b'' too_large = ValueError('Index has too many entries, calibre does not' ' support generating multiple index records at this' ' time.') for i, x in enumerate(self.entries): control_bytes = self.control_bytes[i] leading_text, tags = x buf.seek(0), buf.truncate(0) leading_text = (leading_text.encode('utf-8') if isinstance(leading_text, unicode) else leading_text) raw = bytearray(leading_text) raw.insert(0, len(leading_text)) buf.write(bytes(raw)) buf.write(bytes(bytearray(control_bytes))) for tag in self.tag_types: values = tags.get(tag.name, None) if values is None: continue try: len(values) except TypeError: values = [values] if values: for val in values: try: buf.write(encint(val)) except ValueError: raise ValueError('Invalid values for %r: %r'%( tag, values)) raw = buf.getvalue() offset = index.tell() if offset + self.HEADER_LENGTH >= 0x10000: raise too_large rendered_entries.append(IndexEntry(offset, len(raw), raw)) idxt.write(pack(b'>H', self.HEADER_LENGTH+offset)) index.write(raw) last_lead_text = leading_text index_block = align_block(index.getvalue()) idxt_block = align_block(b'IDXT' + idxt.getvalue()) body = index_block + idxt_block if len(body) + self.HEADER_LENGTH >= 0x10000: raise too_large header = b'INDX' buf.seek(0), buf.truncate(0) buf.write(pack(b'>I', self.HEADER_LENGTH)) buf.write(b'\0'*4) # Unknown buf.write(pack(b'>I', 1)) # Header type? Or index record number? buf.write(b'\0'*4) # Unknown # IDXT block offset buf.write(pack(b'>I', self.HEADER_LENGTH + len(index_block))) # Number of index entries buf.write(pack(b'>I', len(rendered_entries))) buf.write(b'\xff'*8) # Unknown buf.write(b'\0'*156) # Unknown header += buf.getvalue() index_record = header + body tagx = self.generate_tagx() idxt = (b'IDXT' + pack(b'>H', IndexHeader.HEADER_LENGTH + len(tagx)) + b'\0') # Last index idx = bytes(bytearray([len(last_lead_text)])) + last_lead_text idx += pack(b'>H', len(rendered_entries)) header = { 'num_of_entries': len(rendered_entries), 'num_of_cncx': len(self.cncx), 'tagx':tagx, 'last_index':align_block(idx), 'idxt':idxt } header = IndexHeader()(**header) self.records = [header, index_record] self.records.extend(self.cncx.records) return self.records
def __call__(self): self.control_bytes = self.calculate_control_bytes_for_each_entry( self.entries) index_blocks, idxt_blocks, record_counts, last_indices = [BytesIO()], [BytesIO()], [0], [b''] buf = BytesIO() RECORD_LIMIT = 0x10000 - self.HEADER_LENGTH - 1048 # kindlegen uses 1048 (there has to be some margin because of block alignment) for i, (index_num, tags) in enumerate(self.entries): control_bytes = self.control_bytes[i] buf.seek(0), buf.truncate(0) index_num = (index_num.encode('utf-8') if isinstance(index_num, unicode) else index_num) raw = bytearray(index_num) raw.insert(0, len(index_num)) buf.write(bytes(raw)) buf.write(bytes(bytearray(control_bytes))) for tag in self.tag_types: values = tags.get(tag.name, None) if values is None: continue try: len(values) except TypeError: values = [values] if values: for val in values: try: buf.write(encint(val)) except ValueError: raise ValueError('Invalid values for %r: %r'%( tag, values)) raw = buf.getvalue() offset = index_blocks[-1].tell() idxt_pos = idxt_blocks[-1].tell() if offset + idxt_pos + len(raw) + 2 > RECORD_LIMIT: index_blocks.append(BytesIO()) idxt_blocks.append(BytesIO()) record_counts.append(0) offset = idxt_pos = 0 last_indices.append(b'') record_counts[-1] += 1 idxt_blocks[-1].write(pack(b'>H', self.HEADER_LENGTH+offset)) index_blocks[-1].write(raw) last_indices[-1] = index_num index_records = [] for index_block, idxt_block, record_count in zip(index_blocks, idxt_blocks, record_counts): index_block = align_block(index_block.getvalue()) idxt_block = align_block(b'IDXT' + idxt_block.getvalue()) # Create header for this index record header = b'INDX' buf.seek(0), buf.truncate(0) buf.write(pack(b'>I', self.HEADER_LENGTH)) buf.write(b'\0'*4) # Unknown buf.write(pack(b'>I', 1)) # Header type (0 for Index header record and 1 for Index records) buf.write(b'\0'*4) # Unknown # IDXT block offset buf.write(pack(b'>I', self.HEADER_LENGTH + len(index_block))) # Number of index entries in this record buf.write(pack(b'>I', record_count)) buf.write(b'\xff'*8) # Unknown buf.write(b'\0'*156) # Unknown header += buf.getvalue() index_records.append(header + index_block + idxt_block) if len(index_records[-1]) > 0x10000: raise ValueError('Failed to rollover index blocks for very large index.') # Create the Index Header record tagx = self.generate_tagx() # Geometry of the index records is written as index entries pointed to # by the IDXT records buf.seek(0), buf.truncate() idxt = [b'IDXT'] pos = IndexHeader.HEADER_LENGTH + len(tagx) for last_idx, num in zip(last_indices, record_counts): start = buf.tell() idxt.append(pack(b'>H', pos)) buf.write(bytes(bytearray([len(last_idx)])) + last_idx) buf.write(pack(b'>H', num)) pos += buf.tell() - start header = { 'num_of_entries': sum(r for r in record_counts), 'num_of_records': len(index_records), 'num_of_cncx': len(self.cncx), 'tagx':align_block(tagx), 'geometry':align_block(buf.getvalue()), 'idxt':align_block(b''.join(idxt)), } header = IndexHeader()(**header) self.records = [header] + index_records self.records.extend(self.cncx.records) return self.records
def __call__(self): self.control_bytes = self.calculate_control_bytes_for_each_entry( self.entries) rendered_entries = [] index, idxt, buf = BytesIO(), BytesIO(), BytesIO() IndexEntry = namedtuple('IndexEntry', 'offset length raw') last_lead_text = b'' too_large = ValueError( 'Index has too many entries, calibre does not' ' support generating multiple index records at this' ' time.') for i, x in enumerate(self.entries): control_bytes = self.control_bytes[i] leading_text, tags = x buf.seek(0), buf.truncate(0) leading_text = (leading_text.encode('utf-8') if isinstance( leading_text, unicode) else leading_text) raw = bytearray(leading_text) raw.insert(0, len(leading_text)) buf.write(bytes(raw)) buf.write(bytes(bytearray(control_bytes))) for tag in self.tag_types: values = tags.get(tag.name, None) if values is None: continue try: len(values) except TypeError: values = [values] if values: for val in values: try: buf.write(encint(val)) except ValueError: raise ValueError('Invalid values for %r: %r' % (tag, values)) raw = buf.getvalue() offset = index.tell() if offset + self.HEADER_LENGTH >= 0x10000: raise too_large rendered_entries.append(IndexEntry(offset, len(raw), raw)) idxt.write(pack(b'>H', self.HEADER_LENGTH + offset)) index.write(raw) last_lead_text = leading_text index_block = align_block(index.getvalue()) idxt_block = align_block(b'IDXT' + idxt.getvalue()) body = index_block + idxt_block if len(body) + self.HEADER_LENGTH >= 0x10000: raise too_large header = b'INDX' buf.seek(0), buf.truncate(0) buf.write(pack(b'>I', self.HEADER_LENGTH)) buf.write(b'\0' * 4) # Unknown buf.write(pack(b'>I', 1)) # Header type? Or index record number? buf.write(b'\0' * 4) # Unknown # IDXT block offset buf.write(pack(b'>I', self.HEADER_LENGTH + len(index_block))) # Number of index entries buf.write(pack(b'>I', len(rendered_entries))) buf.write(b'\xff' * 8) # Unknown buf.write(b'\0' * 156) # Unknown header += buf.getvalue() index_record = header + body tagx = self.generate_tagx() idxt = (b'IDXT' + pack(b'>H', IndexHeader.HEADER_LENGTH + len(tagx)) + b'\0') # Last index idx = bytes(bytearray([len(last_lead_text)])) + last_lead_text idx += pack(b'>H', len(rendered_entries)) header = { 'num_of_entries': len(rendered_entries), 'num_of_cncx': len(self.cncx), 'tagx': tagx, 'last_index': align_block(idx), 'idxt': idxt } header = IndexHeader()(**header) self.records = [header, index_record] self.records.extend(self.cncx.records) return self.records