def _read_array_header(file: BinaryIO) -> Tuple[int, int]: """ Helper method to read the header of an NdArray chunk. The method reads the shape tuple, verifies the TypeId and seeks the file to the start of the array. The shape tuple is returned. Parameters ---------- file : BinaryIO finalfusion file with a storage at the start of a NdArray chunk. Returns ------- shape : Tuple[int, int] Shape of the storage. Raises ------ FinalfusionFormatError If the TypeId does not match TypeId.f32 """ rows, cols = _read_required_binary(file, "<QI") type_id = TypeId(_read_required_binary(file, "<I")[0]) if TypeId.f32 != type_id: raise FinalfusionFormatError( f"Invalid Type, expected {TypeId.f32}, got {type_id}") file.seek(_pad_float32(file.tell()), 1) return rows, cols
def _read_items_with_indices(file: BinaryIO, length: int) -> Tuple[List[str], Dict[str, int]]: """ Helper method to read items from a vocabulary chunk. Parameters ---------- file : BinaryIO input file length : int number of items to read Returns ------- words : List[str] The word list """ items = [] index = dict() for _ in range(length): item_length = _read_required_binary(file, "<I")[0] item = file.read(item_length).decode("utf-8") idx = _read_required_binary(file, "<Q")[0] items.append(item) index[item] = idx return items, index
def read_chunk(file: BinaryIO) -> 'ExplicitVocab': length, ngram_length, min_n, max_n = _read_required_binary( file, "<QQII") words = _read_items(file, length) ngram_list, ngram_index = _read_items_with_indices(file, ngram_length) indexer = ExplicitIndexer(ngram_list, min_n, max_n, ngram_index) return ExplicitVocab(words, indexer)
def _read_ft_header(file: BinaryIO): """ Helper method to verify version and magic. """ magic, version = _read_required_binary(file, "<ii") if magic != _FT_MAGIC: raise ValueError(f"Magic should be 793_712_314, not: {magic}") if version != 12: raise ValueError(f"Expected version 12, not: {version}")
def _read_ft_cfg(file: BinaryIO) -> Metadata: """ Constructs metadata from fastText config. """ cfg = list(_read_required_binary(file, "<12id")) losses = ['HierarchicalSoftmax', 'NegativeSampling', 'Softmax'] cfg[6] = losses[cfg[6] - 1] models = ['CBOW', 'SkipGram', 'Supervised'] cfg[7] = models[cfg[7] - 1] return Metadata(dict(zip(_FT_REQUIRED_CFG_KEYS, cfg)))
def _read_binary_word(file: BinaryIO, lossy: bool) -> str: """ Helper method to read null-terminated binary strings. """ word = bytearray() while True: byte = file.read(1) if byte == b'\x00': break if byte == b'': raise EOFError word.extend(byte) # discard frequency _ = _read_required_binary(file, "<q") entry_type = _read_required_binary(file, "b")[0] if entry_type != 0: raise ValueError(f'Non word entry: {word}') return word.decode('utf8', errors='replace' if lossy else 'strict')
def _read_ft_storage(file: BinaryIO, vocab: Vocab) -> NdArray: """ Helper method to read fastText storage. If vocab is a SimpleVocab, the matrix is read and returned as is. If vocab is a FastTextVocab, the word representations are precomputed based on the vocab. """ quantized = _read_required_binary(file, "?")[0] if quantized: raise NotImplementedError( "Quantized storage is not supported for fastText models") rows, cols = _read_required_binary(file, "<qq") matrix = np.fromfile(file=file, count=rows * cols, dtype=np.float32).reshape((rows, cols)) if sys.byteorder == 'big': matrix.byteswap(inplace=True) if isinstance(vocab, FastTextVocab): _precompute_word_vecs(vocab, matrix) return NdArray(matrix)
def _read_binary_word(file: BinaryIO) -> str: """ Helper method to read null-terminated binary strings. """ word = bytearray() while True: byte = file.read(1) if byte == b'\x00': break if byte == b'': raise EOFError word.extend(byte) # discard frequency _ = _read_required_binary(file, "<q") entry_type = _read_required_binary(file, "b")[0] if entry_type != 0: raise ValueError(f'Non word entry: {word}') # pylint: disable=fixme # XXX handle unicode errors return word.decode("utf8")
def read_chunk(file: BinaryIO) -> 'Norms': n_norms, dtype = _read_required_binary(file, "<QI") type_id = TypeId(dtype) if TypeId.f32 != type_id: raise FinalfusionFormatError( f"Invalid Type, expected {TypeId.f32}, got {str(type_id)}") padding = _pad_float32(file.tell()) file.seek(padding, 1) array = np.fromfile(file=file, count=n_norms, dtype=np.float32) if sys.byteorder == "big": array.byteswap(inplace=True) return Norms(array)
def read_chunk(file: BinaryIO) -> 'Metadata': chunk_header_size = struct.calcsize("<IQ") # place the file before the chunk header since the chunk size for # metadata the number of bytes that we need to read file.seek(-chunk_header_size, 1) chunk_id, chunk_len = _read_required_binary(file, "<IQ") assert ChunkIdentifier(chunk_id) == Metadata.chunk_identifier() buf = file.read(chunk_len) if len(buf) != chunk_len: raise FinalfusionFormatError( f'Could not read {chunk_len} bytes from file') return Metadata(toml.loads(buf.decode("utf-8")))
def _read_ft_vocab(file: BinaryIO, buckets: int, min_n: int, max_n: int) -> Union[FastTextVocab, SimpleVocab]: """ Helper method to read a vocab from a fastText file Returns a SimpleVocab if min_n is 0, otherwise FastTextVocab is returned. """ # discard n_words vocab_size, _n_words, n_labels = _read_required_binary(file, "<iii") if n_labels: raise NotImplementedError( "fastText prediction models are not supported") # discard n_tokens _read_required_binary(file, "<q") prune_idx_size = _read_required_binary(file, "<q")[0] if prune_idx_size > 0: raise NotImplementedError("Pruned vocabs are not supported") if min_n: return _read_ft_subwordvocab(file, buckets, min_n, max_n, vocab_size) return SimpleVocab([_read_binary_word(file) for _ in range(vocab_size)])
def _read_ft_vocab(file: BinaryIO, buckets: int, min_n: int, max_n: int, lossy: bool) -> Union[FastTextVocab, SimpleVocab]: """ Helper method to read a vocab from a fastText file Returns a FastTextVocab. """ # discard n_words vocab_size, _n_words, n_labels = _read_required_binary(file, "<iii") if n_labels: raise NotImplementedError( "fastText prediction models are not supported") # discard n_tokens _read_required_binary(file, "<q") prune_idx_size = _read_required_binary(file, "<q")[0] if prune_idx_size >= 0: raise NotImplementedError("Pruned vocabs are not supported") words = [_read_binary_word(file, lossy) for _ in range(vocab_size)] indexer = FastTextIndexer(buckets, min_n, max_n) return FastTextVocab(words, indexer)
def _read_quantized_header( file: BinaryIO ) -> Tuple[PQ, Tuple[int, int], Optional[np.ndarray]]: """ Helper method to read the header of a quantized array chunk. Returns a tuple containing PQ, quantized_shape and optional norms. """ projection = _read_required_binary(file, '<I')[0] != 0 read_norms = _read_required_binary(file, '<I')[0] != 0 quantized_len = _read_required_binary(file, '<I')[0] reconstructed_len = _read_required_binary(file, '<I')[0] n_centroids = _read_required_binary(file, '<I')[0] n_embeddings = _read_required_binary(file, '<Q')[0] assert reconstructed_len % quantized_len == 0 type_id = _read_required_binary(file, '<I')[0] if int(TypeId.u8) != type_id: raise FinalfusionFormatError( f"Invalid Type, expected {str(TypeId.u8)}, got {type_id}") type_id = _read_required_binary(file, '<I')[0] if int(TypeId.f32) != type_id: raise FinalfusionFormatError( f"Invalid Type, expected {str(TypeId.f32)}, got {type_id}") file.seek(_pad_float32(file.tell()), 1) if projection: projection = _read_array_as_native(file, np.float32, reconstructed_len**2) projection_shape = (reconstructed_len, reconstructed_len) projection = projection.reshape(projection_shape) else: projection = None quantizer_shape = (quantized_len, n_centroids, reconstructed_len // quantized_len) quantizers_size = quantized_len * n_centroids * (reconstructed_len // quantized_len) quantizers = _read_array_as_native(file, np.float32, quantizers_size) quantizers = quantizers.reshape(quantizer_shape) if read_norms: norms = _read_array_as_native(file, np.float32, n_embeddings) else: norms = None quantizer = PQ(quantizers, projection) return quantizer, (n_embeddings, quantized_len), norms
def _read_items(file: BinaryIO, length: int) -> List[str]: """ Helper method to read items from a vocabulary chunk. Parameters ---------- file : BinaryIO input file length : int number of items to read Returns ------- words : List[str] The word list """ items = [] for _ in range(length): item_length = _read_required_binary(file, "<I")[0] word = file.read(item_length).decode("utf-8") items.append(word) return items
def read_chunk(file: BinaryIO) -> 'FastTextVocab': length, min_n, max_n, buckets = _read_required_binary(file, "<QIII") words = _read_items(file, length) indexer = FastTextIndexer(buckets, min_n, max_n) return FastTextVocab(words, indexer)
def read_chunk(file: BinaryIO) -> 'SimpleVocab': length = _read_required_binary(file, "<Q")[0] words = _read_items(file, length) return SimpleVocab(words)