def build_tagger( embed: Model[Array2d, Array2d], encode: Model[Padded, Padded], predict: Model[Array2d, Array2d], ) -> Model[List[Array2d], Padded]: model = chain( list2padded(), with_array(embed), encode, with_array(predict), ) model.set_ref("embed", embed) model.set_ref("encode", encode) model.set_ref("predict", model.layers[-1]) return model
def MaxoutWindowEncoder(width: int, window_size: int, maxout_pieces: int, depth: int) -> Model[List[Floats2d], List[Floats2d]]: """Encode context using convolutions with maxout activation, layer normalization and residual connections. width (int): The input and output width. These are required to be the same, to allow residual connections. This value will be determined by the width of the inputs. Recommended values are between 64 and 300. window_size (int): The number of words to concatenate around each token to construct the convolution. Recommended value is 1. maxout_pieces (int): The number of maxout pieces to use. Recommended values are 2 or 3. depth (int): The number of convolutional layers. Recommended value is 4. """ cnn = chain( expand_window(window_size=window_size), Maxout( nO=width, nI=width * ((window_size * 2) + 1), nP=maxout_pieces, dropout=0.0, normalize=True, ), ) model = clone(residual(cnn), depth) # type: ignore[arg-type] model.set_dim("nO", width) receptive_field = window_size * depth return with_array(model, pad=receptive_field) # type: ignore[arg-type]
def cnn_tagger(width: int, vector_width: int, nr_classes: int = 17): with Model.define_operators({">>": chain}): model = strings2arrays() >> with_array( HashEmbed(nO=width, nV=vector_width, column=0) >> expand_window( window_size=1) >> Relu(nO=width, nI=width * 3) >> Relu( nO=width, nI=width) >> Softmax(nO=nr_classes, nI=width)) return model
def noop_models(): return [ with_padded(noop()), with_array(noop()), with_array2d(noop()), with_list(noop()), with_ragged(noop()) ]
def get_array_model(): def _trim_array_forward(model, X, is_train): def backprop(dY): return model.ops.alloc2f(dY.shape[0], dY.shape[1] + 1) return X[:, :-1], backprop return with_array(Model("trimarray", _trim_array_forward))
def TransformersTagger( starter: str, n_tags: int = 17 ) -> Model[List[List[str]], List[Floats2d]]: return chain( TransformersTokenizer(starter), Transformer(starter), with_array(Softmax(nO=n_tags)), )
def test_pickle_with_flatten(linear): Xs = [linear.ops.alloc2f(2, 3), linear.ops.alloc2f(4, 3)] model = with_array(linear).initialize() pickled = srsly.pickle_dumps(model) loaded = srsly.pickle_loads(pickled) Ys = loaded.predict(Xs) assert len(Ys) == 2 assert Ys[0].shape == (Xs[0].shape[0], linear.get_dim("nO")) assert Ys[1].shape == (Xs[1].shape[0], linear.get_dim("nO"))
def Tok2Vec_v1( embed: Model[List[Doc], List[Floats2d]], encode: Model[Floats2d, Floats2d], ) -> Model[List[Doc], List[Floats2d]]: """Construct a tok2vec model out of embedding and encoding subnetworks. See https://explosion.ai/blog/deep-learning-formula-nlp embed (Model[List[Doc], List[Floats2d]]): Embed tokens into context-independent word vector representations. encode (Model[List[Floats2d], List[Floats2d]]): Encode context into the embeddings, using an architecture such as a CNN, BiLSTM or transformer. """ receptive_field = encode.attrs.get("receptive_field", 0) tok2vec = chain(embed, with_array(encode, pad=receptive_field)) tok2vec.set_dim("nO", encode.get_dim("nO")) tok2vec.set_ref("embed", embed) tok2vec.set_ref("encode", encode) return tok2vec
def build_tagger_model( tok2vec: Model[List[Doc], List[Floats2d]], nO: Optional[int] = None) -> Model[List[Doc], List[Floats2d]]: """Build a tagger model, using a provided token-to-vector component. The tagger model simply adds a linear layer with softmax activation to predict scores given the token vectors. tok2vec (Model[List[Doc], List[Floats2d]]): The token-to-vector subnetwork. nO (int or None): The number of tags to output. Inferred from the data if None. """ # TODO: glorot_uniform_init seems to work a bit better than zero_init here?! t2v_width = tok2vec.get_dim("nO") if tok2vec.has_dim("nO") else None output_layer = Softmax(nO, t2v_width, init_W=zero_init) softmax = with_array(output_layer) # type: ignore model = chain(tok2vec, softmax) model.set_ref("tok2vec", tok2vec) model.set_ref("softmax", output_layer) model.set_ref("output_layer", output_layer) return model
def MishWindowEncoder(width: int, window_size: int, depth: int) -> Model[List[Floats2d], List[Floats2d]]: """Encode context using convolutions with mish activation, layer normalization and residual connections. width (int): The input and output width. These are required to be the same, to allow residual connections. This value will be determined by the width of the inputs. Recommended values are between 64 and 300. window_size (int): The number of words to concatenate around each token to construct the convolution. Recommended value is 1. depth (int): The number of convolutional layers. Recommended value is 4. """ cnn = chain( expand_window(window_size=window_size), Mish(nO=width, nI=width * ((window_size * 2) + 1), dropout=0.0, normalize=True), ) model = clone(residual(cnn), depth) model.set_dim("nO", width) return with_array(model)
def create_embed_relu_relu_softmax(depth, width, vector_length): with Model.define_operators({">>": chain}): model = strings2arrays() >> with_array( HashEmbed(width, vector_length) >> expand_window(window_size=1) >> ReLu(width, width * 3) >> ReLu(width, width) >> Softmax(17, width)) return model
def CharacterEmbed( width: int, rows: int, nM: int, nC: int, include_static_vectors: bool, feature: Union[int, str] = "LOWER", ) -> Model[List[Doc], List[Floats2d]]: """Construct an embedded representation based on character embeddings, using a feed-forward network. A fixed number of UTF-8 byte characters are used for each word, taken from the beginning and end of the word equally. Padding is used in the centre for words that are too short. For instance, let's say nC=4, and the word is "jumping". The characters used will be jung (two from the start, two from the end). If we had nC=8, the characters would be "jumpping": 4 from the start, 4 from the end. This ensures that the final character is always in the last position, instead of being in an arbitrary position depending on the word length. The characters are embedded in a embedding table with a given number of rows, and the vectors concatenated. A hash-embedded vector of the LOWER of the word is also concatenated on, and the result is then passed through a feed-forward network to construct a single vector to represent the information. feature (int or str): An attribute to embed, to concatenate with the characters. width (int): The width of the output vector and the feature embedding. rows (int): The number of rows in the LOWER hash embedding table. nM (int): The dimensionality of the character embeddings. Recommended values are between 16 and 64. nC (int): The number of UTF-8 bytes to embed per word. Recommended values are between 3 and 8, although it may depend on the length of words in the language. include_static_vectors (bool): Whether to also use static word vectors. Requires a vectors table to be loaded in the Doc objects' vocab. """ feature = intify_attr(feature) if feature is None: raise ValueError(Errors.E911.format(feat=feature)) char_embed = chain( _character_embed.CharacterEmbed(nM=nM, nC=nC), cast(Model[List[Floats2d], Ragged], list2ragged()), ) feature_extractor: Model[List[Doc], Ragged] = chain( FeatureExtractor([feature]), cast(Model[List[Ints2d], Ragged], list2ragged()), with_array(HashEmbed(nO=width, nV=rows, column=0, seed=5)), # type: ignore ) max_out: Model[Ragged, Ragged] if include_static_vectors: max_out = with_array( Maxout(width, nM * nC + (2 * width), nP=3, normalize=True, dropout=0.0) # type: ignore ) model = chain( concatenate( char_embed, feature_extractor, StaticVectors(width, dropout=0.0), ), max_out, cast(Model[Ragged, List[Floats2d]], ragged2list()), ) else: max_out = with_array( Maxout(width, nM * nC + width, nP=3, normalize=True, dropout=0.0) # type: ignore ) model = chain( concatenate( char_embed, feature_extractor, ), max_out, cast(Model[Ragged, List[Floats2d]], ragged2list()), ) return model
def MultiHashEmbed( width: int, attrs: List[Union[str, int]], rows: List[int], include_static_vectors: bool, ) -> Model[List[Doc], List[Floats2d]]: """Construct an embedding layer that separately embeds a number of lexical attributes using hash embedding, concatenates the results, and passes it through a feed-forward subnetwork to build a mixed representation. The features used can be configured with the 'attrs' argument. The suggested attributes are NORM, PREFIX, SUFFIX and SHAPE. This lets the model take into account some subword information, without constructing a fully character-based representation. If pretrained vectors are available, they can be included in the representation as well, with the vectors table will be kept static (i.e. it's not updated). The `width` parameter specifies the output width of the layer and the widths of all embedding tables. If static vectors are included, a learned linear layer is used to map the vectors to the specified width before concatenating it with the other embedding outputs. A single Maxout layer is then used to reduce the concatenated vectors to the final width. The `rows` parameter controls the number of rows used by the `HashEmbed` tables. The HashEmbed layer needs surprisingly few rows, due to its use of the hashing trick. Generally between 2000 and 10000 rows is sufficient, even for very large vocabularies. A number of rows must be specified for each table, so the `rows` list must be of the same length as the `attrs` parameter. width (int): The output width. Also used as the width of the embedding tables. Recommended values are between 64 and 300. attrs (list of attr IDs): The token attributes to embed. A separate embedding table will be constructed for each attribute. rows (List[int]): The number of rows in the embedding tables. Must have the same length as attrs. include_static_vectors (bool): Whether to also use static word vectors. Requires a vectors table to be loaded in the Doc objects' vocab. """ if len(rows) != len(attrs): raise ValueError(f"Mismatched lengths: {len(rows)} vs {len(attrs)}") seed = 7 def make_hash_embed(index): nonlocal seed seed += 1 return HashEmbed(width, rows[index], column=index, seed=seed, dropout=0.0) embeddings = [make_hash_embed(i) for i in range(len(attrs))] concat_size = width * (len(embeddings) + include_static_vectors) max_out: Model[Ragged, Ragged] = with_array( Maxout(width, concat_size, nP=3, dropout=0.0, normalize=True) # type: ignore ) if include_static_vectors: feature_extractor: Model[List[Doc], Ragged] = chain( FeatureExtractor(attrs), cast(Model[List[Ints2d], Ragged], list2ragged()), with_array(concatenate(*embeddings)), ) model = chain( concatenate( feature_extractor, StaticVectors(width, dropout=0.0), ), max_out, cast(Model[Ragged, List[Floats2d]], ragged2list()), ) else: model = chain( FeatureExtractor(list(attrs)), cast(Model[List[Ints2d], Ragged], list2ragged()), with_array(concatenate(*embeddings)), max_out, cast(Model[Ragged, List[Floats2d]], ragged2list()), ) return model
def TextCatEnsemble_v1( width: int, embed_size: int, pretrained_vectors: Optional[bool], exclusive_classes: bool, ngram_size: int, window_size: int, conv_depth: int, dropout: Optional[float], nO: Optional[int] = None, ) -> Model: # Don't document this yet, I'm not sure it's right. cols = [ORTH, LOWER, PREFIX, SUFFIX, SHAPE, ID] with Model.define_operators({">>": chain, "|": concatenate, "**": clone}): lower = HashEmbed(nO=width, nV=embed_size, column=cols.index(LOWER), dropout=dropout, seed=10) prefix = HashEmbed( nO=width // 2, nV=embed_size, column=cols.index(PREFIX), dropout=dropout, seed=11, ) suffix = HashEmbed( nO=width // 2, nV=embed_size, column=cols.index(SUFFIX), dropout=dropout, seed=12, ) shape = HashEmbed( nO=width // 2, nV=embed_size, column=cols.index(SHAPE), dropout=dropout, seed=13, ) width_nI = sum( layer.get_dim("nO") for layer in [lower, prefix, suffix, shape]) trained_vectors = FeatureExtractor(cols) >> with_array( uniqued( (lower | prefix | suffix | shape) >> Maxout( nO=width, nI=width_nI, normalize=True), column=cols.index(ORTH), )) if pretrained_vectors: static_vectors = StaticVectors(width) vector_layer = trained_vectors | static_vectors vectors_width = width * 2 else: vector_layer = trained_vectors vectors_width = width tok2vec = vector_layer >> with_array( Maxout(width, vectors_width, normalize=True) >> residual((expand_window(window_size=window_size) >> Maxout( nO=width, nI=width * ((window_size * 2) + 1), normalize=True)))**conv_depth, pad=conv_depth, ) cnn_model = (tok2vec >> list2ragged() >> ParametricAttention(width) >> reduce_sum() >> residual(Maxout(nO=width, nI=width)) >> Linear(nO=nO, nI=width) >> Dropout(0.0)) linear_model = build_bow_text_classifier( nO=nO, ngram_size=ngram_size, exclusive_classes=exclusive_classes, no_output_layer=False, ) nO_double = nO * 2 if nO else None if exclusive_classes: output_layer = Softmax(nO=nO, nI=nO_double) else: output_layer = Linear(nO=nO, nI=nO_double) >> Dropout(0.0) >> Logistic() model = (linear_model | cnn_model) >> output_layer model.set_ref("tok2vec", tok2vec) if model.has_dim("nO") is not False: model.set_dim("nO", nO) model.set_ref("output_layer", linear_model.get_ref("output_layer")) model.attrs["multi_label"] = not exclusive_classes return model