def build_mean_max_reducer(hidden_size: int) -> Model[Ragged, Floats2d]:
"""Reduce sequences by concatenating their mean and max pooled vectors,
and then combine the concatenated vectors with a hidden layer.
"""
return chain(
concatenate(reduce_last(), reduce_first(), reduce_mean(),
reduce_max()),
Maxout(nO=hidden_size, normalize=True, dropout=0.0),
)
def build_nel_encoder(tok2vec: Model, nO: Optional[int] = None) -> Model:
with Model.define_operators({">>": chain, "**": clone}):
token_width = tok2vec.get_dim("nO")
output_layer = Linear(nO=nO, nI=token_width)
model = (tok2vec >> list2ragged() >> reduce_mean() >> residual(
Maxout(nO=token_width, nI=token_width, nP=2, dropout=0.0)) >>
output_layer)
model.set_ref("output_layer", output_layer)
model.set_ref("tok2vec", tok2vec)
return model
def build_nel_encoder(tok2vec: Model,
nO: Optional[int] = None) -> Model[List[Doc], Floats2d]:
with Model.define_operators({">>": chain, "&": tuplify}):
token_width = tok2vec.maybe_get_dim("nO")
output_layer = Linear(nO=nO, nI=token_width)
model = (((tok2vec >> list2ragged()) & build_span_maker()) >>
extract_spans() >> reduce_mean() >> residual(
Maxout(nO=token_width, nI=token_width, nP=2,
dropout=0.0)) >> output_layer)
model.set_ref("output_layer", output_layer)
model.set_ref("tok2vec", tok2vec)
# flag to show this isn't legacy
model.attrs["include_span_maker"] = True
return model
def test_reduce_mean(Xs):
Xs = [x * 1000 for x in Xs] # use large numbers for numeric stability
model = reduce_mean()
lengths = model.ops.asarray([x.shape[0] for x in Xs], dtype="i")
X = Ragged(model.ops.flatten(Xs), lengths)
Y, backprop = model(X, is_train=True)
assert isinstance(Y, numpy.ndarray)
assert Y.shape == (len(Xs), Xs[0].shape[1])
assert Y.dtype == Xs[0].dtype
assert numpy.all(Y[0] == Y[0][0]) # all values in row should be equal
assert Y[0][0] == Xs[0].mean()
assert numpy.all(Y[1] == Y[1][0])
assert Y[1][0] == Xs[1].mean()
dX = backprop(Y)
assert dX.dataXd.shape == X.dataXd.shape
def build_simple_cnn_text_classifier(
tok2vec: Model,
exclusive_classes: bool,
nO: Optional[int] = None) -> Model[List[Doc], Floats2d]:
"""
Build a simple CNN text classifier, given a token-to-vector model as inputs.
If exclusive_classes=True, a softmax non-linearity is applied, so that the
outputs sum to 1. If exclusive_classes=False, a logistic non-linearity
is applied instead, so that outputs are in the range [0, 1].
"""
fill_defaults = {"b": 0, "W": 0}
with Model.define_operators({">>": chain}):
cnn = tok2vec >> list2ragged() >> reduce_mean()
nI = tok2vec.maybe_get_dim("nO")
if exclusive_classes:
output_layer = Softmax(nO=nO, nI=nI)
fill_defaults["b"] = NEG_VALUE
resizable_layer: Model = resizable(
output_layer,
resize_layer=partial(resize_linear_weighted,
fill_defaults=fill_defaults),
)
model = cnn >> resizable_layer
else:
output_layer = Linear(nO=nO, nI=nI)
resizable_layer = resizable(
output_layer,
resize_layer=partial(resize_linear_weighted,
fill_defaults=fill_defaults),
)
model = cnn >> resizable_layer >> Logistic()
model.set_ref("output_layer", output_layer)
model.attrs["resize_output"] = partial(
resize_and_set_ref,
resizable_layer=resizable_layer,
)
model.set_ref("tok2vec", tok2vec)
model.set_dim(
"nO", nO
) # type: ignore # TODO: remove type ignore once Thinc has been updated
model.attrs["multi_label"] = not exclusive_classes
return model
def test_spancat_model_forward_backward(nO=5):
tok2vec = build_Tok2Vec_model(**get_tok2vec_kwargs())
docs = get_docs()
spans_list = []
lengths = []
for doc in docs:
spans_list.append(doc[:2])
spans_list.append(doc[1:4])
lengths.append(2)
spans = Ragged(
tok2vec.ops.asarray([[s.start, s.end] for s in spans_list], dtype="i"),
tok2vec.ops.asarray(lengths, dtype="i"),
)
model = build_spancat_model(tok2vec, reduce_mean(),
chain(Relu(nO=nO),
Logistic())).initialize(X=(docs, spans))
Y, backprop = model((docs, spans), is_train=True)
assert Y.shape == (spans.dataXd.shape[0], nO)
backprop(Y)
def build_simple_cnn_text_classifier(
tok2vec: Model, exclusive_classes: bool, nO: Optional[int] = None
) -> Model[List[Doc], Floats2d]:
"""
Build a simple CNN text classifier, given a token-to-vector model as inputs.
If exclusive_classes=True, a softmax non-linearity is applied, so that the
outputs sum to 1. If exclusive_classes=False, a logistic non-linearity
is applied instead, so that outputs are in the range [0, 1].
"""
with Model.define_operators({">>": chain}):
cnn = tok2vec >> list2ragged() >> reduce_mean()
if exclusive_classes:
output_layer = Softmax(nO=nO, nI=tok2vec.maybe_get_dim("nO"))
model = cnn >> output_layer
model.set_ref("output_layer", output_layer)
else:
linear_layer = Linear(nO=nO, nI=tok2vec.maybe_get_dim("nO"))
model = cnn >> linear_layer >> Logistic()
model.set_ref("output_layer", linear_layer)
model.set_ref("tok2vec", tok2vec)
model.set_dim("nO", nO)
model.attrs["multi_label"] = not exclusive_classes
return model
def test_init_reduce_mean():
model = reduce_mean()
def test_spancat_model_init():
model = build_spancat_model(build_Tok2Vec_model(**get_tok2vec_kwargs()),
reduce_mean(), Logistic())
model.initialize()