def forward(model: Model, docs: List[Doc], is_train: bool):
if docs is None:
return []
ids = []
output = []
E = model.get_param("E")
nC = model.get_dim("nC")
nM = model.get_dim("nM")
nO = model.get_dim("nO")
# This assists in indexing; it's like looping over this dimension.
# Still consider this weird witch craft...But thanks to Mark Neumann
# for the tip.
nCv = model.ops.xp.arange(nC)
for doc in docs:
doc_ids = model.ops.asarray(doc.to_utf8_array(nr_char=nC))
doc_vectors = model.ops.alloc3f(len(doc), nC, nM)
# Let's say I have a 2d array of indices, and a 3d table of data. What numpy
# incantation do I chant to get
# output[i, j, k] == data[j, ids[i, j], k]?
doc_vectors[:, nCv] = E[nCv, doc_ids[:, nCv]]
output.append(doc_vectors.reshape((len(doc), nO)))
ids.append(doc_ids)
def backprop(d_output):
dE = model.ops.alloc(E.shape, dtype=E.dtype)
for doc_ids, d_doc_vectors in zip(ids, d_output):
d_doc_vectors = d_doc_vectors.reshape((len(doc_ids), nC, nM))
dE[nCv, doc_ids[:, nCv]] += d_doc_vectors[:, nCv]
model.inc_grad("E", dE)
return []
return output, backprop
def build_nel_encoder(tok2vec: Model, nO: Optional[int] = None) -> Model:
with Model.define_operators({">>": chain, "**": clone}):
token_width = tok2vec.get_dim("nO")
output_layer = Linear(nO=nO, nI=token_width)
model = (tok2vec >> list2ragged() >> reduce_mean() >> residual(
Maxout(nO=token_width, nI=token_width, nP=2, dropout=0.0)) >>
output_layer)
model.set_ref("output_layer", output_layer)
model.set_ref("tok2vec", tok2vec)
return model
def build_cloze_multi_task_model(
vocab: "Vocab", tok2vec: Model, maxout_pieces: int, hidden_size: int
) -> Model:
nO = vocab.vectors.data.shape[1]
output_layer = chain(
list2array(),
Maxout(
nO=hidden_size,
nI=tok2vec.get_dim("nO"),
nP=maxout_pieces,
normalize=True,
dropout=0.0,
),
Linear(nO=nO, nI=hidden_size, init_W=zero_init),
)
model = chain(tok2vec, output_layer)
model = build_masked_language_model(vocab, model)
model.set_ref("tok2vec", tok2vec)
model.set_ref("output_layer", output_layer)
return model
def test_model_set_dim():
class MyShim(Shim):
name = "testshim"
model_a = create_model("a")
model = Model(
"test",
lambda X: (X, lambda dY: dY),
dims={
"nI": 5,
"nO": None
},
params={
"W": None,
"b": None
},
refs={
"a": model_a,
"b": None
},
attrs={"foo": "bar"},
shims=[MyShim(None)],
layers=[model_a, model_a],
)
with pytest.raises(ValueError):
model.set_dim("nI", 10)
# force can be used before any parameters are set
model.set_dim("nI", 10, force=True)
model.set_param("W", model.ops.alloc1f(10))
model.set_grad("W", model.ops.alloc1f(10))
assert model.has_dim("nI")
assert model.get_dim("nI") == 10
with pytest.raises(KeyError):
model.set_dim("xyz", 20)
with pytest.raises(ValueError):
model.set_dim("nI", 20)
# force can't be used after any parameter is set
with pytest.raises(ValueError):
model.set_dim("nI", 20, force=True)
def test_model_init():
class MyShim(Shim):
name = "testshim"
model_a = create_model("a")
model = Model(
"test",
lambda X: (X, lambda dY: dY),
dims={
"nI": 10,
"nO": None
},
params={
"W": numpy.zeros((10, )),
"b": None
},
refs={
"a": model_a,
"b": None
},
attrs={"foo": "bar"},
shims=[MyShim(None)],
layers=[model_a, model_a],
)
assert model.has_param("W")
assert model.get_param("W").shape == (10, )
assert model.has_param("b") is None
with pytest.raises(KeyError):
model.get_param("b")
with pytest.raises(KeyError):
model.get_param("X")
model.set_param("X", numpy.zeros((10, )))
assert model.has_param("X")
assert model.get_param("X").shape == (10, )
with model.use_params({(model.id, "X"): numpy.ones((10, ))}):
assert numpy.array_equal(model.get_param("X"), numpy.ones((10, )))
assert numpy.array_equal(model.get_param("X"), numpy.zeros((10, )))
assert not model.has_grad("W")
assert not model.has_grad("xyz")
with pytest.raises(KeyError):
model.get_grad("b")
model.set_param("W", model.ops.alloc1f(10))
model.set_grad("W", model.ops.alloc1f(10))
with pytest.raises(ValueError):
model.inc_grad("W", numpy.zeros((5, 0)))
assert model.has_dim("nI")
assert model.get_dim("nI") == 10
with pytest.raises(KeyError):
model.get_dim("xyz")
with pytest.raises(ValueError):
model.get_dim("nO")
with pytest.raises(KeyError):
model.set_dim("xyz", 20)
with pytest.raises(ValueError):
model.set_dim("nI", 20)
assert model.has_ref("a")
assert model.get_ref("a").name == "a"
assert not model.has_ref("xyz")
with pytest.raises(KeyError):
model.get_ref("xyz")
assert model.has_ref("b") is None
with pytest.raises(ValueError):
model.get_ref("b")
model.set_ref("c", model_a)
assert model.has_ref("c")
assert model.get_ref("c").name == "a"
with pytest.raises(ValueError):
model.set_ref("c", create_model("c"))
assert "foo" in model.attrs
assert "bar" not in model.attrs
assert model.attrs["foo"] == "bar"
with pytest.raises(KeyError):
model.attrs["bar"]
model.attrs["bar"] = "baz"
model_copy = model.copy()
assert model_copy.name == "test"
def init(model: Model, X=None, Y=None):
vectors_table = model.ops.alloc3f(model.get_dim("nC"), model.get_dim("nV"),
model.get_dim("nM"))
model.set_param("E", vectors_table)