def forward(model: Model, docs: List[Doc], is_train: bool):
if docs is None:
return []
ids = []
output = []
E = model.get_param("E")
nC = model.get_dim("nC")
nM = model.get_dim("nM")
nO = model.get_dim("nO")
# This assists in indexing; it's like looping over this dimension.
# Still consider this weird witch craft...But thanks to Mark Neumann
# for the tip.
nCv = model.ops.xp.arange(nC)
for doc in docs:
doc_ids = model.ops.asarray(doc.to_utf8_array(nr_char=nC))
doc_vectors = model.ops.alloc3f(len(doc), nC, nM)
# Let's say I have a 2d array of indices, and a 3d table of data. What numpy
# incantation do I chant to get
# output[i, j, k] == data[j, ids[i, j], k]?
doc_vectors[:, nCv] = E[nCv, doc_ids[:, nCv]]
output.append(doc_vectors.reshape((len(doc), nO)))
ids.append(doc_ids)
def backprop(d_output):
dE = model.ops.alloc(E.shape, dtype=E.dtype)
for doc_ids, d_doc_vectors in zip(ids, d_output):
d_doc_vectors = d_doc_vectors.reshape((len(doc_ids), nC, nM))
dE[nCv, doc_ids[:, nCv]] += d_doc_vectors[:, nCv]
model.inc_grad("E", dE)
return []
return output, backprop
def torch_softmax_with_temperature(
model: Model, X: Floats2d,
targets: Ints1d) -> Tuple[Floats2d, Floats2d]:
import torch
Wt = xp2torch(model.get_param("W"))
bt = xp2torch(model.get_param("b"))
temperature = model.attrs["softmax_temperature"]
Xt = xp2torch(X, requires_grad=True)
Yt_gold = xp2torch(targets).long()
XWbt = (Xt @ Wt) + bt
XWbt_temp = XWbt / temperature
loss = torch.nn.CrossEntropyLoss()
output = loss(XWbt_temp, Yt_gold)
output.backward()
return cast(Floats2d,
torch2xp(torch.nn.functional.softmax(XWbt_temp,
dim=-1))), cast(
Floats2d,
torch2xp(Xt.grad))
def test_model_init():
class MyShim(Shim):
name = "testshim"
model_a = create_model("a")
model = Model(
"test",
lambda X: (X, lambda dY: dY),
dims={
"nI": 10,
"nO": None
},
params={
"W": numpy.zeros((10, )),
"b": None
},
refs={
"a": model_a,
"b": None
},
attrs={"foo": "bar"},
shims=[MyShim(None)],
layers=[model_a, model_a],
)
assert model.has_param("W")
assert model.get_param("W").shape == (10, )
assert model.has_param("b") is None
with pytest.raises(KeyError):
model.get_param("b")
with pytest.raises(KeyError):
model.get_param("X")
model.set_param("X", numpy.zeros((10, )))
assert model.has_param("X")
assert model.get_param("X").shape == (10, )
with model.use_params({(model.id, "X"): numpy.ones((10, ))}):
assert numpy.array_equal(model.get_param("X"), numpy.ones((10, )))
assert numpy.array_equal(model.get_param("X"), numpy.zeros((10, )))
assert not model.has_grad("W")
assert not model.has_grad("xyz")
with pytest.raises(KeyError):
model.get_grad("b")
model.set_param("W", model.ops.alloc1f(10))
model.set_grad("W", model.ops.alloc1f(10))
with pytest.raises(ValueError):
model.inc_grad("W", numpy.zeros((5, 0)))
assert model.has_dim("nI")
assert model.get_dim("nI") == 10
with pytest.raises(KeyError):
model.get_dim("xyz")
with pytest.raises(ValueError):
model.get_dim("nO")
with pytest.raises(KeyError):
model.set_dim("xyz", 20)
with pytest.raises(ValueError):
model.set_dim("nI", 20)
assert model.has_ref("a")
assert model.get_ref("a").name == "a"
assert not model.has_ref("xyz")
with pytest.raises(KeyError):
model.get_ref("xyz")
assert model.has_ref("b") is None
with pytest.raises(ValueError):
model.get_ref("b")
model.set_ref("c", model_a)
assert model.has_ref("c")
assert model.get_ref("c").name == "a"
with pytest.raises(ValueError):
model.set_ref("c", create_model("c"))
assert "foo" in model.attrs
assert "bar" not in model.attrs
assert model.attrs["foo"] == "bar"
with pytest.raises(KeyError):
model.attrs["bar"]
model.attrs["bar"] = "baz"
model_copy = model.copy()
assert model_copy.name == "test"