def init(model: Model, X=None, Y=None): if model.attrs["has_transformer"]: return name = model.attrs["name"] tok_cfg = model._init_tokenizer_config trf_cfg = model._init_transformer_config tokenizer, transformer = huggingface_from_pretrained( name, tok_cfg, trf_cfg) model.attrs["set_transformer"](model, transformer, tokenizer) tokenizer = model.tokenizer # Call the model with a batch of inputs to infer the width if X: # If we're dealing with actual texts, do the work to setup the wordpieces # batch properly docs = X get_spans = model.attrs["get_spans"] nested_spans = get_spans(docs) flat_spans = [] for doc_spans in nested_spans: flat_spans.extend(doc_spans) token_data = huggingface_tokenize(tokenizer, [span.text for span in flat_spans]) wordpieces = WordpieceBatch.from_batch_encoding(token_data) align = get_alignment(flat_spans, wordpieces.strings, tokenizer.all_special_tokens) wordpieces, align = truncate_oversize_splits( wordpieces, align, tokenizer.model_max_length) else: texts = ["hello world", "foo bar"] token_data = huggingface_tokenize(tokenizer, texts) wordpieces = WordpieceBatch.from_batch_encoding(token_data) model.layers[0].initialize(X=wordpieces) model_output = model.layers[0].predict(wordpieces) model.set_dim("nO", model_output.last_hidden_state.shape[-1])
def init(model: Model, X=None, Y=None): if model.attrs["has_transformer"]: return name = model.attrs["name"] tok_cfg = model.attrs["tokenizer_config"] tokenizer, transformer = huggingface_from_pretrained(name, tok_cfg) model.attrs["tokenizer"] = tokenizer model.attrs["set_transformer"](model, transformer) # Call the model with a batch of inputs to infer the width if X: texts = [x.text for x in X] else: texts = ["hello world", "foo bar"] token_data = huggingface_tokenize(model.attrs["tokenizer"], texts) model.layers[0].initialize(X=token_data) tensors = model.layers[0].predict(token_data) t_i = find_last_hidden(tensors) model.set_dim("nO", tensors[t_i].shape[-1])
def test_model_set_dim(): class MyShim(Shim): name = "testshim" model_a = create_model("a") model = Model( "test", lambda X: (X, lambda dY: dY), dims={ "nI": 5, "nO": None }, params={ "W": None, "b": None }, refs={ "a": model_a, "b": None }, attrs={"foo": "bar"}, shims=[MyShim(None)], layers=[model_a, model_a], ) with pytest.raises(ValueError): model.set_dim("nI", 10) # force can be used before any parameters are set model.set_dim("nI", 10, force=True) model.set_param("W", model.ops.alloc1f(10)) model.set_grad("W", model.ops.alloc1f(10)) assert model.has_dim("nI") assert model.get_dim("nI") == 10 with pytest.raises(KeyError): model.set_dim("xyz", 20) with pytest.raises(ValueError): model.set_dim("nI", 20) # force can't be used after any parameter is set with pytest.raises(ValueError): model.set_dim("nI", 20, force=True)
def mlm_initialize(model: Model, X=None, Y=None): wrapped = model.layers[0] wrapped.initialize(X=X, Y=Y) for dim in wrapped.dim_names: if wrapped.has_dim(dim): model.set_dim(dim, wrapped.get_dim(dim))
def test_model_init(): class MyShim(Shim): name = "testshim" model_a = create_model("a") model = Model( "test", lambda X: (X, lambda dY: dY), dims={ "nI": 10, "nO": None }, params={ "W": numpy.zeros((10, )), "b": None }, refs={ "a": model_a, "b": None }, attrs={"foo": "bar"}, shims=[MyShim(None)], layers=[model_a, model_a], ) assert model.has_param("W") assert model.get_param("W").shape == (10, ) assert model.has_param("b") is None with pytest.raises(KeyError): model.get_param("b") with pytest.raises(KeyError): model.get_param("X") model.set_param("X", numpy.zeros((10, ))) assert model.has_param("X") assert model.get_param("X").shape == (10, ) with model.use_params({(model.id, "X"): numpy.ones((10, ))}): assert numpy.array_equal(model.get_param("X"), numpy.ones((10, ))) assert numpy.array_equal(model.get_param("X"), numpy.zeros((10, ))) assert not model.has_grad("W") assert not model.has_grad("xyz") with pytest.raises(KeyError): model.get_grad("b") model.set_param("W", model.ops.alloc1f(10)) model.set_grad("W", model.ops.alloc1f(10)) with pytest.raises(ValueError): model.inc_grad("W", numpy.zeros((5, 0))) assert model.has_dim("nI") assert model.get_dim("nI") == 10 with pytest.raises(KeyError): model.get_dim("xyz") with pytest.raises(ValueError): model.get_dim("nO") with pytest.raises(KeyError): model.set_dim("xyz", 20) with pytest.raises(ValueError): model.set_dim("nI", 20) assert model.has_ref("a") assert model.get_ref("a").name == "a" assert not model.has_ref("xyz") with pytest.raises(KeyError): model.get_ref("xyz") assert model.has_ref("b") is None with pytest.raises(ValueError): model.get_ref("b") model.set_ref("c", model_a) assert model.has_ref("c") assert model.get_ref("c").name == "a" with pytest.raises(ValueError): model.set_ref("c", create_model("c")) assert "foo" in model.attrs assert "bar" not in model.attrs assert model.attrs["foo"] == "bar" with pytest.raises(KeyError): model.attrs["bar"] model.attrs["bar"] = "baz" model_copy = model.copy() assert model_copy.name == "test"