path_load_weights = config["test"]["path_load_weights"]
output_encoder_size = [hidden_size for i in range(n_encoders)]
attention_size = [attention_size for i in range(n_encoders)]
n_heads = [n_heads for i in range(n_encoders)]
# Model definition #
twilbert_model = BertModel(max_len,
vocab_size,
embedding_size,
output_encoder_size,
attention_size,
n_heads,
cross_sharing,
factorize_embeddings,
input_dropout,
output_dropout,
rop_n_hidden,
rop_hidden_size,
None,
None,
pkm,
pkm_params,
input_length=None,
use_rop=use_rop)
twilbert_model.build()
model = twilbert_model.model
twilbert_model.compile(model)
twilbert_model.load(model, path_load_weights)
(batch_size, collapse_mode, lr))
h_test_res[(batch_size, collapse_mode, lr)] = {}
h_dev_res[(batch_size, collapse_mode, lr)] = {}
for it in range(runs):
K.clear_session()
print("Run: %d" % it)
# Load TWilBert model #
twilbert_model = BertModel(max_len,
vocab_size,
embedding_size,
output_encoder_size,
attention_size,
n_heads,
cross_sharing,
factorize_embeddings,
input_dropout,
output_dropout,
rop_n_hidden,
rop_hidden_size,
optimizer,
accum_iters,
pkm,
pkm_params,
input_length=None,
use_rop=use_rop)
twilbert_model.build()
model = twilbert_model.model
pretrained_model = twilbert_model.pretrained_model
twilbert_model.compile(model)
model.load_weights(pretrained_model_weights)
output_encoder_size = [hidden_size for i in range(n_encoders)]
attention_size = [attention_size for i in range(n_encoders)]
n_heads = [n_heads for i in range(n_encoders)]
# Model definition #
twilbert_model = BertModel(max_len,
vocab_size,
embedding_size,
output_encoder_size,
attention_size,
n_heads,
cross_sharing,
factorize_embeddings,
input_dropout,
output_dropout,
rop_n_hidden,
rop_hidden_size,
optimizer,
accum_iters,
pkm,
pkm_params,
initializer_range,
gpu,
multi_gpu,
n_gpus,
use_rop=use_rop)
twilbert_model.build()
model = twilbert_model.model
twilbert_model.compile(model)