def __init__(self, w2i, permissions, options):
super().__init__(options)
self.model = dy.ParameterCollection()
self.trainer = dy.AdamTrainer(self.model)
self.w2i = w2i
self.wdims = options.wembedding_dims
self.ldims = options.lstm_dims
self.all_permissions = permissions
# Model Parameters
self.wlookup = self.model.add_lookup_parameters((len(w2i), self.wdims))
# RNNs
self.sentence_rnn = [
dy.SimpleRNNBuilder(1, self.wdims, self.ldims, self.model)
]
if options.external_embedding is not None:
self.__load_external_embeddings()
def __init__(self, input_dim, hidden_dim, output_dim, learning_rate=0.001):
self._model = dy.ParameterCollection()
self._input_dim = input_dim
self._hidden_dim = hidden_dim
self._output_dim = output_dim
self._rnn = dy.SimpleRNNBuilder(self.LAYERS, self._input_dim,
self._hidden_dim, self._model)
# self._rnn.disable_dropout()
self._W = self._model.add_parameters(
(self._output_dim, self._hidden_dim), init=dy.NormalInitializer())
self._learning_rate = learning_rate
self._trainer = dy.MomentumSGDTrainer(
self._model, learning_rate=self._learning_rate)
self._l2_param = 0.0006
# self._l2_param = 0.0
self._init_layers()
def setUp(self):
# create model
self.m = dy.ParameterCollection()
self.rnn = dy.SimpleRNNBuilder(2, 10, 10, self.m)
train = list(read(train_file))
w2i = defaultdict(lambda: UNK, w2i)
dev = list(read(test_file))
nwords = len(w2i)
ntags = len(t2i)
# DyNet Starts
model = dy.Model()
trainer = dy.AdamTrainer(model)
# Lookup parameters for word embeddings
EMB_SIZE = 32
HID_SIZE = 64
W_emb = model.add_lookup_parameters((nwords, EMB_SIZE)) # Word embeddings
fwdLSTM = dy.SimpleRNNBuilder(1, EMB_SIZE, HID_SIZE, model) # Forward LSTM
bwdLSTM = dy.SimpleRNNBuilder(1, EMB_SIZE, HID_SIZE, model) # Backward LSTM
biaffineParser = DeepBiaffineAttentionDecoder(model, ntags, src_ctx_dim=HID_SIZE * 2,
n_arc_mlp_units=64, n_label_mlp_units=32)
def calc_loss(words, labels, heads):
dy.renew_cg()
word_embs = [dy.lookup(W_emb, x) for x in words]
fwd_init = fwdLSTM.initial_state()
fwd_embs = fwd_init.transduce(word_embs)
bwd_init = bwdLSTM.initial_state()
bwd_embs = bwd_init.transduce(reversed(word_embs))
src_encodings = [dy.reshape(dy.concatenate([f, b]), (HID_SIZE * 2, 1)) for f, b in zip(fwd_embs, reversed(bwd_embs))]
return biaffineParser.decode_loss(src_encodings, ([heads], [labels]))
assert HIDDEN_SIZE % 2 == 0
# Lookup parameters for word embeddings
LOOKUP = model.add_lookup_parameters((nwords, EMBED_SIZE))
if use_teacher_forcing:
TAG_LOOKUP = model.add_lookup_parameters((ntags, TAG_EMBED_SIZE))
if use_schedule:
sampler = ScheduleSampler()
else:
sampler = AlwaysTrueSampler()
# Word-level BiLSTM is just a composition of two LSTMs.
if use_teacher_forcing:
fwdLSTM = dy.SimpleRNNBuilder(1, EMBED_SIZE + TAG_EMBED_SIZE,
HIDDEN_SIZE / 2, model) # Forward LSTM
else:
fwdLSTM = dy.SimpleRNNBuilder(1, EMBED_SIZE, HIDDEN_SIZE / 2,
model) # Forward LSTM
# We cannot insert previous predicted tag to the backward LSTM anyway.
bwdLSTM = dy.SimpleRNNBuilder(1, EMBED_SIZE, HIDDEN_SIZE / 2,
model) # Backward LSTM
# Word-level softmax
W_sm = model.add_parameters((ntags, HIDDEN_SIZE))
b_sm = model.add_parameters(ntags)
# Calculate the scores for one example
def calc_scores(words):
def setUp(self):
# create model
self.m = dy.Model()
self.rnn = dy.SimpleRNNBuilder(2, 10, 10, self.m)
def __init__(self,
task_in_size,
task_hid_size,
task_out_size,
adv_in_size,
adv_hid_size,
adv_out_size,
adv_count,
vocab_size,
dropout,
lstm_size,
adv_depth=1,
rnn_dropout=0.0,
rnn_type='lstm'):
model = dy.Model()
if rnn_type == 'lstm':
self._rnn = dy.LSTMBuilder(lstm_size, 300, task_in_size, model)
elif rnn_type == 'gru':
self._rnn = dy.GRUBuilder(lstm_size, 300, task_in_size, model)
else:
self._rnn = dy.SimpleRNNBuilder(lstm_size, 300, task_in_size,
model)
params = {}
params['w_lookup'] = model.add_lookup_parameters((vocab_size, 300))
in_task = task_in_size
params["task_w1"] = model.add_parameters((task_hid_size, in_task))
params["task_b1"] = model.add_parameters((task_hid_size))
params["task_w2"] = model.add_parameters(
(task_out_size, task_hid_size))
params["task_b2"] = model.add_parameters((task_out_size))
for i in range(adv_count):
for j in range(adv_depth):
params["adv_" + str(i) + "_w" +
str(j + 1)] = model.add_parameters(
(adv_hid_size, adv_in_size))
params["adv_" + str(i) + "_b" +
str(j + 1)] = model.add_parameters((adv_hid_size))
params["adv_" + str(i) + "_w" +
str(adv_depth + 1)] = model.add_parameters(
(adv_out_size, adv_hid_size))
params["adv_" + str(i) + "_b" +
str(adv_depth + 1)] = model.add_parameters((adv_out_size))
params["contra_adv_w1"] = model.add_parameters(
(adv_hid_size, adv_in_size))
params["contra_adv_b1"] = model.add_parameters((adv_hid_size))
params["contra_adv_w2"] = model.add_parameters(
(adv_out_size, adv_hid_size))
params["contra_adv_b2"] = model.add_parameters((adv_out_size))
self._model = model
self._hid_dim = task_hid_size
self._in_dim = task_in_size
self._adv_count = adv_count
self._adv_depth = adv_depth
self._params = params
self._dropout = dropout
self._rnn_dropout = rnn_dropout
def get_model():
global characters, int2char, char2int, VOCAB_SIZE, model, params, LAYERS, INPUT_DIM, HIDDEN_DIM
lstm = dn.LSTMBuilder(LAYERS, INPUT_DIM, HIDDEN_DIM, model)
srnn = dn.SimpleRNNBuilder(LAYERS, INPUT_DIM, HIDDEN_DIM, model)
return lstm, srnn
input_size = SEQUENCE_LENGTH
if len(f) <= input_size:
training_pairs += [f]
else:
training_pairs += [
f[i:i + input_size] for i in range(0,
len(f) - input_size)
]
rng.shuffle(training_pairs)
TRAINING_PAIRS = len(training_pairs)
print(TRAINING_PAIRS, " training pairs")
print(len(typed_output_vocab) - 1, " possible types")
# Set up the neural net
pc = dy.ParameterCollection()
srnn = dy.SimpleRNNBuilder(LAYERS, INPUT_DIM, HIDDEN_DIM, pc)
params: Dict[str, dy.Expression] = {}
params["lookup"] = pc.add_lookup_parameters((VOCAB_SIZE, INPUT_DIM))
params["R"] = pc.add_parameters((OUTPUT_VOCAB_SIZE, HIDDEN_DIM))
params["bias"] = pc.add_parameters((OUTPUT_VOCAB_SIZE))
# Load training data from disk if it exists
model = Path(MODEL_FILE)
if model.is_file():
print("Model file found, loading... ", end="")
try:
pc.populate(MODEL_FILE)
print("OK")
except Exception as e:
print("Failed")
print("Message was:\n\t%s" % (str(e), ))
