def __init__(self, w2i, permissions, options): super().__init__(options) self.model = dy.ParameterCollection() self.trainer = dy.AdamTrainer(self.model) self.w2i = w2i self.wdims = options.wembedding_dims self.ldims = options.lstm_dims self.all_permissions = permissions # Model Parameters self.wlookup = self.model.add_lookup_parameters((len(w2i), self.wdims)) # RNNs self.sentence_rnn = [ dy.SimpleRNNBuilder(1, self.wdims, self.ldims, self.model) ] if options.external_embedding is not None: self.__load_external_embeddings()
def __init__(self, input_dim, hidden_dim, output_dim, learning_rate=0.001): self._model = dy.ParameterCollection() self._input_dim = input_dim self._hidden_dim = hidden_dim self._output_dim = output_dim self._rnn = dy.SimpleRNNBuilder(self.LAYERS, self._input_dim, self._hidden_dim, self._model) # self._rnn.disable_dropout() self._W = self._model.add_parameters( (self._output_dim, self._hidden_dim), init=dy.NormalInitializer()) self._learning_rate = learning_rate self._trainer = dy.MomentumSGDTrainer( self._model, learning_rate=self._learning_rate) self._l2_param = 0.0006 # self._l2_param = 0.0 self._init_layers()
def setUp(self): # create model self.m = dy.ParameterCollection() self.rnn = dy.SimpleRNNBuilder(2, 10, 10, self.m)
train = list(read(train_file)) w2i = defaultdict(lambda: UNK, w2i) dev = list(read(test_file)) nwords = len(w2i) ntags = len(t2i) # DyNet Starts model = dy.Model() trainer = dy.AdamTrainer(model) # Lookup parameters for word embeddings EMB_SIZE = 32 HID_SIZE = 64 W_emb = model.add_lookup_parameters((nwords, EMB_SIZE)) # Word embeddings fwdLSTM = dy.SimpleRNNBuilder(1, EMB_SIZE, HID_SIZE, model) # Forward LSTM bwdLSTM = dy.SimpleRNNBuilder(1, EMB_SIZE, HID_SIZE, model) # Backward LSTM biaffineParser = DeepBiaffineAttentionDecoder(model, ntags, src_ctx_dim=HID_SIZE * 2, n_arc_mlp_units=64, n_label_mlp_units=32) def calc_loss(words, labels, heads): dy.renew_cg() word_embs = [dy.lookup(W_emb, x) for x in words] fwd_init = fwdLSTM.initial_state() fwd_embs = fwd_init.transduce(word_embs) bwd_init = bwdLSTM.initial_state() bwd_embs = bwd_init.transduce(reversed(word_embs)) src_encodings = [dy.reshape(dy.concatenate([f, b]), (HID_SIZE * 2, 1)) for f, b in zip(fwd_embs, reversed(bwd_embs))] return biaffineParser.decode_loss(src_encodings, ([heads], [labels]))
assert HIDDEN_SIZE % 2 == 0 # Lookup parameters for word embeddings LOOKUP = model.add_lookup_parameters((nwords, EMBED_SIZE)) if use_teacher_forcing: TAG_LOOKUP = model.add_lookup_parameters((ntags, TAG_EMBED_SIZE)) if use_schedule: sampler = ScheduleSampler() else: sampler = AlwaysTrueSampler() # Word-level BiLSTM is just a composition of two LSTMs. if use_teacher_forcing: fwdLSTM = dy.SimpleRNNBuilder(1, EMBED_SIZE + TAG_EMBED_SIZE, HIDDEN_SIZE / 2, model) # Forward LSTM else: fwdLSTM = dy.SimpleRNNBuilder(1, EMBED_SIZE, HIDDEN_SIZE / 2, model) # Forward LSTM # We cannot insert previous predicted tag to the backward LSTM anyway. bwdLSTM = dy.SimpleRNNBuilder(1, EMBED_SIZE, HIDDEN_SIZE / 2, model) # Backward LSTM # Word-level softmax W_sm = model.add_parameters((ntags, HIDDEN_SIZE)) b_sm = model.add_parameters(ntags) # Calculate the scores for one example def calc_scores(words):
def setUp(self): # create model self.m = dy.Model() self.rnn = dy.SimpleRNNBuilder(2, 10, 10, self.m)
def __init__(self, task_in_size, task_hid_size, task_out_size, adv_in_size, adv_hid_size, adv_out_size, adv_count, vocab_size, dropout, lstm_size, adv_depth=1, rnn_dropout=0.0, rnn_type='lstm'): model = dy.Model() if rnn_type == 'lstm': self._rnn = dy.LSTMBuilder(lstm_size, 300, task_in_size, model) elif rnn_type == 'gru': self._rnn = dy.GRUBuilder(lstm_size, 300, task_in_size, model) else: self._rnn = dy.SimpleRNNBuilder(lstm_size, 300, task_in_size, model) params = {} params['w_lookup'] = model.add_lookup_parameters((vocab_size, 300)) in_task = task_in_size params["task_w1"] = model.add_parameters((task_hid_size, in_task)) params["task_b1"] = model.add_parameters((task_hid_size)) params["task_w2"] = model.add_parameters( (task_out_size, task_hid_size)) params["task_b2"] = model.add_parameters((task_out_size)) for i in range(adv_count): for j in range(adv_depth): params["adv_" + str(i) + "_w" + str(j + 1)] = model.add_parameters( (adv_hid_size, adv_in_size)) params["adv_" + str(i) + "_b" + str(j + 1)] = model.add_parameters((adv_hid_size)) params["adv_" + str(i) + "_w" + str(adv_depth + 1)] = model.add_parameters( (adv_out_size, adv_hid_size)) params["adv_" + str(i) + "_b" + str(adv_depth + 1)] = model.add_parameters((adv_out_size)) params["contra_adv_w1"] = model.add_parameters( (adv_hid_size, adv_in_size)) params["contra_adv_b1"] = model.add_parameters((adv_hid_size)) params["contra_adv_w2"] = model.add_parameters( (adv_out_size, adv_hid_size)) params["contra_adv_b2"] = model.add_parameters((adv_out_size)) self._model = model self._hid_dim = task_hid_size self._in_dim = task_in_size self._adv_count = adv_count self._adv_depth = adv_depth self._params = params self._dropout = dropout self._rnn_dropout = rnn_dropout
def get_model(): global characters, int2char, char2int, VOCAB_SIZE, model, params, LAYERS, INPUT_DIM, HIDDEN_DIM lstm = dn.LSTMBuilder(LAYERS, INPUT_DIM, HIDDEN_DIM, model) srnn = dn.SimpleRNNBuilder(LAYERS, INPUT_DIM, HIDDEN_DIM, model) return lstm, srnn
input_size = SEQUENCE_LENGTH if len(f) <= input_size: training_pairs += [f] else: training_pairs += [ f[i:i + input_size] for i in range(0, len(f) - input_size) ] rng.shuffle(training_pairs) TRAINING_PAIRS = len(training_pairs) print(TRAINING_PAIRS, " training pairs") print(len(typed_output_vocab) - 1, " possible types") # Set up the neural net pc = dy.ParameterCollection() srnn = dy.SimpleRNNBuilder(LAYERS, INPUT_DIM, HIDDEN_DIM, pc) params: Dict[str, dy.Expression] = {} params["lookup"] = pc.add_lookup_parameters((VOCAB_SIZE, INPUT_DIM)) params["R"] = pc.add_parameters((OUTPUT_VOCAB_SIZE, HIDDEN_DIM)) params["bias"] = pc.add_parameters((OUTPUT_VOCAB_SIZE)) # Load training data from disk if it exists model = Path(MODEL_FILE) if model.is_file(): print("Model file found, loading... ", end="") try: pc.populate(MODEL_FILE) print("OK") except Exception as e: print("Failed") print("Message was:\n\t%s" % (str(e), ))