def build_model(alphabet, input_dim, hidden_dim, layers, feature_types, feat_input_dim, feature_alphabet):
print 'creating model...'
model = pc.Model()
# character embeddings
model.add_lookup_parameters("char_lookup", (len(alphabet), input_dim))
# feature embeddings
model.add_lookup_parameters("feat_lookup", (len(feature_alphabet), feat_input_dim))
# used in softmax output
model.add_parameters("R", (len(alphabet), hidden_dim))
model.add_parameters("bias", len(alphabet))
# rnn's
encoder_frnn = pc.LSTMBuilder(layers, input_dim, hidden_dim, model)
encoder_rrnn = pc.LSTMBuilder(layers, input_dim, hidden_dim, model)
# 4 * HIDDEN_DIM, as it gets previous output, previous char, BLSTM[i]
concatenated_input_dim = 4 * hidden_dim + len(feature_types) * feat_input_dim
decoder_rnn = pc.LSTMBuilder(layers, concatenated_input_dim, hidden_dim, model)
print 'decoder lstm dimensions are {} x {}'.format(concatenated_input_dim, hidden_dim)
print 'finished creating model'
return model, encoder_frnn, encoder_rrnn, decoder_rnn
def build_model(alphabet, input_dim, hidden_dim, layers, feature_types,
feat_input_dim, feature_alphabet):
print 'creating model...'
model = pc.Model()
# character embeddings
model.add_lookup_parameters("char_lookup", (len(alphabet), input_dim))
# feature embeddings
model.add_lookup_parameters("feat_lookup",
(len(feature_alphabet), feat_input_dim))
# used in softmax output
model.add_parameters("R", (len(alphabet), 3 * hidden_dim))
model.add_parameters("bias", len(alphabet))
# rnn's
encoder_frnn = pc.LSTMBuilder(layers, input_dim, hidden_dim, model)
encoder_rrnn = pc.LSTMBuilder(layers, input_dim, hidden_dim, model)
print 'encoder lstms dimensions: {} x {}'.format(input_dim, hidden_dim)
# attention MLPs - Loung-style with extra v_a from Bahdanau
# concatenation layer for h_input (2*hidden_dim), h_output (hidden_dim)
model.add_parameters("W_a", (1, 3 * hidden_dim))
# concatenation layer for h (hidden dim), c (2 * hidden_dim)
model.add_parameters("W_c", (3 * hidden_dim, 3 * hidden_dim))
# attention MLP's - Bahdanau-style
# concatenation layer for h_input (2*hidden_dim), h_output (hidden_dim)
model.add_parameters("W__a", (hidden_dim, hidden_dim))
# concatenation layer for h (hidden dim), c (2 * hidden_dim)
model.add_parameters("U__a", (hidden_dim, 2 * hidden_dim))
# concatenation layer for h_input (2*hidden_dim), h_output (hidden_dim)
model.add_parameters("v__a", (1, hidden_dim))
# 2 * input - gets dual feedback and feature encoding
decoder_lstm_input = 2 * input_dim + len(feature_types) * feat_input_dim
decoder_rnn = pc.LSTMBuilder(layers, decoder_lstm_input, hidden_dim, model)
print 'decoder lstm dimensions: {} x {}'.format(decoder_lstm_input,
hidden_dim)
print 'finished creating model'
return model, encoder_frnn, encoder_rrnn, decoder_rnn
def __init__(
self,
word_count,
tag_count,
word_dims,
tag_dims,
lstm_units,
hidden_units,
struct_out,
label_out,
droprate=0,
struct_spans=4,
label_spans=3,
):
self.word_count = word_count
self.tag_count = tag_count
self.word_dims = word_dims
self.tag_dims = tag_dims
self.lstm_units = lstm_units
self.hidden_units = hidden_units
self.struct_out = struct_out
self.label_out = label_out
self.droprate = droprate
self.model = pycnn.Model()
self.trainer = pycnn.AdadeltaTrainer(self.model,
lam=0,
eps=1e-7,
rho=0.99)
random.seed(1)
self.activation = pycnn.rectify
self.model.add_lookup_parameters('word-embed', (word_count, word_dims))
self.model.add_lookup_parameters('tag-embed', (tag_count, tag_dims))
self.fwd_lstm1 = LSTM(word_dims + tag_dims, lstm_units, self.model)
self.back_lstm1 = LSTM(word_dims + tag_dims, lstm_units, self.model)
self.fwd_lstm2 = LSTM(2 * lstm_units, lstm_units, self.model)
self.back_lstm2 = LSTM(2 * lstm_units, lstm_units, self.model)
self.model.add_parameters(
'struct-hidden-W',
(hidden_units, 4 * struct_spans * lstm_units),
)
self.model.add_parameters('struct-hidden-b', hidden_units)
self.model.add_parameters('struct-out-W', (struct_out, hidden_units))
self.model.add_parameters('struct-out-b', struct_out)
self.model.add_parameters(
'label-hidden-W',
(hidden_units, 4 * label_spans * lstm_units),
)
self.model.add_parameters('label-hidden-b', hidden_units)
self.model.add_parameters('label-out-W', (label_out, hidden_units))
self.model.add_parameters('label-out-b', label_out)
import gpycnn as G
print
import pycnn as C
cm = C.Model()
gm = G.Model()
cpW = cm.add_parameters("W",(1000,1000))
gpW = gm.add_parameters("W",(1000,1000))
def do_cpu():
C.renew_cg()
W = C.parameter(cpW)
W = W*W*W*W*W*W*W
z = C.squared_distance(W,W)
z.value()
z.backward()
def do_gpu():
G.renew_cg()
W = G.parameter(gpW)
W = W*W*W*W*W*W*W
z = G.squared_distance(W,W)
z.value()
z.backward()
import time
s = time.time()
do_cpu()
print "CPU time:",time.time() - s
EOS = "<EOS>"
characters = list("abcdefghijklmnopqrstuvwxyz ")
characters.append(EOS)
int2char = list(characters)
char2int = {c: i for i, c in enumerate(characters)}
VOCAB_SIZE = len(characters)
LSTM_NUM_OF_LAYERS = 2
EMBEDDINGS_SIZE = 32
STATE_SIZE = 32
ATTENTION_SIZE = 32
model = pc.Model()
enc_fwd_lstm = pc.LSTMBuilder(LSTM_NUM_OF_LAYERS, EMBEDDINGS_SIZE, STATE_SIZE,
model)
enc_bwd_lstm = pc.LSTMBuilder(LSTM_NUM_OF_LAYERS, EMBEDDINGS_SIZE, STATE_SIZE,
model)
dec_lstm = pc.LSTMBuilder(LSTM_NUM_OF_LAYERS, STATE_SIZE * 2, STATE_SIZE,
model)
model.add_lookup_parameters("lookup", (VOCAB_SIZE, EMBEDDINGS_SIZE))
model.add_parameters("attention_w1", (ATTENTION_SIZE, STATE_SIZE * 2))
model.add_parameters("attention_w2",
(ATTENTION_SIZE, STATE_SIZE * LSTM_NUM_OF_LAYERS * 2))
model.add_parameters("attention_v", (1, ATTENTION_SIZE))
model.add_parameters("decoder_w", (VOCAB_SIZE, STATE_SIZE))