def __init__(self, cl_logits_input_dim=None):
self.global_step = tf.train.get_or_create_global_step()
self.vocab_freqs = _get_vocab_freqs()
# Cache VatxtInput objects
self.cl_inputs = None
self.lm_inputs = None
# Cache intermediate Tensors that are reused
self.tensors = {}
# Construct layers which are reused in constructing the LM and
# Classification graphs. Instantiating them all once here ensures that
# variable reuse works correctly.
self.layers = {}
self.layers['embedding'] = layers_lib.Embedding(
FLAGS.vocab_size, FLAGS.embedding_dims, FLAGS.normalize_embeddings,
self.vocab_freqs, FLAGS.keep_prob_emb)
self.layers['lstm'] = layers_lib.LSTM(FLAGS.rnn_cell_size,
FLAGS.rnn_num_layers,
FLAGS.keep_prob_lstm_out)
self.layers['lm_loss'] = layers_lib.SoftmaxLoss(
FLAGS.vocab_size,
FLAGS.num_candidate_samples,
self.vocab_freqs,
name='LM_loss')
cl_logits_input_dim = cl_logits_input_dim or FLAGS.rnn_cell_size
self.layers['cl_logits'] = layers_lib.cl_logits_subgraph(
[FLAGS.cl_hidden_size] * FLAGS.cl_num_layers, cl_logits_input_dim,
FLAGS.num_classes, FLAGS.keep_prob_cl_hidden)
def __init__(self):
super(VatxtBidirModel,
self).__init__(cl_logits_input_dim=FLAGS.rnn_cell_size * 2)
# Reverse LSTM and LM loss for bidirectional models
self.layers['lstm_reverse'] = layers_lib.LSTM(FLAGS.rnn_cell_size,
FLAGS.rnn_num_layers,
FLAGS.keep_prob_lstm_out,
name='LSTM_Reverse')
self.layers['lm_loss_reverse'] = layers_lib.SoftmaxLoss(
FLAGS.vocab_size,
FLAGS.num_candidate_samples,
self.vocab_freqs,
name='LM_loss_reverse')
def my_conv_net(n_classes):
# initialization
training_name = 'mode'
loss_name = 'loss'
ground_truth = 'y'
input_layer = layers.Input()
inp = layers.L2Norm(0.01, loss_name, input_layer)
# Convoluton layers
conv_count = 3
res_shape = [(32, 32), (16, 16), (8, 8)]
conv_filters = [(32, 32), (128, 128), (256, 512)]
conv_shapes = [((3, 3, 3), (3, 3)), ((32, 3, 3), (3, 3)),
((128, 3, 3), (3, 3))]
for i in range(0, conv_count):
inp = conv_bn_conv_bn_pool2x2(inp, conv_filters[i], conv_shapes[i],
res_shape[i], training_name)
flat = 4 * 4 * 512
inp = layers.Reshape((flat, ), inp)
# Fully-connected layers
fc_count = 2
fc_sizes = [(flat, 2048), (2048, 256)]
for i in range(0, fc_count):
inp = fc_bn_dropout(inp, fc_sizes[i], training_name)
# Last fc layer
y = layers.Affine((fc_sizes[-1][-1], n_classes), inp)
loss = layers.SoftmaxLoss(ground_truth, loss_name, y)
model = net.NeuralNetwork(input_layer, loss, loss_name, ground_truth,
training_name, layers.params, layers.grads)
return model
activation=T.tanh)
outlayers = []
cost = 0.
out_errors = []
total_errs = 0
params = layer2.params
logger.info('n_in in each softmax: %d and n_out: %d', num_of_hidden_units, 2)
for i in range(n_targets):
oneOutLayer = layers.OutputLayer(input=layer2.output,
n_in=num_of_hidden_units,
n_out=2)
# oneOutLayer = MyLogisticRegression(input=layer1.output, n_in=num_of_hidden_units, n_out=2)
onelogistic = layers.SoftmaxLoss(input=oneOutLayer.score_y_given_x,
n_in=2,
n_out=2)
params += oneOutLayer.params
outlayers.append(oneOutLayer)
total_errs += onelogistic.errors(y[:, i])
cost += onelogistic.negative_log_likelihood(y[:, i])
# total_errors
total_errs /= n_targets
# the cost we minimize during training is the NLL of the model
# L2_reg = T.sum(allweights** 2)
# if l_reg != '' and l_reg == 'L2':
# cost += l_weight * L2_reg
def __init__(self, hps, f_idxes=None, init_emb=None):
# Create the model
self.hps = hps
self.enc_len = hps.bucket[0]
self.dec_len = hps.bucket[1]
self.device = self.hps.device
self.global_step = tf.Variable(0, trainable=False)
self.f_idxes = f_idxes
self.learning_rate = tf.Variable(float(self.hps.learning_rate),
trainable=False)
self.learning_rate_decay_op = \
self.learning_rate.assign(self.learning_rate * self.hps.decay_rate)
self.__build_placeholders()
# Build modules and layers
with tf.device(self.device):
self.layers = {}
self.layers['word_emb'] = layers_lib.Embedding(self.hps.vocab_size,
self.hps.emb_size,
name="word_emb")
# Build genre embedding
# NOTE: We set fixed 36 phonology categories
self.layers['ph_emb'] = layers_lib.Embedding(36,
self.hps.ph_emb_size,
name="ph_emb")
self.layers['len_emb'] = layers_lib.Embedding(
self.dec_len + 1, self.hps.len_emb_size, name="len_emb")
# Build Encoder
self.layers['enc'] = layers_lib.BidirEncoder(self.hps.hidden_size,
self.keep_prob,
name="enc")
# The decoder cell
self.layers['dec'] = layers_lib.Decoder(self.hps.hidden_size,
self.hps.vocab_size,
self.keep_prob,
name="dec")
# History memory reading and writing layers
self.layers['attn_read'] = layers_lib.AttentionLayer("attn_read")
self.layers['attn_write'] = layers_lib.AttnWriteLayer(
self.hps.his_mem_slots, self.hps.mem_size, "attn_write")
# global trace update
self.layers['global_trace'] = layers_lib.GTraceLayer(
3, self.hps.global_trace_size, "global_trace")
# NOTE: a layer to compress the states to a smaller size for larger number of slots
self.layers['mlp_compress'] = layers_lib.MLPLayer(
[self.hps.mem_size], ['tanh'],
keep_prob=self.keep_prob,
name='mlp_compress')
self.layers['mlp_key_initial'] = layers_lib.MLPLayer(
[self.hps.hidden_size], ['tanh'],
keep_prob=self.keep_prob,
name='mlp_key_initial')
self.layers['mlp_enc_initial'] = layers_lib.MLPLayer(
[self.hps.hidden_size], ['tanh'],
keep_prob=self.keep_prob,
name='mlp_enc_initial')
self.layers['mlp_dec_merge'] = layers_lib.MLPLayer(
[self.hps.hidden_size], [None],
keep_prob=self.keep_prob,
name='mlp_dec_merge')
self.layers['mlp_topic_trace'] = layers_lib.MLPLayer(
[self.hps.topic_trace_size], ['tanh'],
keep_prob=self.keep_prob,
name='mlp_topic_trace')
self.layers['mlp_init_null'] = layers_lib.MLPLayer(
[self.hps.hidden_size], [None],
keep_prob=self.keep_prob,
name='mlp_init_null')
# loss
self.layers['softmax_loss'] = layers_lib.SoftmaxLoss(
name='softmax_loss')
# for pre-training
self.layers['mlp_dec_merge_ae'] = layers_lib.MLPLayer(
[self.hps.hidden_size], [None],
keep_prob=self.keep_prob,
name='mlp_dec_merge_ae')
from net import *
from cs231n.solver import *
import layers
if __name__ == "__main__":
# Instantiation example
i1 = layers.Input()
c1 = layers.Conv((8, 3, 3, 3), {'stride': 1, 'pad': 1}, i1)
flat = 8 * 28 * 28
s1 = layers.Reshape((flat, ), c1)
a1 = layers.Affine((flat, 10), s1)
l1 = layers.SoftmaxLoss('y', 'loss', a1)
try:
layers.load_network('network')
except IOError:
pass
model = NeuralNetwork(i1, l1, 'loss', layers.params, layers.grads)
data = {
'X_train': np.ones((2**10, 3, 28, 28)) * 0.1,
'y_train': np.ones(2**10, dtype=np.int) * 2,
'X_val': np.ones((2**3, 3, 28, 28)) * 0.1,
'y_val': np.ones(2**3, dtype=np.int) * 2
}
solver = Solver(model,
data,
update_rule='sgd',
optim_config={
'learning_rate': 1e-3,
},
