def __init__(self, cl_logits_input_dim=None): self.global_step = tf.train.get_or_create_global_step() self.vocab_freqs = _get_vocab_freqs() # Cache VatxtInput objects self.cl_inputs = None self.lm_inputs = None # Cache intermediate Tensors that are reused self.tensors = {} # Construct layers which are reused in constructing the LM and # Classification graphs. Instantiating them all once here ensures that # variable reuse works correctly. self.layers = {} self.layers['embedding'] = layers_lib.Embedding( FLAGS.vocab_size, FLAGS.embedding_dims, FLAGS.normalize_embeddings, self.vocab_freqs, FLAGS.keep_prob_emb) self.layers['lstm'] = layers_lib.LSTM(FLAGS.rnn_cell_size, FLAGS.rnn_num_layers, FLAGS.keep_prob_lstm_out) self.layers['lm_loss'] = layers_lib.SoftmaxLoss( FLAGS.vocab_size, FLAGS.num_candidate_samples, self.vocab_freqs, name='LM_loss') cl_logits_input_dim = cl_logits_input_dim or FLAGS.rnn_cell_size self.layers['cl_logits'] = layers_lib.cl_logits_subgraph( [FLAGS.cl_hidden_size] * FLAGS.cl_num_layers, cl_logits_input_dim, FLAGS.num_classes, FLAGS.keep_prob_cl_hidden)
def __init__(self): super(VatxtBidirModel, self).__init__(cl_logits_input_dim=FLAGS.rnn_cell_size * 2) # Reverse LSTM and LM loss for bidirectional models self.layers['lstm_reverse'] = layers_lib.LSTM(FLAGS.rnn_cell_size, FLAGS.rnn_num_layers, FLAGS.keep_prob_lstm_out, name='LSTM_Reverse') self.layers['lm_loss_reverse'] = layers_lib.SoftmaxLoss( FLAGS.vocab_size, FLAGS.num_candidate_samples, self.vocab_freqs, name='LM_loss_reverse')
def my_conv_net(n_classes): # initialization training_name = 'mode' loss_name = 'loss' ground_truth = 'y' input_layer = layers.Input() inp = layers.L2Norm(0.01, loss_name, input_layer) # Convoluton layers conv_count = 3 res_shape = [(32, 32), (16, 16), (8, 8)] conv_filters = [(32, 32), (128, 128), (256, 512)] conv_shapes = [((3, 3, 3), (3, 3)), ((32, 3, 3), (3, 3)), ((128, 3, 3), (3, 3))] for i in range(0, conv_count): inp = conv_bn_conv_bn_pool2x2(inp, conv_filters[i], conv_shapes[i], res_shape[i], training_name) flat = 4 * 4 * 512 inp = layers.Reshape((flat, ), inp) # Fully-connected layers fc_count = 2 fc_sizes = [(flat, 2048), (2048, 256)] for i in range(0, fc_count): inp = fc_bn_dropout(inp, fc_sizes[i], training_name) # Last fc layer y = layers.Affine((fc_sizes[-1][-1], n_classes), inp) loss = layers.SoftmaxLoss(ground_truth, loss_name, y) model = net.NeuralNetwork(input_layer, loss, loss_name, ground_truth, training_name, layers.params, layers.grads) return model
activation=T.tanh) outlayers = [] cost = 0. out_errors = [] total_errs = 0 params = layer2.params logger.info('n_in in each softmax: %d and n_out: %d', num_of_hidden_units, 2) for i in range(n_targets): oneOutLayer = layers.OutputLayer(input=layer2.output, n_in=num_of_hidden_units, n_out=2) # oneOutLayer = MyLogisticRegression(input=layer1.output, n_in=num_of_hidden_units, n_out=2) onelogistic = layers.SoftmaxLoss(input=oneOutLayer.score_y_given_x, n_in=2, n_out=2) params += oneOutLayer.params outlayers.append(oneOutLayer) total_errs += onelogistic.errors(y[:, i]) cost += onelogistic.negative_log_likelihood(y[:, i]) # total_errors total_errs /= n_targets # the cost we minimize during training is the NLL of the model # L2_reg = T.sum(allweights** 2) # if l_reg != '' and l_reg == 'L2': # cost += l_weight * L2_reg
def __init__(self, hps, f_idxes=None, init_emb=None): # Create the model self.hps = hps self.enc_len = hps.bucket[0] self.dec_len = hps.bucket[1] self.device = self.hps.device self.global_step = tf.Variable(0, trainable=False) self.f_idxes = f_idxes self.learning_rate = tf.Variable(float(self.hps.learning_rate), trainable=False) self.learning_rate_decay_op = \ self.learning_rate.assign(self.learning_rate * self.hps.decay_rate) self.__build_placeholders() # Build modules and layers with tf.device(self.device): self.layers = {} self.layers['word_emb'] = layers_lib.Embedding(self.hps.vocab_size, self.hps.emb_size, name="word_emb") # Build genre embedding # NOTE: We set fixed 36 phonology categories self.layers['ph_emb'] = layers_lib.Embedding(36, self.hps.ph_emb_size, name="ph_emb") self.layers['len_emb'] = layers_lib.Embedding( self.dec_len + 1, self.hps.len_emb_size, name="len_emb") # Build Encoder self.layers['enc'] = layers_lib.BidirEncoder(self.hps.hidden_size, self.keep_prob, name="enc") # The decoder cell self.layers['dec'] = layers_lib.Decoder(self.hps.hidden_size, self.hps.vocab_size, self.keep_prob, name="dec") # History memory reading and writing layers self.layers['attn_read'] = layers_lib.AttentionLayer("attn_read") self.layers['attn_write'] = layers_lib.AttnWriteLayer( self.hps.his_mem_slots, self.hps.mem_size, "attn_write") # global trace update self.layers['global_trace'] = layers_lib.GTraceLayer( 3, self.hps.global_trace_size, "global_trace") # NOTE: a layer to compress the states to a smaller size for larger number of slots self.layers['mlp_compress'] = layers_lib.MLPLayer( [self.hps.mem_size], ['tanh'], keep_prob=self.keep_prob, name='mlp_compress') self.layers['mlp_key_initial'] = layers_lib.MLPLayer( [self.hps.hidden_size], ['tanh'], keep_prob=self.keep_prob, name='mlp_key_initial') self.layers['mlp_enc_initial'] = layers_lib.MLPLayer( [self.hps.hidden_size], ['tanh'], keep_prob=self.keep_prob, name='mlp_enc_initial') self.layers['mlp_dec_merge'] = layers_lib.MLPLayer( [self.hps.hidden_size], [None], keep_prob=self.keep_prob, name='mlp_dec_merge') self.layers['mlp_topic_trace'] = layers_lib.MLPLayer( [self.hps.topic_trace_size], ['tanh'], keep_prob=self.keep_prob, name='mlp_topic_trace') self.layers['mlp_init_null'] = layers_lib.MLPLayer( [self.hps.hidden_size], [None], keep_prob=self.keep_prob, name='mlp_init_null') # loss self.layers['softmax_loss'] = layers_lib.SoftmaxLoss( name='softmax_loss') # for pre-training self.layers['mlp_dec_merge_ae'] = layers_lib.MLPLayer( [self.hps.hidden_size], [None], keep_prob=self.keep_prob, name='mlp_dec_merge_ae')
from net import * from cs231n.solver import * import layers if __name__ == "__main__": # Instantiation example i1 = layers.Input() c1 = layers.Conv((8, 3, 3, 3), {'stride': 1, 'pad': 1}, i1) flat = 8 * 28 * 28 s1 = layers.Reshape((flat, ), c1) a1 = layers.Affine((flat, 10), s1) l1 = layers.SoftmaxLoss('y', 'loss', a1) try: layers.load_network('network') except IOError: pass model = NeuralNetwork(i1, l1, 'loss', layers.params, layers.grads) data = { 'X_train': np.ones((2**10, 3, 28, 28)) * 0.1, 'y_train': np.ones(2**10, dtype=np.int) * 2, 'X_val': np.ones((2**3, 3, 28, 28)) * 0.1, 'y_val': np.ones(2**3, dtype=np.int) * 2 } solver = Solver(model, data, update_rule='sgd', optim_config={ 'learning_rate': 1e-3, },