def evaluate_forward(net): length = 20 net.forward_layer(layers.NumpyData(name='prev_hidden', data=np.zeros((1, hyper['mem_cells'], 1, 1)))) net.forward_layer(layers.NumpyData(name='prev_mem', data=np.zeros((1, hyper['mem_cells'], 1, 1)))) filler = layers.Filler(type='uniform', min=-hyper['init_range'], max=hyper['init_range']) accum = np.array([0.]) predictions = [] for step in range(length): value = 0.5 net.forward_layer(layers.NumpyData(name='value', data=np.array(value).reshape((1, 1, 1, 1)))) accum += value prev_hidden = 'prev_hidden' prev_mem = 'prev_mem' net.forward_layer(layers.Concat(name='lstm_concat', bottoms=[prev_hidden, 'value'])) net.forward_layer(layers.Lstm(name='lstm', bottoms=['lstm_concat', prev_mem], param_names=['input_value', 'input_gate', 'forget_gate', 'output_gate'], weight_filler=filler, tops=['next_hidden', 'next_mem'], num_cells=hyper['mem_cells'])) net.forward_layer(layers.InnerProduct(name='ip', bottoms=['next_hidden'], num_output=1)) predictions.append(float(net.tops['ip'].data.flatten()[0])) # set up for next prediction by copying LSTM outputs back to inputs net.tops['prev_hidden'].data_tensor.copy_from(net.tops['next_hidden'].data_tensor) net.tops['prev_mem'].data_tensor.copy_from(net.tops['next_mem'].data_tensor) net.reset_forward() return predictions
def forward(net, hyper, sentence_batches): batch = next(sentence_batches) #sentence_batch = np.array(pad_batch([x['body'] for x in batch], hyper)) sentence_batch = np.array(pad_batch(batch, hyper)) length = min(sentence_batch.shape[1], 100) assert length > 0 filler = layers.Filler(type='uniform', max=hyper['init_range'], min=(-hyper['init_range'])) net.forward_layer(layers.NumpyData(name='lstm_seed', data=np.zeros((hyper['batch_size'], hyper['mem_cells'], 1, 1)))) net.forward_layer(layers.NumpyData(name='label', data=np.zeros((hyper['batch_size'] * length, 1, 1, 1)))) loss = [] for step in range(length): net.forward_layer(layers.DummyData(name=('word%d' % step), shape=[hyper['batch_size'], 1, 1, 1])) if step == 0: prev_hidden = 'lstm_seed' prev_mem = 'lstm_seed' word = np.zeros(sentence_batch[:, 0].shape) else: prev_hidden = 'lstm%d_hidden' % (step - 1) prev_mem = 'lstm%d_mem' % (step - 1) word = sentence_batch[:, step - 1] net.tops['word%d' % step].data[:,0,0,0] = word net.forward_layer(layers.Wordvec(name=('wordvec%d' % step), bottoms=['word%d' % step], dimension=hyper['mem_cells'], vocab_size=hyper['vocab_size'], param_names=['wordvec_param'], weight_filler=filler)) net.forward_layer(layers.Concat(name='lstm_concat%d' % step, bottoms=[prev_hidden, 'wordvec%d' % step])) net.forward_layer(layers.Lstm(name='lstm%d' % step, bottoms=['lstm_concat%d' % step, prev_mem], param_names=['lstm_input_value', 'lstm_input_gate', 'lstm_forget_gate', 'lstm_output_gate'], tops=['lstm%d_hidden' % step, 'lstm%d_mem' % step], num_cells=hyper['mem_cells'], weight_filler=filler)) net.forward_layer(layers.Dropout(name='dropout%d' % step, bottoms=['lstm%d_hidden' % step], dropout_ratio=0.16)) label = np.reshape(sentence_batch[:, step], (hyper['batch_size'], 1, 1, 1)) net.forward_layer(layers.NumpyData(name='label%d' % step, data=label)) net.forward_layer(layers.InnerProduct(name='ip%d' % step, bottoms=['dropout%d' % step], param_names=['softmax_ip_weights', 'softmax_ip_bias'], num_output=hyper['vocab_size'], weight_filler=filler)) loss.append(net.forward_layer(layers.SoftmaxWithLoss(name='softmax_loss%d' % step, ignore_label=hyper['zero_symbol'], bottoms=['ip%d' % step, 'label%d' % step]))) return np.mean(loss)
def forward(net): length = random.randrange(5, 15) # initialize all weights in [-0.1, 0.1] filler = layers.Filler(type='uniform', min=-hyper['init_range'], max=hyper['init_range']) # initialize the LSTM memory with all 0's net.forward_layer(layers.NumpyData(name='lstm_seed', data=np.zeros((hyper['batch_size'], hyper['mem_cells'], 1, 1)))) accum = np.zeros((hyper['batch_size'],)) # Begin recurrence through 5 - 15 inputs for step in range(length): # Set up the value blob net.forward_layer(layers.DummyData(name='value%d' % step, shape=[hyper['batch_size'], 1, 1, 1])) value = np.array([random.random() for _ in range(hyper['batch_size'])]) accum += value # Set data of value blob to contain a batch of random numbers net.tops['value%d' % step].data[:, 0, 0, 0] = value if step == 0: prev_hidden = 'lstm_seed' prev_mem = 'lstm_seed' else: prev_hidden = 'lstm%d_hidden' % (step - 1) prev_mem = 'lstm%d_mem' % (step - 1) # Concatenate the hidden output with the next input value net.forward_layer(layers.Concat(name='lstm_concat%d' % step, bottoms=[prev_hidden, 'value%d' % step])) # Run the LSTM for one more step net.forward_layer(layers.Lstm(name='lstm%d' % step, bottoms=['lstm_concat%d' % step, prev_mem], param_names=['input_value', 'input_gate', 'forget_gate', 'output_gate'], tops=['lstm%d_hidden' % step, 'lstm%d_mem' % step], num_cells=hyper['mem_cells'], weight_filler=filler)) # Add a fully connected layer with a bottom blob set to be the last used LSTM cell # Note that the network structure is now a function of the data net.forward_layer(layers.InnerProduct(name='ip', bottoms=['lstm%d_hidden' % (length - 1)], num_output=1, weight_filler=filler)) # Add a label for the sum of the inputs net.forward_layer(layers.NumpyData(name='label', data=np.reshape(accum, (hyper['batch_size'], 1, 1, 1)))) # Compute the Euclidean loss between the preiction and label, used for backprop loss = net.forward_layer(layers.EuclideanLoss(name='euclidean', bottoms=['ip', 'label'])) return loss
def eval_forward(net, hyper): output_words = [] filler = layers.Filler(type='uniform', max=hyper['init_range'], min=(-hyper['init_range'])) net.forward_layer(layers.NumpyData(name='lstm_hidden_prev', data=np.zeros((1, hyper['mem_cells'], 1, 1)))) net.forward_layer(layers.NumpyData(name='lstm_mem_prev', data=np.zeros((1, hyper['mem_cells'], 1, 1)))) length = hyper['length'] for step in range(length): net.forward_layer(layers.NumpyData(name=('word'), data=np.zeros((1, 1, 1, 1)))) prev_hidden = 'lstm_hidden_prev' prev_mem = 'lstm_mem_prev' word = np.zeros((1, 1, 1, 1)) if step == 0: #output = ord('.') output = vocab[' '] else: output = softmax_choice(net.tops['softmax'].data) output_words.append(output) net.tops['word'].data[0,0,0,0] = output net.forward_layer(layers.Wordvec(name=('wordvec'), bottoms=['word'], dimension=hyper['mem_cells'], vocab_size=hyper['vocab_size'], param_names=['wordvec_param'], weight_filler=filler)) net.forward_layer(layers.Concat(name='lstm_concat', bottoms=[prev_hidden, 'wordvec'])) net.forward_layer(layers.Lstm(name='lstm', bottoms=['lstm_concat', prev_mem], param_names=['lstm_input_value', 'lstm_input_gate', 'lstm_forget_gate', 'lstm_output_gate'], tops=['lstm_hidden_next', 'lstm_mem_next'], num_cells=hyper['mem_cells'], weight_filler=filler)) net.forward_layer(layers.Dropout(name='dropout', bottoms=['lstm_hidden_next'], dropout_ratio=0.16)) net.forward_layer(layers.InnerProduct(name='ip', bottoms=['dropout'], param_names=['softmax_ip_weights', 'softmax_ip_bias'], num_output=hyper['vocab_size'], weight_filler=filler)) net.tops['ip'].data[:] *= hyper['i_temperature'] net.forward_layer(layers.Softmax(name='softmax', ignore_label=hyper['zero_symbol'], bottoms=['ip'])) net.tops['lstm_hidden_prev'].data_tensor.copy_from(net.tops['lstm_hidden_next'].data_tensor) net.tops['lstm_mem_prev'].data_tensor.copy_from(net.tops['lstm_mem_next'].data_tensor) net.reset_forward() print ''.join([ivocab[x].encode('utf8') for x in output_words]) return 0.
import apollo from apollo import layers import numpy as np net = apollo.Net() for i in range(1000): example = np.array(np.random.random()).reshape((1, 1, 1, 1)) net.forward_layer(layers.NumpyData(name='data', data=example)) net.forward_layer(layers.NumpyData(name='label', data=(example * 3))) net.forward_layer( layers.Convolution(name='conv', kernel_size=1, bottoms=['data'], num_output=1)) loss = net.forward_layer( layers.EuclideanLoss(name='loss', bottoms=['conv', 'label'])) net.backward() net.update(lr=0.1) if i % 100 == 0: print loss
def batch_is(self, batch, phase): self.batch = batch image_array = batch.image_array bbox_label = batch.bbox_label_array conf_label = batch.conf_label_array binary_label_array = batch.binary_label_array net = self.net net.forward_layer(layers.NumpyData(name="data", data=image_array)) net.forward_layer(layers.NumpyData(name="bbox_label", data=bbox_label)) net.forward_layer(layers.NumpyData(name="conf_label", data=conf_label)) net.forward_layer( layers.NumpyData(name="binary_label", data=binary_label_array)) # parameters weight_filler = layers.Filler(type="xavier") bias_filler = layers.Filler(type="constant", value=0.2) conv_lr_mults = [1.0, 2.0] conv_decay_mults = [1.0, 0.0] # bunch of conv / relu layers net.forward_layer( layers.Convolution(name="conv1", bottoms=["data"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, stride=1, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=150)) net.forward_layer( layers.ReLU(name="relu1", bottoms=["conv1"], tops=["conv1"])) net.forward_layer( layers.Convolution(name="conv2", bottoms=["conv1"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, stride=1, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=150)) net.forward_layer( layers.ReLU(name="relu2", bottoms=["conv2"], tops=["conv2"])) # only pooling layer net.forward_layer( layers.Pooling(name="pool4", bottoms=["conv2"], kernel_size=2, stride=2)) # finished 2nd pool net.forward_layer( layers.Convolution(name="conv3", bottoms=["pool4"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, stride=1, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=150)) net.forward_layer( layers.ReLU(name="relu3", bottoms=["conv3"], tops=["conv3"])) net.forward_layer( layers.Convolution(name="conv4", bottoms=["conv3"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, stride=1, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=150)) net.forward_layer( layers.ReLU(name="relu4", bottoms=["conv4"], tops=["conv4"])) net.forward_layer( layers.Convolution(name="conv5", bottoms=["conv4"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, stride=1, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=150)) net.forward_layer( layers.ReLU(name="relu5", bottoms=["conv5"], tops=["conv5"])) net.forward_layer( layers.Convolution(name="conv6", bottoms=["conv5"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, stride=1, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=150)) net.forward_layer( layers.ReLU(name="relu6", bottoms=["conv6"], tops=["conv6"])) net.forward_layer( layers.Convolution(name="conv_0", bottoms=["conv6"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, stride=1, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=150)) net.forward_layer( layers.ReLU(name="relu_0", bottoms=["conv_0"], tops=["conv_0"])) net.forward_layer( layers.Convolution(name="conv_1", bottoms=["conv_0"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, stride=1, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=150)) net.forward_layer( layers.ReLU(name="relu_1", bottoms=["conv_1"], tops=["conv_1"])) net.forward_layer( layers.Convolution(name="conv_2", bottoms=["conv_1"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, stride=1, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=150)) net.forward_layer( layers.ReLU(name="relu_2", bottoms=["conv_2"], tops=["conv_2"])) # inner product layers net.forward_layer( layers.Convolution(name="conv8", bottoms=["conv_2"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=4000)) net.forward_layer( layers.ReLU(name="relu8", bottoms=["conv8"], tops=["conv8"])) layers.Dropout(name="drop0", bottoms=["conv8"], tops=["conv8"], dropout_ratio=0.5, phase=phase), net.forward_layer( layers.Convolution(name="L7", bottoms=["conv8"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=4000)) net.forward_layer( layers.ReLU(name="relu9", bottoms=["L7"], tops=["L7"])) layers.Dropout(name="drop1", bottoms=["L7"], tops=["L7"], dropout_ratio=0.5, phase=phase), # binary prediction layers: is a character here ? net.forward_layer( layers.Convolution(name="binary_conf_pred", bottoms=["L7"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=2)) binary_softmax_loss = net.forward_layer( layers.SoftmaxWithLoss( name='binary_softmax_loss', bottoms=['binary_conf_pred', 'binary_label'])) net.forward_layer( layers.Softmax(name='binary_softmax', bottoms=['binary_conf_pred'])) # character predictions label_softmax_loss = 0 net.forward_layer( layers.Convolution(name="label_conf_pred", bottoms=["L7"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=11)) label_softmax_loss = net.forward_layer( layers.SoftmaxWithLoss(name='label_softmax_loss', bottoms=['label_conf_pred', 'conf_label'], loss_weight=1., ignore_label=0)) net.forward_layer( layers.Softmax(name='label_softmax', bottoms=['label_conf_pred'])) # bounding box prediction net.forward_layer( layers.Convolution(name="bbox_pred", bottoms=["L7"], param_lr_mults=conv_lr_mults, param_decay_mults=[0., 0.], kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=4)) net.forward_layer( layers.Concat(name='bbox_mask', bottoms=4 * ['binary_label'])) net.forward_layer( layers.Eltwise(name='bbox_pred_masked', bottoms=['bbox_pred', 'bbox_mask'], operation='PROD')) net.forward_layer( layers.Eltwise(name='bbox_label_masked', bottoms=['bbox_label', 'bbox_mask'], operation='PROD')) bbox_loss = net.forward_layer( layers.L1Loss(name='l1_loss', bottoms=['bbox_pred_masked', 'bbox_label_masked'], loss_weight=0.001)) return (binary_softmax_loss, label_softmax_loss, bbox_loss)
def forward(net, sentence_batches): source_batch, target_batch = next(sentence_batches) filler = layers.Filler(type='uniform', max=hyper['init_range'], min=(-hyper['init_range'])) net.forward_layer( layers.NumpyData(name='source_lstm_seed', data=np.zeros( (hyper['batch_size'], hyper['mem_cells'], 1, 1)))) hidden_bottoms = ['source_lstm_seed'] mem_bottoms = ['source_lstm_seed'] lengths = [ min(len([1 for token in x if token != hyper['pad_symbol']]), hyper['max_len']) for x in source_batch ] for step in range(source_batch.shape[1]): net.forward_layer( layers.DummyData(name=('source_word%d' % step), shape=[hyper['batch_size'], 1, 1, 1])) if step == 0: prev_hidden = 'source_lstm_seed' prev_mem = 'source_lstm_seed' else: prev_hidden = 'source_lstm%d_hidden' % (step - 1) prev_mem = 'source_lstm%d_mem' % (step - 1) next_hidden = 'source_lstm%d_hidden' % (step) next_mem = 'source_lstm%d_mem' % (step) hidden_bottoms.append(next_hidden) mem_bottoms.append(next_mem) word = source_batch[:, step] net.tops['source_word%d' % step].data[:, 0, 0, 0] = word wordvec = layers.Wordvec(name=('source_wordvec%d' % step), bottoms=['source_word%d' % step], dimension=hyper['mem_cells'], vocab_size=hyper['vocab_size'], param_names=['source_wordvec_param'], weight_filler=filler) concat = layers.Concat( name='source_lstm_concat%d' % step, bottoms=[prev_hidden, 'source_wordvec%d' % step]) lstm = layers.Lstm( name='source_lstm%d' % step, bottoms=['source_lstm_concat%d' % step, prev_mem], param_names=[ 'source_lstm_input_value', 'source_lstm_input_gate', 'source_lstm_forget_gate', 'source_lstm_output_gate' ], tops=['source_lstm%d_hidden' % step, 'source_lstm%d_mem' % step], num_cells=hyper['mem_cells'], weight_filler=filler) net.forward_layer(wordvec) net.forward_layer(concat) net.forward_layer(lstm) net.forward_layer( layers.CapSequence(name='hidden_seed', sequence_lengths=lengths, bottoms=hidden_bottoms)) net.forward_layer( layers.CapSequence(name='mem_seed', sequence_lengths=lengths, bottoms=mem_bottoms)) loss = [] for step in range(target_batch.shape[1]): if step == 0: prev_hidden = 'hidden_seed' prev_mem = 'mem_seed' word = np.zeros(target_batch[:, 0].shape) else: prev_hidden = 'lstm%d_hidden' % (step - 1) prev_mem = 'lstm%d_mem' % (step - 1) word = target_batch[:, step - 1] word = layers.NumpyData(name=('word%d' % step), data=np.reshape( word, (hyper['batch_size'], 1, 1, 1))) wordvec = layers.Wordvec(name=('wordvec%d' % step), bottoms=['word%d' % step], dimension=hyper['mem_cells'], vocab_size=hyper['vocab_size'], param_names=['source_wordvec_param'], weight_filler=filler) concat = layers.Concat(name='lstm_concat%d' % step, bottoms=[prev_hidden, 'wordvec%d' % step]) lstm = layers.Lstm(name='lstm%d' % step, bottoms=['lstm_concat%d' % step, prev_mem], param_names=[ 'lstm_input_value', 'lstm_input_gate', 'lstm_forget_gate', 'lstm_output_gate' ], tops=['lstm%d_hidden' % step, 'lstm%d_mem' % step], num_cells=hyper['mem_cells'], weight_filler=filler) dropout = layers.Dropout(name='dropout%d' % step, bottoms=['lstm%d_hidden' % step], dropout_ratio=0.16) net.forward_layer(word) net.forward_layer(wordvec) net.forward_layer(concat) net.forward_layer(lstm) net.forward_layer(dropout) net.forward_layer( layers.NumpyData(name='label%d' % step, data=np.reshape(target_batch[:, step], (hyper['batch_size'], 1, 1, 1)))) net.forward_layer( layers.InnerProduct(name='ip%d' % step, bottoms=['dropout%d' % step], param_names=['ip_weight', 'ip_bias'], num_output=hyper['vocab_size'], weight_filler=filler)) loss.append( net.forward_layer( layers.SoftmaxWithLoss( name='softmax_loss%d' % step, ignore_label=hyper['pad_symbol'], bottoms=['ip%d' % step, 'label%d' % step]))) loss.append( net.forward_layer( layers.Softmax(name='softmax%d' % step, ignore_label=hyper['pad_symbol'], bottoms=['ip%d' % step]))) return np.mean(loss)