def _copy_tlstm_net(data_list, nn, proj_type):
arg1_model = nn.layers[0].copy(data_list[0])
arg2_model = nn.layers[1].copy(data_list[1])
if proj_type == 'max_pool':
proj_variables = [arg1_model.max_pooled_h, arg2_model.max_pooled_h]
elif proj_type == 'mean_pool':
proj_variables = [arg1_model.mean_pooled_h, arg2_model.mean_pooled_h]
elif proj_type == 'sum_pool':
proj_variables = [arg1_model.sum_pooled_h, arg2_model.sum_pooled_h]
elif proj_type == 'top':
proj_variables = [arg1_model.top_h, arg2_model.top_h]
else:
raise ValueError('Invalid projection type: %s' % proj_type)
X_list = proj_variables
new_hidden_layers = []
for hidden_layer in nn.layers[2:-1]:
new_hidden_layer = hidden_layer.copy(X_list)
X_list = [new_hidden_layer.activation]
new_hidden_layers.append(new_hidden_layer)
output_layer = nn.layers[-1].copy(X_list)
new_nn = NeuralNet()
layers = [arg1_model, arg2_model] + new_hidden_layers + [output_layer]
new_nn.layers = layers
new_nn.input.extend(arg1_model.input)
new_nn.output.extend(output_layer.output)
new_nn.predict = output_layer.predict
new_nn.hinge_loss = output_layer.hinge_loss
new_nn.crossentropy = output_layer.crossentropy
return new_nn
def _make_tlstm_net(data_list, wbm, num_output_units,
num_hidden_layers, num_hidden_units, use_hinge, proj_type):
rng = np.random.RandomState(100)
indices = T.lvector()
arg1_model = BinaryForestLSTM(data_list[0], rng, wbm, X_list=[indices])
arg2_model = BinaryForestLSTM(data_list[1], rng, wbm, X_list=[indices])
#f = theano.function(inputs=arg1_model.input,
#outputs=[arg1_model.max_pooled_h, arg1_model.all_max_pooled_h.shape])
#print f(np.array([2]))
#f = theano.function(inputs=arg1_model.input,
#outputs=[arg1_model.sum_pooled_h, arg1_model.all_sum_pooled_h.shape])
#print f(np.array([2]))
#f = theano.function(inputs=arg2_model.input,
#outputs=[arg2_model.max_pooled_h, arg2_model.all_max_pooled_h.shape])
#print f(np.array([2]))
if proj_type == 'max_pool':
proj_variables = [arg1_model.max_pooled_h, arg2_model.max_pooled_h]
elif proj_type == 'mean_pool':
proj_variables = [arg1_model.mean_pooled_h, arg2_model.mean_pooled_h]
elif proj_type == 'sum_pool':
proj_variables = [arg1_model.sum_pooled_h, arg2_model.sum_pooled_h]
elif proj_type == 'top':
proj_variables = [arg1_model.top_h, arg2_model.top_h]
else:
raise ValueError('Invalid projection type: %s' % proj_type)
hidden_layers = []
n_in_list = [wbm.num_units, wbm.num_units]
X_list = proj_variables
for i in range(num_hidden_layers):
hidden_layer = LinearLayer(rng,
n_in_list=n_in_list,
n_out=num_hidden_units,
use_sparse=False,
X_list=X_list,
activation_fn=T.tanh)
n_in_list = [num_hidden_units]
X_list = [hidden_layer.activation]
hidden_layers.append(hidden_layer)
output_layer = LinearLayer(rng,
n_in_list=n_in_list,
n_out=num_output_units,
use_sparse=False,
X_list=X_list,
Y=T.lvector(),
activation_fn=None if use_hinge else T.nnet.softmax)
nn = NeuralNet()
layers = [arg1_model, arg2_model] + hidden_layers + [output_layer]
nn.params.extend(arg1_model.params)
nn.params.extend(arg2_model.params)
nn.params.extend(output_layer.params)
for hidden_layer in hidden_layers:
nn.params.extend(hidden_layer.params)
nn.layers = layers
nn.input.extend(arg1_model.input)
nn.output.extend(output_layer.output)
nn.predict = output_layer.predict
nn.hinge_loss = output_layer.hinge_loss
nn.crossentropy = output_layer.crossentropy
return nn
def _net_experiment_lstm_helper(json_file, data_triplet, wbm, num_reps,
LSTMModel, num_hidden_layers, num_hidden_units, use_hinge, proj_type,
use_bl, arg_shared_weights):
rng = np.random.RandomState(100)
arg1_model = LSTMModel(rng, wbm.num_units)
if arg_shared_weights:
arg2_model = LSTMModel(rng, wbm.num_units,
W=arg1_model.W, U=arg1_model.U, b=arg1_model.b)
else:
arg2_model = LSTMModel(rng, wbm.num_units)
if proj_type == 'max_pool':
proj_variables = [arg1_model.max_pooled_h, arg2_model.max_pooled_h]
elif proj_type == 'mean_pool':
proj_variables = [arg1_model.mean_pooled_h, arg2_model.mean_pooled_h]
elif proj_type == 'sum_pool':
proj_variables = [arg1_model.sum_pooled_h, arg2_model.sum_pooled_h]
elif proj_type == 'top':
proj_variables = [arg1_model.top_h, arg2_model.top_h]
else:
raise ValueError('Invalid projection type: %s' % proj_type)
hidden_layers = []
if use_bl:
output_layer = BilinearLayer(rng,
n_in1=wbm.num_units,
n_in2=wbm.num_units,
n_out=data_triplet.output_dimensions()[0],
X1=proj_variables[0],
X2=proj_variables[1],
Y=T.lvector(),
activation_fn=None if use_hinge else T.nnet.softmax)
else:
n_in_list = [wbm.num_units, wbm.num_units]
X_list = proj_variables
for i in range(num_hidden_layers):
hidden_layer = LinearLayer(rng,
n_in_list=n_in_list,
n_out=num_hidden_units,
use_sparse=False,
X_list=X_list,
activation_fn=T.tanh)
n_in_list = [num_hidden_units]
X_list = [hidden_layer.activation]
hidden_layers.append(hidden_layer)
output_layer = LinearLayer(rng,
n_in_list=n_in_list,
n_out=data_triplet.output_dimensions()[0],
use_sparse=False,
X_list=X_list,
Y=T.lvector(),
activation_fn=None if use_hinge else T.nnet.softmax)
nn = NeuralNet()
layers = [arg1_model, arg2_model, output_layer] + hidden_layers
nn.params.extend(arg1_model.params)
if not arg_shared_weights:
nn.params.extend(arg2_model.params)
nn.params.extend(output_layer.params)
for hidden_layer in hidden_layers:
nn.params.extend(hidden_layer.params)
nn.layers = layers
nn.input.extend(arg1_model.input)
nn.input.extend(arg2_model.input)
nn.output.extend(output_layer.output)
nn.predict = output_layer.predict
nn.hinge_loss = output_layer.hinge_loss
nn.crossentropy = output_layer.crossentropy
learning_rate = 0.001
lr_smoother = 0.01
trainer = AdagradTrainer(nn,
nn.hinge_loss if use_hinge else nn.crossentropy,
learning_rate, lr_smoother, data_triplet, LSTMModel.make_givens)
for rep in xrange(num_reps):
random_seed = rep
rng = np.random.RandomState(random_seed)
for layer in layers:
layer.reset(rng)
trainer.reset()
minibatch_size = np.random.randint(20, 60)
minibatch_size = 1
n_epochs = 50
start_time = timeit.default_timer()
best_iter, best_dev_acc, best_test_acc = \
trainer.train_minibatch_triplet(minibatch_size, n_epochs)
end_time = timeit.default_timer()
print end_time - start_time
print best_iter, best_dev_acc, best_test_acc
result_dict = {
'test accuracy': best_test_acc,
'best dev accuracy': best_dev_acc,
'best iter': best_iter,
'random seed': random_seed,
'minibatch size': minibatch_size,
'learning rate': learning_rate,
'lr smoother': lr_smoother,
'experiment name': experiment_name,
'num hidden units': num_hidden_units,
'num hidden layers': num_hidden_layers,
'cost function': 'hinge loss' if use_hinge else 'crossentropy',
'projection' : proj_type,
}
json_file.write('%s\n' % json.dumps(result_dict, sort_keys=True))
def _construct_net(data_triplet, wbm,
num_hidden_layers, num_hidden_units, proj_type):
rng = np.random.RandomState(100)
arg1_model = BinaryTreeLSTM(rng, wbm.num_units)
arg2_model = BinaryTreeLSTM(rng, wbm.num_units)
arg1_node_label_layer = LinearLayerTensorOutput(rng,
n_in=wbm.num_units,
n_out=data_triplet.output_dimensions()[0],
X=arg1_model.h)
arg2_node_label_layer = LinearLayerTensorOutput(rng,
n_in=wbm.num_units,
n_out=data_triplet.output_dimensions()[1],
X=arg2_model.h)
if proj_type == 'max_pool':
proj_variables = [arg1_model.max_pooled_h, arg2_model.max_pooled_h]
elif proj_type == 'mean_pool':
proj_variables = [arg1_model.mean_pooled_h, arg2_model.mean_pooled_h]
elif proj_type == 'sum_pool':
proj_variables = [arg1_model.sum_pooled_h, arg2_model.sum_pooled_h]
elif proj_type == 'top':
proj_variables = [arg1_model.top_h, arg2_model.top_h]
else:
raise ValueError('Invalid projection type: %s' % proj_type)
hidden_layers = []
n_in_list = [wbm.num_units, wbm.num_units]
X_list = proj_variables
for i in range(num_hidden_layers):
hidden_layer = LinearLayer(rng,
n_in_list=n_in_list,
n_out=num_hidden_units,
use_sparse=False,
X_list=X_list,
activation_fn=T.tanh)
n_in_list = [num_hidden_units]
X_list = [hidden_layer.activation]
hidden_layers.append(hidden_layer)
label_output_layer = LinearLayer(rng,
n_in_list=n_in_list,
n_out=data_triplet.output_dimensions()[2],
use_sparse=False,
X_list=X_list,
Y=T.lvector(),
activation_fn=T.nnet.softmax)
nn = NeuralNet()
layers = [arg1_model, arg2_model,
arg1_node_label_layer, arg2_node_label_layer, label_output_layer] \
+ hidden_layers
for layer in layers:
nn.params.extend(layer.params)
nn.layers = layers
nn.input.extend(arg1_model.input)
nn.input.extend(arg2_model.input)
nn.output.extend(arg1_node_label_layer.output +
arg2_node_label_layer.output +
label_output_layer.output)
nn.predict = label_output_layer.predict
nn.crossentropy = label_output_layer.crossentropy + \
0.5 * arg1_node_label_layer.crossentropy + \
0.5 * arg2_node_label_layer.crossentropy
nn.misc_function = arg1_node_label_layer.miscs + arg2_node_label_layer.miscs
"""
nn.crossentropy = arg2_node_label_layer.crossentropy + arg1_node_label_layer.crossentropy
nn.params = arg2_node_label_layer.params + arg1_node_label_layer.params
nn.misc_function = [arg1_node_label_layer.W[0:3,0:3], arg2_node_label_layer.W[0:3,0:3]]
nn.crossentropy = label_output_layer.crossentropy
if len(hidden_layers) > 0:
nn.layers = [arg1_model, arg2_model, hidden_layers[0], label_output_layer]
nn.params = arg1_model.params + arg2_model.params + hidden_layers[0].params + label_output_layer.params
else:
nn.layers = [arg1_model, arg2_model, label_output_layer]
nn.params = arg1_model.params + arg2_model.params + label_output_layer.params
nn.misc_function = [label_output_layer.predict[0],
label_output_layer.W_list[0][0:3,0:3],
arg1_node_label_layer.W[0:3,0:3],
label_output_layer.activation]
"""
return nn