def att_experiment2(dir_list, args):
"""Simple feedforward net without the attention model.
This is just to test out the improvement we can gain from applying dropout
"""
experiment_name = sys._getframe().f_code.co_name
sense_lf = l.SecondLevelLabel()
num_units = int(args[0])
num_hidden_layers = int(args[1])
dropout_arg = args[2]
assert(dropout_arg =='d' or dropout_arg =='n')
dropout = True if dropout_arg == 'd' else False
relation_list_list = [extract_implicit_relations(dir, sense_lf)
for dir in dir_list]
wbm = util.get_wbm(num_units)
data_list = []
word2vec_ff = util._get_word2vec_ff(num_units, 'sum_pool')
data_list = [word2vec_ff(relation_list) \
for relation_list in relation_list_list]
label_vectors, label_alphabet = \
util.label_vectorize(relation_list_list, sense_lf)
data_triplet = DataTriplet(
data_list, [[x] for x in label_vectors], [label_alphabet])
num_reps = 10
num_hidden_unit_list = [0] if num_hidden_layers == 0 \
else [1000, 200, 300, 400, 600]
json_file = util.set_logger('%s_%sunits_%sh_%s' % \
(experiment_name, num_units, num_hidden_layers, dropout_arg))
for num_hidden_units in num_hidden_unit_list:
_train_feedforward_net(experiment_name,
json_file, data_triplet, wbm, num_reps,
num_hidden_layers, num_hidden_units, dropout)
def att_experiment1(dir_list, args):
experiment_name = sys._getframe().f_code.co_name
sense_lf = l.SecondLevelLabel()
num_units = int(args[0])
num_hidden_layers = int(args[1])
num_att_hidden_layer = int(args[2])
dropout_arg = args[3]
assert(dropout_arg =='d' or dropout_arg =='n')
dropout = True if dropout_arg == 'd' else False
relation_list_list = [extract_implicit_relations(dir, sense_lf)
for dir in dir_list]
wbm = util.get_wbm(num_units)
data_list = []
for relation_list in relation_list_list:
data = prep_serrated_matrix_relations(relation_list, wbm, 50)
data_list.append(data)
label_vectors, label_alphabet = \
util.label_vectorize(relation_list_list, sense_lf)
data_triplet = DataTriplet(
data_list, [[x] for x in label_vectors], [label_alphabet])
num_reps = 10
num_hidden_unit_list = [0] if num_hidden_layers == 0 \
else [300, 400, 600, 800]
json_file = util.set_logger('%s_%sunits_%sh_%sh_%s' % \
(experiment_name, num_units,
num_hidden_layers, num_att_hidden_layer, dropout_arg))
for num_hidden_units in num_hidden_unit_list:
_att_experiment_ff_helper(experiment_name, AttentionModelSimple,
json_file, data_triplet, wbm, num_reps,
num_att_hidden_layer, num_hidden_layers, num_hidden_units,
dropout)
def net_experiment_lstm(dir_list, args):
"""
Args : Five required arguments
linear (l) or bilinear(bl)
num units is the number of the units in the embedding (NOT HIDDEN LAYERS)
num hidden layers is the number of hidden layers
proj_type must be one of {mean_pool, sum_pool, max_pool, top}
shared
"""
assert(len(args) == 5)
if args[0] == 'bl':
use_bl = True
elif args[0] == 'l':
use_bl = False
else:
raise ValueError('First argument must be l or bl')
num_units = int(args[1])
num_hidden_layers = int(args[2])
proj_type = args[3]
if args[4] == 'shared':
arg_shared_weights = True
elif args[4] == 'noshared':
arg_shared_weights = False
else:
raise ValueError('Last argument must be shared or noshared')
experiment_name = sys._getframe().f_code.co_name
json_file = util.set_logger('%s_%s_%sunits_%sh_%s_%s' % \
(experiment_name, args[0], num_units,
num_hidden_layers, proj_type, args[4]))
sense_lf = l.SecondLevelLabel()
relation_list_list = [extract_implicit_relations(dir, sense_lf)
for dir in dir_list]
wbm = util.get_wbm(num_units)
data_list = []
for relation_list in relation_list_list:
data = prep_serrated_matrix_relations(relation_list, wbm, 30)
data_list.append(data)
label_vectors, label_alphabet = \
util.label_vectorize(relation_list_list, sense_lf)
data_triplet = DataTriplet(
data_list, [[x] for x in label_vectors], [label_alphabet])
num_reps = 10
num_hidden_unit_list = [0] if num_hidden_layers == 0 \
else [20, 50, 200, 300, 400]
for num_hidden_units in num_hidden_unit_list:
_net_experiment_lstm_helper(experiment_name,
json_file, data_triplet, num_units, num_reps,
SerialLSTM,
num_hidden_layers=num_hidden_layers,
num_hidden_units=num_hidden_units,
use_hinge=False,
proj_type=proj_type,
use_bl=use_bl,
arg_shared_weights=arg_shared_weights
)
def get_data_srm(dir_list, wbm, max_length=75):
sense_lf = l.SecondLevelLabel()
relation_list_list = [extract_implicit_relations(dir, sense_lf)
for dir in dir_list]
data_list = []
for relation_list in relation_list_list:
data = prep_serrated_matrix_relations(relation_list, wbm, max_length)
data_list.append(data)
label_vectors, label_alphabet = \
label_vectorize(relation_list_list, sense_lf)
data_triplet = DataTriplet(
data_list, [[x] for x in label_vectors], [label_alphabet])
return data_triplet
def get_net():
"""This is for debugging purposes. Should be torn apart to your taste
This way you can investigate the activation as you pass the data through.
"""
num_units = 50
sense_lf = l.SecondLevelLabel()
num_hidden_layers = 1
dropout = True
num_hidden_units = 400
num_att_hidden_layer = 1
dir_list = ['conll15-st-05-19-15-train', 'conll15-st-05-19-15-dev', 'conll15-st-05-19-15-test']
dir_list = ['conll15-st-05-19-15-dev', 'conll15-st-05-19-15-dev', 'conll15-st-05-19-15-test']
relation_list_list = [extract_implicit_relations(dir, sense_lf)
for dir in dir_list]
wbm = util.get_wbm(num_units)
data_list = []
for relation_list in relation_list_list:
data = prep_serrated_matrix_relations(relation_list, wbm, 50)
data_list.append(data)
label_vectors, label_alphabet = \
util.label_vectorize(relation_list_list, sense_lf)
data_triplet = DataTriplet(
data_list, [[x] for x in label_vectors], [label_alphabet])
rng = np.random.RandomState(100)
arg1_model = AttentionModelSimple(rng,
wbm.num_units, num_att_hidden_layer, num_hidden_units,
dropout=False)
arg2_model = AttentionModelSimple(rng,
wbm.num_units, num_att_hidden_layer, num_hidden_units,
dropout=False)
nn, all_layers = make_multilayer_net_from_layers(
input_layers=[arg1_model, arg2_model],
Y=T.lvector(), use_sparse=False,
num_hidden_layers=num_hidden_layers,
num_hidden_units=num_hidden_units,
num_output_units=data_triplet.output_dimensions()[0],
output_activation_fn=T.nnet.softmax,
dropout=dropout)
nn.input = arg1_model.input + arg2_model.input
print 'before num params %s' % len(nn.params)
arg1_model.params[-1].set_value([100])
arg2_model.params[-1].set_value([100])
nn.params = nn.params[(len(arg1_model.params) + len(arg2_model.params)):]
print 'after num params %s' % len(nn.params)
return nn, data_triplet
def net_experiment_stlstm(dir_list, args):
num_units = int(args[0])
num_hidden_layers = int(args[1])
proj_type = args[2]
experiment_name = sys._getframe().f_code.co_name
json_file = util.set_logger('%s__%sunits_%sh_%s' % \
(experiment_name, num_units, num_hidden_layers, proj_type))
sense_lf = l.SecondLevelLabel()
relation_list_list = [extract_implicit_relations(dir, sense_lf)
for dir in dir_list]
wbm = util.get_wbm(num_units)
labels = ['NP', 'VP' , 'S', 'PP', 'SBAR', 'ADVP', 'ADJP',
'NP_', 'VP_', 'S_', 'PP_', 'SBAR_', 'ADVP_', 'ADJP_',
'OTHERS']
node_label_alphabet = {}
for label in labels:
node_label_alphabet[label] = len(node_label_alphabet)
data_list = []
node_label_tuple_triplet = []
for relation_list in relation_list_list:
data, node_labels_tuple = \
prep_stlstm_serrated_matrix_relations(relation_list, wbm, 30, node_label_alphabet)
data_list.append(data)
node_label_tuple_triplet.append(node_labels_tuple)
label_vector_triplet, label_alphabet = \
util.label_vectorize(relation_list_list, sense_lf)
outputs = []
for x,y in zip(node_label_tuple_triplet, label_vector_triplet):
d = [m for m in x]
d.append(y)
outputs.append(d)
output_alphabet = [node_label_alphabet, node_label_alphabet, label_alphabet]
data_triplet = DataTriplet(data_list, outputs, output_alphabet)
num_reps = 15
num_hidden_unit_list = [0] if num_hidden_layers == 0 \
else [50, 200, 300, 400]
for num_hidden_units in num_hidden_unit_list:
_net_experiment_stlstm_helper(experiment_name,
json_file, data_triplet, wbm, num_reps,
num_hidden_layers=num_hidden_layers,
num_hidden_units=num_hidden_units,
proj_type=proj_type,
)
def _load_continuous_sparse_features(dir_list, embedding_size,
sparse_feature_file, proj_type):
sense_lf = l.SecondLevelLabel()
relation_list_list = [extract_implicit_relations(dir, sense_lf)
for dir in dir_list]
num_features = 250000
id_to_sfv = read_sparse_vectors(dir_list, sparse_feature_file)
sfv_data_list = [get_sfv(relation_list, id_to_sfv, num_features)
for relation_list in relation_list_list]
word2vec_ff = util._get_word2vec_ff(embedding_size, proj_type)
word2vec_data_list = [word2vec_ff(relation_list)
for relation_list in relation_list_list]
data_list = [[x] + y for x, y in zip(sfv_data_list, word2vec_data_list)]
label_vector_triplet, label_alphabet = \
util.label_vectorize(relation_list_list, sense_lf)
data_triplet = DataTriplet(
data_list, [[x] for x in label_vector_triplet], [label_alphabet])
return data_triplet
def net_experiment4_1(dir_list, args):
experiment_name = sys._getframe().f_code.co_name
sense_lf = l.SecondLevelLabel()
num_units = int(args[0])
num_hidden_layers = int(args[1])
projection = args[2]
json_file = set_logger('%s_%sunits_%sh_%s' % \
(experiment_name, num_units, num_hidden_layers, projection))
relation_list_list = [extract_implicit_relations(dir, sense_lf) for dir in dir_list]
word2vec_ff = _get_word2vec_ff(num_units, projection)
data_list = [word2vec_ff(relation_list) for relation_list in relation_list_list]
label_vectors, label_alphabet = util.label_vectorize(relation_list_list, sense_lf)
data_triplet = DataTriplet(data_list, [[x] for x in label_vectors], [label_alphabet])
num_hidden_unit_list = [50, 200, 300, 400]
num_reps = 20
for num_hidden_unit in num_hidden_unit_list:
_net_experiment4_helper(json_file, num_hidden_layers, num_hidden_unit, num_reps,
data_triplet, True)
_net_experiment4_helper(json_file, num_hidden_layers, num_hidden_unit, num_reps,
data_triplet, False)
def net_experiment_tree_lstm(dir_list, args):
"""
Args : Five required arguments
linear (l) or bilinear(bl)
num units is the number of the units in the embedding (NOT HIDDEN LAYERS)
num hidden layers is the number of hidden layers
proj_type must be one of {mean_pool, sum_pool, max_pool, top}
shared
"""
assert(len(args) >= 5)
if args[0] == 'bl':
use_bl = True
elif args[0] == 'l':
use_bl = False
else:
raise ValueError('First argument must be l or bl')
num_units = int(args[1])
num_hidden_layers = int(args[2])
proj_type = args[3]
if args[4] == 'shared':
arg_shared_weights = True
elif args[4] == 'noshared':
arg_shared_weights = False
else:
raise ValueError('Last argument must be shared or noshared')
if len(args) == 6 and args[5] == 'left':
all_left_branching = True
else:
all_left_branching = False
experiment_name = sys._getframe().f_code.co_name ## the name of method
if all_left_branching:
json_file = set_logger('%s_%s_%sunits_%sh_%s_%s_left' % \
(experiment_name, args[0], num_units,
num_hidden_layers, proj_type, args[4]))
else:
json_file = set_logger('%s_%s_%sunits_%sh_%s_%s' % \
(experiment_name, args[0], num_units,
num_hidden_layers, proj_type, args[4]))
sense_lf = l.SecondLevelLabel()
relation_list_list = [extract_implicit_relations(dir, sense_lf)
for dir in dir_list]
wbm = _get_wbm(num_units)
data_list = []
for relation_list in relation_list_list:
data = prep_tree_lstm_serrated_matrix_relations(
relation_list, wbm, 35)
data_list.append(data)
label_vectors, label_alphabet = \
util.label_vectorize(relation_list_list, sense_lf)
data_triplet = DataTriplet(
data_list, [[x] for x in label_vectors], [label_alphabet])
num_reps = 15
num_hidden_unit_list = [0] if num_hidden_layers == 0 \
else [50, 200, 300, 400]
for num_hidden_units in num_hidden_unit_list:
_net_experiment_lstm_helper(json_file, data_triplet, wbm, num_reps,
BinaryTreeLSTM,
num_hidden_layers=num_hidden_layers,
num_hidden_units=num_hidden_units,
use_hinge=False,
proj_type=proj_type,
use_bl=use_bl,
arg_shared_weights=arg_shared_weights
)
def net_experiment_tlstm(dir_list, args):
"""Tree-structured LSTM experiment version 2
This version is different from net_experiment_tree_lstm in that
you use BinaryForestLSTM instead. The tree structures along with
the data themselvesa are encoded internally in the model.
This way, we can be sure that the algorithm is doing the right thing.
Scan loop is very slow as well so I hope that this works better.
ipython experiments.py net_experiment_tlstm l 50 1 mean_pool
"""
assert(len(args) >= 4)
if args[0] == 'bl':
use_bl = True
raise ValueError('bilinear model is not supported yet')
elif args[0] == 'l':
use_bl = False
else:
raise ValueError('First argument must be l or bl')
num_units = int(args[1])
num_hidden_layers = int(args[2])
proj_type = args[3]
if len(args) == 5 and args[4] == 'left':
all_left_branching = True
print 'use left branching trees'
else:
all_left_branching = False
experiment_name = sys._getframe().f_code.co_name
if all_left_branching:
name_file = '%s_%s_%sunits_%sh_%s_left' % \
(experiment_name, args[0], num_units,
num_hidden_layers, proj_type)
json_file = set_logger(name_file)
model_file = name_file + '.model'
else:
name_file = '%s_%s_%sunits_%sh_%s' % \
(experiment_name, args[0], num_units,
num_hidden_layers, proj_type)
json_file = set_logger(name_file)
model_file = name_file + '.model'
sense_lf = l.SecondLevelLabel()
relation_list_list = [extract_implicit_relations(dir, sense_lf)[0:5]
for dir in dir_list]
wbm = _get_wbm(num_units)
data_list = []
for relation_list in relation_list_list:
data = prep_trees(relation_list)
data_list.append(data)
label_vectors, label_alphabet = \
util.label_vectorize(relation_list_list, sense_lf)
data_triplet = DataTriplet(
data_list, [[x] for x in label_vectors], [label_alphabet])
num_reps = 15
num_hidden_unit_list = [0] if num_hidden_layers == 0 \
else [50, 200, 300, 400]
for num_hidden_units in num_hidden_unit_list:
_net_experiment_tlstm_helper(json_file, model_file,
data_triplet, wbm, num_reps,
num_hidden_layers=num_hidden_layers,
num_hidden_units=num_hidden_units,
use_hinge=False,
proj_type=proj_type,
)