def att_experiment2(dir_list, args): """Simple feedforward net without the attention model. This is just to test out the improvement we can gain from applying dropout """ experiment_name = sys._getframe().f_code.co_name sense_lf = l.SecondLevelLabel() num_units = int(args[0]) num_hidden_layers = int(args[1]) dropout_arg = args[2] assert(dropout_arg =='d' or dropout_arg =='n') dropout = True if dropout_arg == 'd' else False relation_list_list = [extract_implicit_relations(dir, sense_lf) for dir in dir_list] wbm = util.get_wbm(num_units) data_list = [] word2vec_ff = util._get_word2vec_ff(num_units, 'sum_pool') data_list = [word2vec_ff(relation_list) \ for relation_list in relation_list_list] label_vectors, label_alphabet = \ util.label_vectorize(relation_list_list, sense_lf) data_triplet = DataTriplet( data_list, [[x] for x in label_vectors], [label_alphabet]) num_reps = 10 num_hidden_unit_list = [0] if num_hidden_layers == 0 \ else [1000, 200, 300, 400, 600] json_file = util.set_logger('%s_%sunits_%sh_%s' % \ (experiment_name, num_units, num_hidden_layers, dropout_arg)) for num_hidden_units in num_hidden_unit_list: _train_feedforward_net(experiment_name, json_file, data_triplet, wbm, num_reps, num_hidden_layers, num_hidden_units, dropout)
def att_experiment1(dir_list, args): experiment_name = sys._getframe().f_code.co_name sense_lf = l.SecondLevelLabel() num_units = int(args[0]) num_hidden_layers = int(args[1]) num_att_hidden_layer = int(args[2]) dropout_arg = args[3] assert(dropout_arg =='d' or dropout_arg =='n') dropout = True if dropout_arg == 'd' else False relation_list_list = [extract_implicit_relations(dir, sense_lf) for dir in dir_list] wbm = util.get_wbm(num_units) data_list = [] for relation_list in relation_list_list: data = prep_serrated_matrix_relations(relation_list, wbm, 50) data_list.append(data) label_vectors, label_alphabet = \ util.label_vectorize(relation_list_list, sense_lf) data_triplet = DataTriplet( data_list, [[x] for x in label_vectors], [label_alphabet]) num_reps = 10 num_hidden_unit_list = [0] if num_hidden_layers == 0 \ else [300, 400, 600, 800] json_file = util.set_logger('%s_%sunits_%sh_%sh_%s' % \ (experiment_name, num_units, num_hidden_layers, num_att_hidden_layer, dropout_arg)) for num_hidden_units in num_hidden_unit_list: _att_experiment_ff_helper(experiment_name, AttentionModelSimple, json_file, data_triplet, wbm, num_reps, num_att_hidden_layer, num_hidden_layers, num_hidden_units, dropout)
def net_experiment_lstm(dir_list, args): """ Args : Five required arguments linear (l) or bilinear(bl) num units is the number of the units in the embedding (NOT HIDDEN LAYERS) num hidden layers is the number of hidden layers proj_type must be one of {mean_pool, sum_pool, max_pool, top} shared """ assert(len(args) == 5) if args[0] == 'bl': use_bl = True elif args[0] == 'l': use_bl = False else: raise ValueError('First argument must be l or bl') num_units = int(args[1]) num_hidden_layers = int(args[2]) proj_type = args[3] if args[4] == 'shared': arg_shared_weights = True elif args[4] == 'noshared': arg_shared_weights = False else: raise ValueError('Last argument must be shared or noshared') experiment_name = sys._getframe().f_code.co_name json_file = util.set_logger('%s_%s_%sunits_%sh_%s_%s' % \ (experiment_name, args[0], num_units, num_hidden_layers, proj_type, args[4])) sense_lf = l.SecondLevelLabel() relation_list_list = [extract_implicit_relations(dir, sense_lf) for dir in dir_list] wbm = util.get_wbm(num_units) data_list = [] for relation_list in relation_list_list: data = prep_serrated_matrix_relations(relation_list, wbm, 30) data_list.append(data) label_vectors, label_alphabet = \ util.label_vectorize(relation_list_list, sense_lf) data_triplet = DataTriplet( data_list, [[x] for x in label_vectors], [label_alphabet]) num_reps = 10 num_hidden_unit_list = [0] if num_hidden_layers == 0 \ else [20, 50, 200, 300, 400] for num_hidden_units in num_hidden_unit_list: _net_experiment_lstm_helper(experiment_name, json_file, data_triplet, num_units, num_reps, SerialLSTM, num_hidden_layers=num_hidden_layers, num_hidden_units=num_hidden_units, use_hinge=False, proj_type=proj_type, use_bl=use_bl, arg_shared_weights=arg_shared_weights )
def get_net(): """This is for debugging purposes. Should be torn apart to your taste This way you can investigate the activation as you pass the data through. """ num_units = 50 sense_lf = l.SecondLevelLabel() num_hidden_layers = 1 dropout = True num_hidden_units = 400 num_att_hidden_layer = 1 dir_list = ['conll15-st-05-19-15-train', 'conll15-st-05-19-15-dev', 'conll15-st-05-19-15-test'] dir_list = ['conll15-st-05-19-15-dev', 'conll15-st-05-19-15-dev', 'conll15-st-05-19-15-test'] relation_list_list = [extract_implicit_relations(dir, sense_lf) for dir in dir_list] wbm = util.get_wbm(num_units) data_list = [] for relation_list in relation_list_list: data = prep_serrated_matrix_relations(relation_list, wbm, 50) data_list.append(data) label_vectors, label_alphabet = \ util.label_vectorize(relation_list_list, sense_lf) data_triplet = DataTriplet( data_list, [[x] for x in label_vectors], [label_alphabet]) rng = np.random.RandomState(100) arg1_model = AttentionModelSimple(rng, wbm.num_units, num_att_hidden_layer, num_hidden_units, dropout=False) arg2_model = AttentionModelSimple(rng, wbm.num_units, num_att_hidden_layer, num_hidden_units, dropout=False) nn, all_layers = make_multilayer_net_from_layers( input_layers=[arg1_model, arg2_model], Y=T.lvector(), use_sparse=False, num_hidden_layers=num_hidden_layers, num_hidden_units=num_hidden_units, num_output_units=data_triplet.output_dimensions()[0], output_activation_fn=T.nnet.softmax, dropout=dropout) nn.input = arg1_model.input + arg2_model.input print 'before num params %s' % len(nn.params) arg1_model.params[-1].set_value([100]) arg2_model.params[-1].set_value([100]) nn.params = nn.params[(len(arg1_model.params) + len(arg2_model.params)):] print 'after num params %s' % len(nn.params) return nn, data_triplet
def net_experiment_stlstm(dir_list, args): num_units = int(args[0]) num_hidden_layers = int(args[1]) proj_type = args[2] experiment_name = sys._getframe().f_code.co_name json_file = util.set_logger('%s__%sunits_%sh_%s' % \ (experiment_name, num_units, num_hidden_layers, proj_type)) sense_lf = l.SecondLevelLabel() relation_list_list = [extract_implicit_relations(dir, sense_lf) for dir in dir_list] wbm = util.get_wbm(num_units) labels = ['NP', 'VP' , 'S', 'PP', 'SBAR', 'ADVP', 'ADJP', 'NP_', 'VP_', 'S_', 'PP_', 'SBAR_', 'ADVP_', 'ADJP_', 'OTHERS'] node_label_alphabet = {} for label in labels: node_label_alphabet[label] = len(node_label_alphabet) data_list = [] node_label_tuple_triplet = [] for relation_list in relation_list_list: data, node_labels_tuple = \ prep_stlstm_serrated_matrix_relations(relation_list, wbm, 30, node_label_alphabet) data_list.append(data) node_label_tuple_triplet.append(node_labels_tuple) label_vector_triplet, label_alphabet = \ util.label_vectorize(relation_list_list, sense_lf) outputs = [] for x,y in zip(node_label_tuple_triplet, label_vector_triplet): d = [m for m in x] d.append(y) outputs.append(d) output_alphabet = [node_label_alphabet, node_label_alphabet, label_alphabet] data_triplet = DataTriplet(data_list, outputs, output_alphabet) num_reps = 15 num_hidden_unit_list = [0] if num_hidden_layers == 0 \ else [50, 200, 300, 400] for num_hidden_units in num_hidden_unit_list: _net_experiment_stlstm_helper(experiment_name, json_file, data_triplet, wbm, num_reps, num_hidden_layers=num_hidden_layers, num_hidden_units=num_hidden_units, proj_type=proj_type, )
def _load_continuous_sparse_features(dir_list, embedding_size, sparse_feature_file, proj_type): sense_lf = l.SecondLevelLabel() relation_list_list = [extract_implicit_relations(dir, sense_lf) for dir in dir_list] num_features = 250000 id_to_sfv = read_sparse_vectors(dir_list, sparse_feature_file) sfv_data_list = [get_sfv(relation_list, id_to_sfv, num_features) for relation_list in relation_list_list] word2vec_ff = util._get_word2vec_ff(embedding_size, proj_type) word2vec_data_list = [word2vec_ff(relation_list) for relation_list in relation_list_list] data_list = [[x] + y for x, y in zip(sfv_data_list, word2vec_data_list)] label_vector_triplet, label_alphabet = \ util.label_vectorize(relation_list_list, sense_lf) data_triplet = DataTriplet( data_list, [[x] for x in label_vector_triplet], [label_alphabet]) return data_triplet
def net_experiment4_1(dir_list, args): experiment_name = sys._getframe().f_code.co_name sense_lf = l.SecondLevelLabel() num_units = int(args[0]) num_hidden_layers = int(args[1]) projection = args[2] json_file = set_logger('%s_%sunits_%sh_%s' % \ (experiment_name, num_units, num_hidden_layers, projection)) relation_list_list = [extract_implicit_relations(dir, sense_lf) for dir in dir_list] word2vec_ff = _get_word2vec_ff(num_units, projection) data_list = [word2vec_ff(relation_list) for relation_list in relation_list_list] label_vectors, label_alphabet = util.label_vectorize(relation_list_list, sense_lf) data_triplet = DataTriplet(data_list, [[x] for x in label_vectors], [label_alphabet]) num_hidden_unit_list = [50, 200, 300, 400] num_reps = 20 for num_hidden_unit in num_hidden_unit_list: _net_experiment4_helper(json_file, num_hidden_layers, num_hidden_unit, num_reps, data_triplet, True) _net_experiment4_helper(json_file, num_hidden_layers, num_hidden_unit, num_reps, data_triplet, False)
def net_experiment_tree_lstm(dir_list, args): """ Args : Five required arguments linear (l) or bilinear(bl) num units is the number of the units in the embedding (NOT HIDDEN LAYERS) num hidden layers is the number of hidden layers proj_type must be one of {mean_pool, sum_pool, max_pool, top} shared """ assert(len(args) >= 5) if args[0] == 'bl': use_bl = True elif args[0] == 'l': use_bl = False else: raise ValueError('First argument must be l or bl') num_units = int(args[1]) num_hidden_layers = int(args[2]) proj_type = args[3] if args[4] == 'shared': arg_shared_weights = True elif args[4] == 'noshared': arg_shared_weights = False else: raise ValueError('Last argument must be shared or noshared') if len(args) == 6 and args[5] == 'left': all_left_branching = True else: all_left_branching = False experiment_name = sys._getframe().f_code.co_name ## the name of method if all_left_branching: json_file = set_logger('%s_%s_%sunits_%sh_%s_%s_left' % \ (experiment_name, args[0], num_units, num_hidden_layers, proj_type, args[4])) else: json_file = set_logger('%s_%s_%sunits_%sh_%s_%s' % \ (experiment_name, args[0], num_units, num_hidden_layers, proj_type, args[4])) sense_lf = l.SecondLevelLabel() relation_list_list = [extract_implicit_relations(dir, sense_lf) for dir in dir_list] wbm = _get_wbm(num_units) data_list = [] for relation_list in relation_list_list: data = prep_tree_lstm_serrated_matrix_relations( relation_list, wbm, 35) data_list.append(data) label_vectors, label_alphabet = \ util.label_vectorize(relation_list_list, sense_lf) data_triplet = DataTriplet( data_list, [[x] for x in label_vectors], [label_alphabet]) num_reps = 15 num_hidden_unit_list = [0] if num_hidden_layers == 0 \ else [50, 200, 300, 400] for num_hidden_units in num_hidden_unit_list: _net_experiment_lstm_helper(json_file, data_triplet, wbm, num_reps, BinaryTreeLSTM, num_hidden_layers=num_hidden_layers, num_hidden_units=num_hidden_units, use_hinge=False, proj_type=proj_type, use_bl=use_bl, arg_shared_weights=arg_shared_weights )
def net_experiment_tlstm(dir_list, args): """Tree-structured LSTM experiment version 2 This version is different from net_experiment_tree_lstm in that you use BinaryForestLSTM instead. The tree structures along with the data themselvesa are encoded internally in the model. This way, we can be sure that the algorithm is doing the right thing. Scan loop is very slow as well so I hope that this works better. ipython experiments.py net_experiment_tlstm l 50 1 mean_pool """ assert(len(args) >= 4) if args[0] == 'bl': use_bl = True raise ValueError('bilinear model is not supported yet') elif args[0] == 'l': use_bl = False else: raise ValueError('First argument must be l or bl') num_units = int(args[1]) num_hidden_layers = int(args[2]) proj_type = args[3] if len(args) == 5 and args[4] == 'left': all_left_branching = True print 'use left branching trees' else: all_left_branching = False experiment_name = sys._getframe().f_code.co_name if all_left_branching: name_file = '%s_%s_%sunits_%sh_%s_left' % \ (experiment_name, args[0], num_units, num_hidden_layers, proj_type) json_file = set_logger(name_file) model_file = name_file + '.model' else: name_file = '%s_%s_%sunits_%sh_%s' % \ (experiment_name, args[0], num_units, num_hidden_layers, proj_type) json_file = set_logger(name_file) model_file = name_file + '.model' sense_lf = l.SecondLevelLabel() relation_list_list = [extract_implicit_relations(dir, sense_lf)[0:5] for dir in dir_list] wbm = _get_wbm(num_units) data_list = [] for relation_list in relation_list_list: data = prep_trees(relation_list) data_list.append(data) label_vectors, label_alphabet = \ util.label_vectorize(relation_list_list, sense_lf) data_triplet = DataTriplet( data_list, [[x] for x in label_vectors], [label_alphabet]) num_reps = 15 num_hidden_unit_list = [0] if num_hidden_layers == 0 \ else [50, 200, 300, 400] for num_hidden_units in num_hidden_unit_list: _net_experiment_tlstm_helper(json_file, model_file, data_triplet, wbm, num_reps, num_hidden_layers=num_hidden_layers, num_hidden_units=num_hidden_units, use_hinge=False, proj_type=proj_type, )