def att_experiment2(dir_list, args):
    """Simple feedforward net without the attention model. 

    This is just to test out the improvement we can gain from applying dropout
    """
    experiment_name = sys._getframe().f_code.co_name    
    sense_lf = l.SecondLevelLabel()
    num_units = int(args[0])
    num_hidden_layers = int(args[1])
    dropout_arg = args[2]
    assert(dropout_arg =='d' or dropout_arg =='n')
    dropout = True if dropout_arg == 'd' else False

    relation_list_list = [extract_implicit_relations(dir, sense_lf) 
            for dir in dir_list]
    wbm = util.get_wbm(num_units)
    data_list = []
    word2vec_ff = util._get_word2vec_ff(num_units, 'sum_pool')
    data_list = [word2vec_ff(relation_list) \
            for relation_list in relation_list_list]
    label_vectors, label_alphabet = \
            util.label_vectorize(relation_list_list, sense_lf)
    data_triplet = DataTriplet(
            data_list, [[x] for x in label_vectors], [label_alphabet])

    num_reps = 10
    num_hidden_unit_list = [0] if num_hidden_layers == 0 \
            else [1000, 200, 300, 400, 600] 

    json_file = util.set_logger('%s_%sunits_%sh_%s' % \
            (experiment_name, num_units, num_hidden_layers, dropout_arg))
    for num_hidden_units in num_hidden_unit_list:
        _train_feedforward_net(experiment_name,
                json_file, data_triplet, wbm, num_reps, 
                num_hidden_layers, num_hidden_units, dropout)
def att_experiment1(dir_list, args):
    experiment_name = sys._getframe().f_code.co_name    
    sense_lf = l.SecondLevelLabel()
    num_units = int(args[0])
    num_hidden_layers = int(args[1])
    num_att_hidden_layer = int(args[2])
    dropout_arg = args[3]
    assert(dropout_arg =='d' or dropout_arg =='n')
    dropout = True if dropout_arg == 'd' else False
    

    relation_list_list = [extract_implicit_relations(dir, sense_lf) 
            for dir in dir_list]
    wbm = util.get_wbm(num_units)
    data_list = []
    for relation_list in relation_list_list:
        data = prep_serrated_matrix_relations(relation_list, wbm, 50)
        data_list.append(data)
    label_vectors, label_alphabet = \
            util.label_vectorize(relation_list_list, sense_lf)
    data_triplet = DataTriplet(
            data_list, [[x] for x in label_vectors], [label_alphabet])
    num_reps = 10
    num_hidden_unit_list = [0] if num_hidden_layers == 0 \
            else [300, 400, 600, 800] 

    json_file = util.set_logger('%s_%sunits_%sh_%sh_%s' % \
            (experiment_name, num_units, 
                num_hidden_layers, num_att_hidden_layer, dropout_arg))
    for num_hidden_units in num_hidden_unit_list:
        _att_experiment_ff_helper(experiment_name, AttentionModelSimple,
                json_file, data_triplet, wbm, num_reps, 
                num_att_hidden_layer, num_hidden_layers, num_hidden_units,
                dropout)
Ejemplo n.º 3
0
def net_experiment_lstm(dir_list, args):
    """

    Args : Five required arguments
        linear (l) or bilinear(bl)
        num units is the number of the units in the embedding (NOT HIDDEN LAYERS)
        num hidden layers is the number of hidden layers
        proj_type must be one of {mean_pool, sum_pool, max_pool, top}
        shared 
    """
    assert(len(args) == 5)

    if args[0] == 'bl':
        use_bl = True
    elif args[0] == 'l':
        use_bl = False
    else:
        raise ValueError('First argument must be l or bl')
    num_units = int(args[1])
    num_hidden_layers = int(args[2])
    proj_type = args[3]
    if args[4] == 'shared':
        arg_shared_weights = True
    elif args[4] == 'noshared':
        arg_shared_weights = False
    else:
        raise ValueError('Last argument must be shared or noshared')

    experiment_name = sys._getframe().f_code.co_name    
    json_file = util.set_logger('%s_%s_%sunits_%sh_%s_%s' % \
            (experiment_name, args[0], num_units, 
                num_hidden_layers, proj_type, args[4]))
    sense_lf = l.SecondLevelLabel()
    relation_list_list = [extract_implicit_relations(dir, sense_lf) 
            for dir in dir_list]

    wbm = util.get_wbm(num_units)
    data_list = []
    for relation_list in relation_list_list:
        data = prep_serrated_matrix_relations(relation_list, wbm, 30)
        data_list.append(data)
    label_vectors, label_alphabet = \
            util.label_vectorize(relation_list_list, sense_lf)
    data_triplet = DataTriplet(
            data_list, [[x] for x in label_vectors], [label_alphabet])

    num_reps = 10
    num_hidden_unit_list = [0] if num_hidden_layers == 0 \
            else [20, 50, 200, 300, 400] 
    for num_hidden_units in num_hidden_unit_list:
        _net_experiment_lstm_helper(experiment_name,
                json_file, data_triplet, num_units, num_reps, 
                SerialLSTM,
                num_hidden_layers=num_hidden_layers, 
                num_hidden_units=num_hidden_units, 
                use_hinge=False, 
                proj_type=proj_type,
                use_bl=use_bl,
                arg_shared_weights=arg_shared_weights
                )
Ejemplo n.º 4
0
def get_data_srm(dir_list, wbm, max_length=75):
    sense_lf = l.SecondLevelLabel()
    relation_list_list = [extract_implicit_relations(dir, sense_lf) 
            for dir in dir_list]
    data_list = []
    for relation_list in relation_list_list:
        data = prep_serrated_matrix_relations(relation_list, wbm, max_length)
        data_list.append(data)
    label_vectors, label_alphabet = \
            label_vectorize(relation_list_list, sense_lf)
    data_triplet = DataTriplet(
            data_list, [[x] for x in label_vectors], [label_alphabet])
    return data_triplet
def get_net():
    """This is for debugging purposes. Should be torn apart to your taste

    This way you can investigate the activation as you pass the data through.
    """
    num_units = 50
    sense_lf = l.SecondLevelLabel()
    num_hidden_layers = 1
    dropout = True
    num_hidden_units = 400
    num_att_hidden_layer = 1

    dir_list = ['conll15-st-05-19-15-train', 'conll15-st-05-19-15-dev', 'conll15-st-05-19-15-test']
    dir_list = ['conll15-st-05-19-15-dev', 'conll15-st-05-19-15-dev', 'conll15-st-05-19-15-test']
    relation_list_list = [extract_implicit_relations(dir, sense_lf) 
            for dir in dir_list]
    wbm = util.get_wbm(num_units)
    data_list = []
    for relation_list in relation_list_list:
        data = prep_serrated_matrix_relations(relation_list, wbm, 50)
        data_list.append(data)
    label_vectors, label_alphabet = \
            util.label_vectorize(relation_list_list, sense_lf)
    data_triplet = DataTriplet(
            data_list, [[x] for x in label_vectors], [label_alphabet])

    rng = np.random.RandomState(100)

    arg1_model = AttentionModelSimple(rng, 
            wbm.num_units, num_att_hidden_layer, num_hidden_units,
            dropout=False)
    arg2_model = AttentionModelSimple(rng, 
            wbm.num_units, num_att_hidden_layer, num_hidden_units,
            dropout=False)
    nn, all_layers = make_multilayer_net_from_layers(
            input_layers=[arg1_model, arg2_model],
            Y=T.lvector(), use_sparse=False,
            num_hidden_layers=num_hidden_layers,
            num_hidden_units=num_hidden_units,
            num_output_units=data_triplet.output_dimensions()[0],
            output_activation_fn=T.nnet.softmax,
            dropout=dropout)
    nn.input = arg1_model.input + arg2_model.input
    print 'before num params %s' % len(nn.params)
    arg1_model.params[-1].set_value([100])
    arg2_model.params[-1].set_value([100])
    nn.params = nn.params[(len(arg1_model.params) + len(arg2_model.params)):]
    print 'after num params %s' % len(nn.params)
    return nn, data_triplet
Ejemplo n.º 6
0
def net_experiment_stlstm(dir_list, args):
    num_units = int(args[0])
    num_hidden_layers = int(args[1])
    proj_type = args[2]
    experiment_name = sys._getframe().f_code.co_name    
    json_file = util.set_logger('%s__%sunits_%sh_%s' % \
            (experiment_name,  num_units, num_hidden_layers, proj_type))
    sense_lf = l.SecondLevelLabel()
    relation_list_list = [extract_implicit_relations(dir, sense_lf) 
            for dir in dir_list]
    wbm = util.get_wbm(num_units)
    labels = ['NP', 'VP' , 'S', 'PP', 'SBAR', 'ADVP', 'ADJP', 
            'NP_', 'VP_', 'S_', 'PP_', 'SBAR_', 'ADVP_', 'ADJP_', 
            'OTHERS']
    node_label_alphabet = {}
    for label in labels:
        node_label_alphabet[label] = len(node_label_alphabet)

    data_list = []
    node_label_tuple_triplet = []
    for relation_list in relation_list_list:
        data, node_labels_tuple = \
                prep_stlstm_serrated_matrix_relations(relation_list, wbm, 30, node_label_alphabet)
        data_list.append(data)
        node_label_tuple_triplet.append(node_labels_tuple)
    label_vector_triplet, label_alphabet = \
            util.label_vectorize(relation_list_list, sense_lf)

    outputs = []
    for x,y in zip(node_label_tuple_triplet, label_vector_triplet):
        d = [m for m in x]
        d.append(y)
        outputs.append(d)
    output_alphabet = [node_label_alphabet, node_label_alphabet, label_alphabet]
    data_triplet = DataTriplet(data_list, outputs, output_alphabet)

    num_reps = 15
    num_hidden_unit_list = [0] if num_hidden_layers == 0 \
            else [50, 200, 300, 400] 

    for num_hidden_units in num_hidden_unit_list:
        _net_experiment_stlstm_helper(experiment_name, 
                json_file, data_triplet, wbm, num_reps, 
                num_hidden_layers=num_hidden_layers, 
                num_hidden_units=num_hidden_units, 
                proj_type=proj_type,
                )
Ejemplo n.º 7
0
def _load_continuous_sparse_features(dir_list, embedding_size,
        sparse_feature_file, proj_type):
    sense_lf = l.SecondLevelLabel()
    relation_list_list = [extract_implicit_relations(dir, sense_lf) 
            for dir in dir_list]

    num_features = 250000
    id_to_sfv = read_sparse_vectors(dir_list, sparse_feature_file)
    sfv_data_list = [get_sfv(relation_list, id_to_sfv, num_features) 
            for relation_list in relation_list_list]

    word2vec_ff = util._get_word2vec_ff(embedding_size, proj_type)
    word2vec_data_list = [word2vec_ff(relation_list) 
            for relation_list in relation_list_list]

    data_list = [[x] + y for x, y in zip(sfv_data_list, word2vec_data_list)]

    label_vector_triplet, label_alphabet = \
            util.label_vectorize(relation_list_list, sense_lf)
    data_triplet = DataTriplet(
            data_list, [[x] for x in label_vector_triplet], [label_alphabet])
    return data_triplet
Ejemplo n.º 8
0
def net_experiment4_1(dir_list, args):
    experiment_name = sys._getframe().f_code.co_name    
    sense_lf = l.SecondLevelLabel()
    num_units = int(args[0])
    num_hidden_layers = int(args[1])
    projection = args[2]

    json_file = set_logger('%s_%sunits_%sh_%s' % \
            (experiment_name, num_units, num_hidden_layers, projection))

    relation_list_list = [extract_implicit_relations(dir, sense_lf) for dir in dir_list]
    word2vec_ff = _get_word2vec_ff(num_units, projection)
    data_list = [word2vec_ff(relation_list) for relation_list in relation_list_list]
    label_vectors, label_alphabet = util.label_vectorize(relation_list_list, sense_lf)
    data_triplet = DataTriplet(data_list, [[x] for x in label_vectors], [label_alphabet])
    num_hidden_unit_list = [50, 200, 300, 400] 
    num_reps = 20
    for num_hidden_unit in num_hidden_unit_list:
        _net_experiment4_helper(json_file, num_hidden_layers, num_hidden_unit, num_reps,
                data_triplet, True)
        _net_experiment4_helper(json_file, num_hidden_layers, num_hidden_unit, num_reps,
                data_triplet, False)
Ejemplo n.º 9
0
def net_experiment_tree_lstm(dir_list, args):
    """
    Args : Five required arguments
        linear (l) or bilinear(bl)
        num units is the number of the units in the embedding (NOT HIDDEN LAYERS)
        num hidden layers is the number of hidden layers
        proj_type must be one of {mean_pool, sum_pool, max_pool, top}
        shared 
    """
    assert(len(args) >= 5)

    if args[0] == 'bl':
        use_bl = True
    elif args[0] == 'l':
        use_bl = False
    else:
        raise ValueError('First argument must be l or bl')
    num_units = int(args[1])
    num_hidden_layers = int(args[2])
    proj_type = args[3]
    if args[4] == 'shared':
        arg_shared_weights = True
    elif args[4] == 'noshared':
        arg_shared_weights = False
    else:
        raise ValueError('Last argument must be shared or noshared')

    if len(args) == 6 and args[5] == 'left':
        all_left_branching = True
    else:
        all_left_branching = False

    experiment_name = sys._getframe().f_code.co_name ## the name of method
    if all_left_branching:
        json_file = set_logger('%s_%s_%sunits_%sh_%s_%s_left' % \
                (experiment_name, args[0], num_units, 
                    num_hidden_layers, proj_type, args[4]))
    else:
        json_file = set_logger('%s_%s_%sunits_%sh_%s_%s' % \
                (experiment_name, args[0], num_units, 
                    num_hidden_layers, proj_type, args[4]))
    sense_lf = l.SecondLevelLabel()
    relation_list_list = [extract_implicit_relations(dir, sense_lf) 
            for dir in dir_list]

    wbm = _get_wbm(num_units)
    data_list = []
    for relation_list in relation_list_list:
        data = prep_tree_lstm_serrated_matrix_relations(
                relation_list, wbm, 35)
        data_list.append(data)

    label_vectors, label_alphabet = \
            util.label_vectorize(relation_list_list, sense_lf)
    data_triplet = DataTriplet(
            data_list, [[x] for x in label_vectors], [label_alphabet])

    num_reps = 15
    num_hidden_unit_list = [0] if num_hidden_layers == 0 \
            else [50, 200, 300, 400] 
    for num_hidden_units in num_hidden_unit_list:
        _net_experiment_lstm_helper(json_file, data_triplet, wbm, num_reps, 
                BinaryTreeLSTM,
                num_hidden_layers=num_hidden_layers, 
                num_hidden_units=num_hidden_units, 
                use_hinge=False, 
                proj_type=proj_type,
                use_bl=use_bl,
                arg_shared_weights=arg_shared_weights
                )
Ejemplo n.º 10
0
def net_experiment_tlstm(dir_list, args):
    """Tree-structured LSTM experiment version 2

    This version is different from net_experiment_tree_lstm in that 
    you use BinaryForestLSTM instead. The tree structures along with 
    the data themselvesa are encoded internally in the model. 
    This way, we can be sure that the algorithm is doing the right thing.
    Scan loop is very slow as well so I hope that this works better. 

ipython experiments.py net_experiment_tlstm l 50 1 mean_pool
    """
    assert(len(args) >= 4)

    if args[0] == 'bl':
        use_bl = True
        raise ValueError('bilinear model is not supported yet')
    elif args[0] == 'l':
        use_bl = False
    else:
        raise ValueError('First argument must be l or bl')
    num_units = int(args[1])
    num_hidden_layers = int(args[2])
    proj_type = args[3]
    if len(args) == 5 and args[4] == 'left':
        all_left_branching = True
        print 'use left branching trees'
    else:
        all_left_branching = False

    experiment_name = sys._getframe().f_code.co_name    
    if all_left_branching:
        name_file = '%s_%s_%sunits_%sh_%s_left' % \
                (experiment_name, args[0], num_units, 
                    num_hidden_layers, proj_type)
        json_file = set_logger(name_file)
        model_file = name_file + '.model'
    else:
        name_file = '%s_%s_%sunits_%sh_%s' % \
                (experiment_name, args[0], num_units, 
                    num_hidden_layers, proj_type)
        json_file = set_logger(name_file)
        model_file = name_file + '.model'
    sense_lf = l.SecondLevelLabel()
    relation_list_list = [extract_implicit_relations(dir, sense_lf)[0:5]
            for dir in dir_list]
    wbm = _get_wbm(num_units)

    data_list = []
    for relation_list in relation_list_list:
        data = prep_trees(relation_list)
        data_list.append(data)

    label_vectors, label_alphabet = \
            util.label_vectorize(relation_list_list, sense_lf)
    data_triplet = DataTriplet(
            data_list, [[x] for x in label_vectors], [label_alphabet])

    num_reps = 15
    num_hidden_unit_list = [0] if num_hidden_layers == 0 \
            else [50, 200, 300, 400] 
    for num_hidden_units in num_hidden_unit_list:
        _net_experiment_tlstm_helper(json_file, model_file,
                data_triplet, wbm, num_reps, 
                num_hidden_layers=num_hidden_layers, 
                num_hidden_units=num_hidden_units, 
                use_hinge=False, 
                proj_type=proj_type,
                )