Python make_data_cv示例

示例#1

def check_train_error():
    print '...load the expriment data set'
    data = cPickle.load(open('./data/experiment_dataset2'))
    docs, type2id, pop2id, word2id, embedding, rand_embedding = data

    print '... construct the train/valid/test set'
    test_docs = docs[:10000]
    datasets = make_data_cv(test_docs, 0, word2id, max_l=1000, filter_h=5)

    print '....Load model parameters'
    # load the trained parameters
    params = load_model('./data/pop_model.pkl')

    print '....start test the model'
    # construct the model
    rs = construct_model(params, datasets, filter_hs=[3, 4, 5], batch_size=200)
    rs["pop2id"] = pop2id

    with open('./rs.json', 'w') as r:
        json.dump(rs, r)

示例#2

文件： deconv.py 项目： Tskatom/Protest_Event_Encoder

def check_train_error():
    print '...load the expriment data set'
    data = cPickle.load(open('./data/experiment_dataset2'))
    docs, type2id, pop2id, word2id, embedding, rand_embedding = data
    
    print '... construct the train/valid/test set'
    test_docs = docs[:10000]
    datasets = make_data_cv(test_docs, 0, word2id, max_l=1000, filter_h=5)

    print '....Load model parameters'
    # load the trained parameters
    params= load_model('./data/pop_model.pkl')

    print '....start test the model'
    # construct the model
    rs = construct_model(params, datasets, filter_hs=[3,4,5], batch_size=200)
    rs["pop2id"] = pop2id

    with open('./rs.json', 'w') as r:
        json.dump(rs, r)

示例#3

def main():
    print '...load the expriment data set'
    data = cPickle.load(open('./data/experiment_dataset2'))
    docs, type2id, pop2id, word2id, embedding, rand_embedding = data

    print '... construct the train/valid/test set'
    test_docs = docs[:10000]
    datasets = make_data_cv(test_docs, 0, word2id, max_l=1000, filter_h=5)

    print '... construct ngrams'
    ns = [3, 4, 5]
    n_grams = find_ngrams(datasets[0])
    print '....Load model parameters'
    # load the trained parameters
    params = load_model('./data/pop_model.pkl')

    print '....start test the model'
    # construct the model
    # dump the result
    with open("./data/top_ngrams.pkl", 'wb') as tn:
        for n, n_gram in enumerate(n_grams):
            n_gram_texts = get_top_features(n_gram, params, word2id, n)
            cPickle.dump(n_gram_texts, tn)

示例#4

文件： explore_feature.py 项目： Tskatom/Protest_Event_Encoder

def main():
    print '...load the expriment data set'
    data = cPickle.load(open('./data/experiment_dataset2'))
    docs, type2id, pop2id, word2id, embedding, rand_embedding = data
    
    print '... construct the train/valid/test set'
    test_docs = docs[:10000]
    datasets = make_data_cv(test_docs, 0, word2id, max_l=1000, filter_h=5)

    print '... construct ngrams'
    ns = [3, 4, 5]
    n_grams = find_ngrams(datasets[0])
    print '....Load model parameters'
    # load the trained parameters
    params= load_model('./data/pop_model.pkl')

    print '....start test the model'
    # construct the model
    # dump the result
    with open("./data/top_ngrams.pkl", 'wb') as tn:
        for n, n_gram in enumerate(n_grams):
            n_gram_texts = get_top_features(n_gram, params, word2id, n)
            cPickle.dump(n_gram_texts, tn)