예제 #1
0
def evaluation():
    _, test_data, _, _ = preprocessor.load_data()
    models = ['random', 'frequent']
    for model in models:
        _, test_data, _, _ = preprocessor.load_data()
        perform, fname = eval_model(test_data, model)
        print('MODEL:', model, ', accuracy:',perform)
        print('Result is save at', fname)
        print('')
예제 #2
0
def test():
    import targetid
    import etri
    
    _, test_data, _, _ = preprocessor.load_data()
    test = test_data
    testing = [  test[1] ]
    fid_result = frame_identifier(testing, 'random')
    print(fid_result)
    _, test_data, _, _ = preprocessor.load_data()
    test = test_data
    testing = [  test[1] ]
    fid_result = frame_identifier(testing, 'frequent')
    print(fid_result)
        self.scores = {}
        self.score(val_data, user_ls)
        vbf_dict = self.scores
        results = {u:evaluate_threshold(self.thresh[u], vbf_dict[u]) 
                   for u in (user_ls if (user_ls != None) 
                             else self.thresh.keys())}
        return results


if __name__=='__main__':
    from CV import CV
    from preprocessor import split_samples, load_data, filter_users_val

    P.np.seterr(all='ignore')

    all_data, pkd = filter_users_val(split_samples(load_data()))

    for u in all_data.keys():
        if all_data[u] == []:
            del all_data[u]
            del pkd[u]


    gbfa = CV(lambda: GammaBFAuth(all_data), 
              all_data, 
              pkd)

    with open('./bf_result.csv', 'rw+') as res_file:
        result_writer = csv.writer(res_file)
        result_writer.writerow(['user',
                                'CV_IPR', 'CV_FRR', 'CV_GT', 'CV_IT'])
예제 #4
0
    print('No GPU found. Please use a GPU to train your neural network.')

# Number of words in a sequence.
# This parameter is used both for training and generating.
sequence_length = args.sequence_len

try:
    print("Loading the model...")
    _, vocab_to_int, int_to_vocab, token_dict = preprocessor.load_preprocess()
    trained_rnn = preprocessor.load_model('./save/trained_rnn')

except:
    print("Unable to load a checkpoint. Input data needs to be preprocessed.")

    data_dir = './data/Seinfeld_Scripts.txt'
    text = preprocessor.load_data(data_dir)

    int_text, vocab_to_int, int_to_vocab, token_dict = preprocessor.preprocess_and_save_data(
        data_dir, token_lookup, create_lookup_tables)

    batch_size = args.batch_size  # 32

    train_loader = batch_data(int_text, sequence_length, batch_size)

    print("Training...")

    #num_epochs = 20
    num_epochs = args.n_epochs
    learning_rate = args.learning_rate  #0.0001
    vocab_size = len(vocab_to_int)
    output_size = vocab_size
if __name__=='__main__':
    """
    test_data = {'a' : [coll.defaultdict(list, {'aa' : range(10,100,10),
                                                'ab' : range(1,10)}),
                        coll.defaultdict(list, {'ac' : range(1,10,2),
                                                'aa' : [2,2,2]}),
                        coll.defaultdict(list, {'ad' : range(10, 20, 3)})
                    ],
                 'b' : [coll.defaultdict(list, {'ba' : range(1,20,3),
                                                'bb' : range(1,15)}),
                        coll.defaultdict(list, {'bc' : range(2,20,4)}),
                        coll.defaultdict(list, {'bd' : range(50,300,150)})
                    ],
    }
    """
    test_data = pp.split_samples(pp.load_data())
    for u in test_data.keys():
        if u not in {'9999999','SERLHOU'}:
            del test_data[u]
    print test_data.keys()
    test_cv = CV(DensityAuth, test_data)
    '''

    for i in test_cv.partition_data('shit', test_data['a'], 1):
        f**k.pprint(i)
    '''

    for i in test_cv.validate():
        pass
    print "DONESKI"
예제 #6
0
from keras.models import load_model
from preprocessor import load_data
import numpy as np

_, VOC = load_data(limit=100)
r_VOC = dict(zip(VOC.values(), VOC.keys()))

model = load_model('model_title_based.h5')

print(VOC)

title = '初晴落景      '
title_int = []
for t in title:
    i = VOC.get(t)
    if i is None:
        raise Exception('{} is not in VOC'.format(t))
    title_int.append(i)
title_int = np.array([title_int])
print(title_int)

content = '^'
c_index = 1

while content[-1] != '$' and len(content) < 74:
    c_content = content
    for i in range(74 - c_index):
        c_content += ' '

    p = [VOC[i] for i in c_content]
    x = np.array([p])
예제 #7
0
        print('')
        print('interrupted by user')
        pass


def run():
    model = build_graph()
    train(model, x_train, y_train)
    model.save(MODEL_FILE_NAME)


if __name__ == '__main__':
    
    EPOCHS = 80
    BATCH_SIZE = 8
    NETWORK_SIZE = (2, 256)
    DATA_SIZE = 10000
    CELL_TYPE = 'gru'

    i = 3
    MODEL_FILE_NAME = './models/model'+str(i)+'.h5'
    CHECKPOINT_WEIGHTS_DIR = './weights/'+str(i)
    file_path = './data/poetry_add'+str(i)+'.txt'
    (x_train, y_train), VOC = load_data(file_path)
    VOC_SIZE = len(VOC)

    print('training set information:')
    print('vocabulary size:', VOC_SIZE)
    print('X shape:', x_train.shape)
    print('Y shape:', y_train.shape)
    run()
예제 #8
0
def load_data():
    training, test, training_fe = preprocessor.load_data()
    #result = training + test
    result = training + test
    return result
예제 #9
0
                 dest='mode',
                 type='choice',
                 choices=['train', 'test', 'parsing'],
                 default='test')

optpr.mode = 'train'

# In[3]:

print('GPU:', torch.cuda.get_device_name(0),
      ', # of gpu:(' + str(torch.cuda.device_count()) + ')')
print('Torch Version:', torch.version.cuda)

# In[4]:

training_data, test_data, dev_data, exemplar_data = preprocessor.load_data()

# In[5]:

preprocessor.data_stat()

# In[6]:

configuration = {
    'token_dim': 60,
    'hidden_dim': 64,
    'pos_dim': 4,
    'lu_dim': 64,
    'lu_pos_dim': 5,
    'lstm_input_dim': 64,
    'lstm_dim': 64,