예제 #1
0
def evaluate(args, setup):
    directory = os.path.abspath(args['<location>'])
    sub_dirs = [os.path.join(directory, sub_dir)
                for sub_dir in os.listdir(directory)]
    best_loss = np.inf
    best_exp = ''

    for sub_dir in sub_dirs:
        if not os.path.isdir(sub_dir):
            continue
        print '>>> checking %s' % sub_dir
        os.chdir(sub_dir)
        cps = contrib.find_checkpoints(sub_dir)
        if cps:
            with gzip.open(cps[-1], 'rb') as fp:
                trainer = cPickle.load(fp)
                print trainer.best_loss
                if trainer.best_loss < best_loss:
                    best_loss = trainer.best_loss
                    best_exp = sub_dir
        else:
            print '>>> no checkpoints found in this folder.'

    r_string = '>>> found the best experiment in\n>' \
               '>> %s\n>>> with a validation loss of %f' % (best_exp, best_loss)
    print r_string
    with open(os.path.join(directory, 'result.txt'), 'w') as result:
        result.write(r_string)
예제 #2
0
def evaluate(args):
    dir = os.path.abspath(args['<location>'])
    sub_dirs = [os.path.join(dir, sub_dir) for sub_dir in os.listdir(dir)]
    best_loss = np.inf
    best_exp = ''

    for sub_dir in sub_dirs:
        if not os.path.isdir(sub_dir):
            continue
        os.chdir(sub_dir)
        cps = contrib.find_checkpoints('.')
        if cps:
            print '>>> checking %s' % sub_dir
            with gzip.open(cps[-1], 'rb') as fp:
                try:
                    trainer = cPickle.load(fp)
                    if trainer.best_loss < best_loss:
                        best_loss = trainer.best_loss
                        best_exp = sub_dir
                except:
                    print '>>> detected corrupted checkpoint'

    r_string = '>>> found the best experiment in\n>>> %s\n>>> with a validation loss of %f' % (
        best_exp, best_loss)
    print r_string
    with open(os.path.join(dir, 'result.txt'), 'w') as result:
        result.write(r_string)
예제 #3
0
파일: alc.py 프로젝트: m0r17z/alchemie
def evaluate(args):
    dir = os.path.abspath(args['<location>'])
    sub_dirs = [os.path.join(dir, sub_dir)
                       for sub_dir in os.listdir(dir)]
    best_loss = np.inf
    best_exp = ''

    for sub_dir in sub_dirs:
        if not os.path.isdir(sub_dir):
            continue
        os.chdir(sub_dir)
        cps = contrib.find_checkpoints('.')
        if cps:
            print '>>> checking %s' %sub_dir
	    with gzip.open(cps[-1], 'rb') as fp:
                try:
			trainer = cPickle.load(fp)
                	if trainer.best_loss < best_loss:
                    		best_loss = trainer.best_loss
                    		best_exp = sub_dir
		except:
			print '>>> detected corrupted checkpoint'

    r_string = '>>> found the best experiment in\n>>> %s\n>>> with a validation loss of %f' %(best_exp, best_loss)
    print r_string
    with open(os.path.join(dir, 'result.txt'),'w') as result:
        result.write(r_string)
예제 #4
0
파일: alc.py 프로젝트: kosklain/alchemie
def make_trainer(pars, mod, data):
    cps = contrib.find_checkpoints('.')
    if cps:
        with gzip.open(cps[-1], 'rb') as fp:
            trainer = cPickle.load(fp)
    else:
        trainer = mod.new_trainer(pars, data)

    return trainer
예제 #5
0
def make_trainer(pars, mod, data):
    cps = contrib.find_checkpoints('.')
    if cps:
        with gzip.open(cps[-1], 'rb') as fp:
            trainer = cPickle.load(fp)
            mod.make_data_dict(trainer, data)
    else:
        trainer = mod.new_trainer(pars, data)

    return trainer
예제 #6
0
def make_trainer(pars, setup, data):
    cps = contrib.find_checkpoints('.')
    if cps:
        print '>>> reloading trainer from checkpoint'
        with gzip.open(cps[-1], 'rb') as fp:
            trainer = cPickle.load(fp)
            trainer.data = data
    else:
        print '>>> creating new trainer'
        trainer = setup.new_trainer(pars, data)

    return trainer
예제 #7
0
파일: tsne_model.py 프로젝트: m0r17z/thesis
def visualize_tsne(args):
    model_dir = os.path.abspath(args['<model>'])
    data_dir = os.path.abspath(args['<data>'])
    os.chdir(model_dir)
    cps = contrib.find_checkpoints('.')

    if cps:
        with gzip.open(cps[-1], 'rb') as fp:
                trainer = cPickle.load(fp)
                trainer.model.parameters.data[...] = trainer.best_pars
                data = h5.File(data_dir,'r')
                TX = data['test_set/test_set'][:5000]
                TZ = data['test_labels/real_test_labels'][:5000]
                TZ = one_hot(TZ,13)
                print 'data loaded.'

                if args['<mode>'] == 'cnn':
                    f_transformed = trainer.model.function(['inpt'],'mlp-layer-2-inpt')
                    print 'transform-function generated.'
                    data = minibatches(TX, trainer.model.batch_size, 0)
                    trans_TX = np.concatenate([f_transformed(element) for element in data], axis=0)
                else:
                    f_transformed = trainer.model.function(['inpt'],'layer-2-inpt')
                    print 'transform-function generated.'
                    trans_TX = f_transformed(TX)

                trans_TX = np.array(trans_TX, dtype=np.float32)
                print 'data transformed'
                trans_n_input = trans_TX.shape[1]
                trans_tsne = Tsne(trans_n_input, 2, perplexity=5)
                print 'TSNE initialized.'
                trans_TX_r = trans_tsne.fit_transform(trans_TX)
                print 'data TSNEd'

                fig = plt.figure(figsize=(16, 16))
                ax = fig.add_subplot(111)
                TZ_am = TZ.argmax(axis=1)
                ax.scatter(trans_TX_r[TZ_am==0, 0], trans_TX_r[TZ_am==0, 1], c='g', lw=0, alpha=1, s=100, marker='o')
                ax.scatter(trans_TX_r[TZ_am==1, 0], trans_TX_r[TZ_am==1, 1], c='b', lw=0, alpha=1, s=100, marker='v')
                ax.scatter(trans_TX_r[TZ_am==2, 0], trans_TX_r[TZ_am==2, 1], c='yellow', lw=0, alpha=1, s=100, marker='^')
                ax.scatter(trans_TX_r[TZ_am==3, 0], trans_TX_r[TZ_am==3, 1], c='r', lw=0, alpha=1, s=100, marker='<')
                ax.scatter(trans_TX_r[TZ_am==4, 0], trans_TX_r[TZ_am==4, 1], c='g', lw=0, alpha=1, s=100, marker='>')
                ax.scatter(trans_TX_r[TZ_am==5, 0], trans_TX_r[TZ_am==5, 1], c='m', lw=0, alpha=1, s=100, marker='8')
                ax.scatter(trans_TX_r[TZ_am==6, 0], trans_TX_r[TZ_am==6, 1], c='crimson', lw=0, alpha=1, s=100, marker='s')
                ax.scatter(trans_TX_r[TZ_am==7, 0], trans_TX_r[TZ_am==7, 1], c='lawngreen', lw=0, alpha=1, s=100, marker='p')
                ax.scatter(trans_TX_r[TZ_am==8, 0], trans_TX_r[TZ_am==8, 1], c='gold', lw=0, alpha=1, s=100, marker='*')
                ax.scatter(trans_TX_r[TZ_am==9, 0], trans_TX_r[TZ_am==9, 1], c='darkorange', lw=0, alpha=1, s=100, marker='h')
                ax.scatter(trans_TX_r[TZ_am==10, 0], trans_TX_r[TZ_am==10, 1], c='k', lw=0, alpha=1, s=100, marker='H')
                ax.scatter(trans_TX_r[TZ_am==11, 0], trans_TX_r[TZ_am==11, 1], c='magenta', lw=0, alpha=1, s=100, marker='d')
                ax.scatter(trans_TX_r[TZ_am==12, 0], trans_TX_r[TZ_am==12, 1], c='turquoise', lw=0, alpha=1, s=100, marker='D')
                plt.savefig(os.path.join('/nthome/maugust/thesis',args['<output>']))
예제 #8
0
파일: eval_model.py 프로젝트: m0r17z/thesis
def evaluate_mlp(args):
    dir = os.path.abspath(args['<location>'])
    data = os.path.abspath(args['<data>'])
    mode = args['<mode>']
    os.chdir(dir)
    cps = contrib.find_checkpoints('.')

    if cps:
        with gzip.open(cps[-1], 'rb') as fp:
            trainer = cPickle.load(fp)
            trainer.model.parameters.data[...] = trainer.best_pars
            cPickle.dump(trainer.best_pars, open('best_pars.pkl','wb'))
            data = h5.File(data,'r')
            TX = data['test_set/test_set']
            TA = data['test_annotations/test_annotations']
            TZ = data['test_labels/real_test_labels']
            TZ = one_hot(TZ,13)

            n_wrong = 1 - T.eq(T.argmax(trainer.model.exprs['output'], axis=1),
                               T.argmax(trainer.model.exprs['target'], axis=1)).mean()
            f_n_wrong = trainer.model.function(['inpt', 'target'], n_wrong)

            f_pos = T.mean(T.neq(T.argmax(trainer.model.exprs['output'], axis=1),0) * T.eq(T.argmax(trainer.model.exprs['target'], axis=1), 0))
            f_f_pos = trainer.model.function(['inpt', 'target'], f_pos)

            f_neg = T.mean(T.eq(T.argmax(trainer.model.exprs['output'], axis=1),0) * T.neq(T.argmax(trainer.model.exprs['target'], axis=1), 0))
            f_f_neg = trainer.model.function(['inpt', 'target'], f_neg)

            if mode == 'cnn':
                print 'using cnn model'
                emp_loss = trainer.model.apply_minibatches_function(f_n_wrong,TX,TZ)
                f_p = trainer.model.apply_minibatches_function(f_f_pos,TX,TZ)
                f_n = trainer.model.apply_minibatches_function(f_f_neg,TX,TZ)
            else:
                emp_loss = f_n_wrong(TX,TZ)
                f_p = f_f_pos(TX,TZ)
                f_n = f_f_neg(TX,TZ)

            P_pos = np.argmax(trainer.model.predict(TX),axis=1)
            Z_pos = np.argmax(TZ, axis=1)

            neighbour_fails = .0
            relevant_fails = 0

            for i in np.arange(len(P_pos)):
                if P_pos[i] > 0 and Z_pos[i] > 0 and P_pos[i] != Z_pos[i]:
                    relevant_fails += 1
                    if is_neighbour(P_pos[i],Z_pos[i]):
                        neighbour_fails += 1

            if not relevant_fails == 0:
		neighbour_fails /= relevant_fails


            emp_loss_s = 'model achieved %f%% classification error on the test set' %emp_loss
            f_p_s = '\nmodel achieved %f%% false positives on the test set' %f_p
            f_n_s = '\nmodel achieved %f%% false negatives on the test set' %f_n
            neigh_s = '\nmodel achieved %f%% neighbour misspredictions on the test set' %neighbour_fails

            print emp_loss_s
            print f_p_s
            print f_n_s
            print neigh_s
            with open(os.path.join(dir,'eval_result.txt'),'w') as f:
                f.write(emp_loss_s)
                f.write(f_p_s)
                f.write(f_n_s)
                f.write(neigh_s)

    return 0

    '''indices = np.random.rand(50) * 10000