def evaluate(args, setup):
directory = os.path.abspath(args['<location>'])
sub_dirs = [os.path.join(directory, sub_dir)
for sub_dir in os.listdir(directory)]
best_loss = np.inf
best_exp = ''
for sub_dir in sub_dirs:
if not os.path.isdir(sub_dir):
continue
print '>>> checking %s' % sub_dir
os.chdir(sub_dir)
cps = contrib.find_checkpoints(sub_dir)
if cps:
with gzip.open(cps[-1], 'rb') as fp:
trainer = cPickle.load(fp)
print trainer.best_loss
if trainer.best_loss < best_loss:
best_loss = trainer.best_loss
best_exp = sub_dir
else:
print '>>> no checkpoints found in this folder.'
r_string = '>>> found the best experiment in\n>' \
'>> %s\n>>> with a validation loss of %f' % (best_exp, best_loss)
print r_string
with open(os.path.join(directory, 'result.txt'), 'w') as result:
result.write(r_string)
def evaluate(args):
dir = os.path.abspath(args['<location>'])
sub_dirs = [os.path.join(dir, sub_dir) for sub_dir in os.listdir(dir)]
best_loss = np.inf
best_exp = ''
for sub_dir in sub_dirs:
if not os.path.isdir(sub_dir):
continue
os.chdir(sub_dir)
cps = contrib.find_checkpoints('.')
if cps:
print '>>> checking %s' % sub_dir
with gzip.open(cps[-1], 'rb') as fp:
try:
trainer = cPickle.load(fp)
if trainer.best_loss < best_loss:
best_loss = trainer.best_loss
best_exp = sub_dir
except:
print '>>> detected corrupted checkpoint'
r_string = '>>> found the best experiment in\n>>> %s\n>>> with a validation loss of %f' % (
best_exp, best_loss)
print r_string
with open(os.path.join(dir, 'result.txt'), 'w') as result:
result.write(r_string)
def evaluate(args):
dir = os.path.abspath(args['<location>'])
sub_dirs = [os.path.join(dir, sub_dir)
for sub_dir in os.listdir(dir)]
best_loss = np.inf
best_exp = ''
for sub_dir in sub_dirs:
if not os.path.isdir(sub_dir):
continue
os.chdir(sub_dir)
cps = contrib.find_checkpoints('.')
if cps:
print '>>> checking %s' %sub_dir
with gzip.open(cps[-1], 'rb') as fp:
try:
trainer = cPickle.load(fp)
if trainer.best_loss < best_loss:
best_loss = trainer.best_loss
best_exp = sub_dir
except:
print '>>> detected corrupted checkpoint'
r_string = '>>> found the best experiment in\n>>> %s\n>>> with a validation loss of %f' %(best_exp, best_loss)
print r_string
with open(os.path.join(dir, 'result.txt'),'w') as result:
result.write(r_string)
def make_trainer(pars, mod, data):
cps = contrib.find_checkpoints('.')
if cps:
with gzip.open(cps[-1], 'rb') as fp:
trainer = cPickle.load(fp)
else:
trainer = mod.new_trainer(pars, data)
return trainer
def make_trainer(pars, mod, data):
cps = contrib.find_checkpoints('.')
if cps:
with gzip.open(cps[-1], 'rb') as fp:
trainer = cPickle.load(fp)
mod.make_data_dict(trainer, data)
else:
trainer = mod.new_trainer(pars, data)
return trainer
def make_trainer(pars, setup, data):
cps = contrib.find_checkpoints('.')
if cps:
print '>>> reloading trainer from checkpoint'
with gzip.open(cps[-1], 'rb') as fp:
trainer = cPickle.load(fp)
trainer.data = data
else:
print '>>> creating new trainer'
trainer = setup.new_trainer(pars, data)
return trainer
def visualize_tsne(args):
model_dir = os.path.abspath(args['<model>'])
data_dir = os.path.abspath(args['<data>'])
os.chdir(model_dir)
cps = contrib.find_checkpoints('.')
if cps:
with gzip.open(cps[-1], 'rb') as fp:
trainer = cPickle.load(fp)
trainer.model.parameters.data[...] = trainer.best_pars
data = h5.File(data_dir,'r')
TX = data['test_set/test_set'][:5000]
TZ = data['test_labels/real_test_labels'][:5000]
TZ = one_hot(TZ,13)
print 'data loaded.'
if args['<mode>'] == 'cnn':
f_transformed = trainer.model.function(['inpt'],'mlp-layer-2-inpt')
print 'transform-function generated.'
data = minibatches(TX, trainer.model.batch_size, 0)
trans_TX = np.concatenate([f_transformed(element) for element in data], axis=0)
else:
f_transformed = trainer.model.function(['inpt'],'layer-2-inpt')
print 'transform-function generated.'
trans_TX = f_transformed(TX)
trans_TX = np.array(trans_TX, dtype=np.float32)
print 'data transformed'
trans_n_input = trans_TX.shape[1]
trans_tsne = Tsne(trans_n_input, 2, perplexity=5)
print 'TSNE initialized.'
trans_TX_r = trans_tsne.fit_transform(trans_TX)
print 'data TSNEd'
fig = plt.figure(figsize=(16, 16))
ax = fig.add_subplot(111)
TZ_am = TZ.argmax(axis=1)
ax.scatter(trans_TX_r[TZ_am==0, 0], trans_TX_r[TZ_am==0, 1], c='g', lw=0, alpha=1, s=100, marker='o')
ax.scatter(trans_TX_r[TZ_am==1, 0], trans_TX_r[TZ_am==1, 1], c='b', lw=0, alpha=1, s=100, marker='v')
ax.scatter(trans_TX_r[TZ_am==2, 0], trans_TX_r[TZ_am==2, 1], c='yellow', lw=0, alpha=1, s=100, marker='^')
ax.scatter(trans_TX_r[TZ_am==3, 0], trans_TX_r[TZ_am==3, 1], c='r', lw=0, alpha=1, s=100, marker='<')
ax.scatter(trans_TX_r[TZ_am==4, 0], trans_TX_r[TZ_am==4, 1], c='g', lw=0, alpha=1, s=100, marker='>')
ax.scatter(trans_TX_r[TZ_am==5, 0], trans_TX_r[TZ_am==5, 1], c='m', lw=0, alpha=1, s=100, marker='8')
ax.scatter(trans_TX_r[TZ_am==6, 0], trans_TX_r[TZ_am==6, 1], c='crimson', lw=0, alpha=1, s=100, marker='s')
ax.scatter(trans_TX_r[TZ_am==7, 0], trans_TX_r[TZ_am==7, 1], c='lawngreen', lw=0, alpha=1, s=100, marker='p')
ax.scatter(trans_TX_r[TZ_am==8, 0], trans_TX_r[TZ_am==8, 1], c='gold', lw=0, alpha=1, s=100, marker='*')
ax.scatter(trans_TX_r[TZ_am==9, 0], trans_TX_r[TZ_am==9, 1], c='darkorange', lw=0, alpha=1, s=100, marker='h')
ax.scatter(trans_TX_r[TZ_am==10, 0], trans_TX_r[TZ_am==10, 1], c='k', lw=0, alpha=1, s=100, marker='H')
ax.scatter(trans_TX_r[TZ_am==11, 0], trans_TX_r[TZ_am==11, 1], c='magenta', lw=0, alpha=1, s=100, marker='d')
ax.scatter(trans_TX_r[TZ_am==12, 0], trans_TX_r[TZ_am==12, 1], c='turquoise', lw=0, alpha=1, s=100, marker='D')
plt.savefig(os.path.join('/nthome/maugust/thesis',args['<output>']))
def evaluate_mlp(args):
dir = os.path.abspath(args['<location>'])
data = os.path.abspath(args['<data>'])
mode = args['<mode>']
os.chdir(dir)
cps = contrib.find_checkpoints('.')
if cps:
with gzip.open(cps[-1], 'rb') as fp:
trainer = cPickle.load(fp)
trainer.model.parameters.data[...] = trainer.best_pars
cPickle.dump(trainer.best_pars, open('best_pars.pkl','wb'))
data = h5.File(data,'r')
TX = data['test_set/test_set']
TA = data['test_annotations/test_annotations']
TZ = data['test_labels/real_test_labels']
TZ = one_hot(TZ,13)
n_wrong = 1 - T.eq(T.argmax(trainer.model.exprs['output'], axis=1),
T.argmax(trainer.model.exprs['target'], axis=1)).mean()
f_n_wrong = trainer.model.function(['inpt', 'target'], n_wrong)
f_pos = T.mean(T.neq(T.argmax(trainer.model.exprs['output'], axis=1),0) * T.eq(T.argmax(trainer.model.exprs['target'], axis=1), 0))
f_f_pos = trainer.model.function(['inpt', 'target'], f_pos)
f_neg = T.mean(T.eq(T.argmax(trainer.model.exprs['output'], axis=1),0) * T.neq(T.argmax(trainer.model.exprs['target'], axis=1), 0))
f_f_neg = trainer.model.function(['inpt', 'target'], f_neg)
if mode == 'cnn':
print 'using cnn model'
emp_loss = trainer.model.apply_minibatches_function(f_n_wrong,TX,TZ)
f_p = trainer.model.apply_minibatches_function(f_f_pos,TX,TZ)
f_n = trainer.model.apply_minibatches_function(f_f_neg,TX,TZ)
else:
emp_loss = f_n_wrong(TX,TZ)
f_p = f_f_pos(TX,TZ)
f_n = f_f_neg(TX,TZ)
P_pos = np.argmax(trainer.model.predict(TX),axis=1)
Z_pos = np.argmax(TZ, axis=1)
neighbour_fails = .0
relevant_fails = 0
for i in np.arange(len(P_pos)):
if P_pos[i] > 0 and Z_pos[i] > 0 and P_pos[i] != Z_pos[i]:
relevant_fails += 1
if is_neighbour(P_pos[i],Z_pos[i]):
neighbour_fails += 1
if not relevant_fails == 0:
neighbour_fails /= relevant_fails
emp_loss_s = 'model achieved %f%% classification error on the test set' %emp_loss
f_p_s = '\nmodel achieved %f%% false positives on the test set' %f_p
f_n_s = '\nmodel achieved %f%% false negatives on the test set' %f_n
neigh_s = '\nmodel achieved %f%% neighbour misspredictions on the test set' %neighbour_fails
print emp_loss_s
print f_p_s
print f_n_s
print neigh_s
with open(os.path.join(dir,'eval_result.txt'),'w') as f:
f.write(emp_loss_s)
f.write(f_p_s)
f.write(f_n_s)
f.write(neigh_s)
return 0
'''indices = np.random.rand(50) * 10000
