def test_error(net, dp, output_restrict = None, batch = None):
if batch == None:
batch = dp.get_next_batch(128)
net.train_batch(batch.data, batch.labels, TEST)
print net.get_batch_information()
output = to_cpu(net.output)
labels = to_cpu(batch.labels)
correct1 = 0
correct5 = 0
if output_restrict != None:
output[output_restrict,:] = output[output_restrict,:] + 10
max_labels = numpy.argsort(output, 0)
for i in range(output.shape[1]):
if max_labels[999,i] == labels[i]:
correct1 += 1
correct5 += 1
elif max_labels[998,i] == labels[i]:
correct5 += 1
elif max_labels[997,i] == labels[i]:
correct5 += 1
elif max_labels[996,i] == labels[i]:
correct5 += 1
elif max_labels[995,i] == labels[i]:
correct5 += 1
return correct1, correct5, output.shape[1]
def train_data(net, batches, stat, train):
total_cases = 0
total_cost = 0
total_correct = 0
total_error = 0
N = len(batches)
order = range(N)
np.random.shuffle(order)
for n in xrange(N):
if train == TRAIN:
# batch = batches[n]
batch = batches[order[n]]
# batch = batches[np.random.randint(len(batches))]
else:
batch = batches[n]
net.train_batch(batch.data, batch.labels, train)
cost, correct, num_case = net.get_batch_information()
total_cases += num_case
total_correct += correct * num_case
total_cost += cost * num_case
if total_cases > 0:
total_error = (1. - 1.0 * total_correct / total_cases)
total_cost = (1.0 * total_cost / total_cases)
if not 'error' in stat:
stat['error'] = list()
stat['error'].append(total_error)
if not 'cost' in stat:
stat['cost'] = list()
stat['cost'].append(total_cost)
def train_data(net, batches, stat, train):
total_cases = 0
total_cost = 0
total_correct = 0
total_error = 0
N = len(batches)
order = range(N)
np.random.shuffle(order)
for n in xrange(N):
if train == TRAIN:
# batch = batches[n]
batch = batches[order[n]]
# batch = batches[np.random.randint(len(batches))]
else:
batch = batches[n]
net.train_batch(batch.data, batch.labels, train)
cost, correct, num_case = net.get_batch_information()
total_cases += num_case
total_correct += correct * num_case
total_cost += cost * num_case
if total_cases > 0:
total_error = (1. - 1.0*total_correct/total_cases)
total_cost = (1.0*total_cost/total_cases)
if not 'error' in stat:
stat['error'] = list()
stat['error'].append(total_error)
if not 'cost' in stat:
stat['cost'] = list()
stat['cost'].append(total_cost)
def train_single(net, num_epoch, train_batches, test_batches):
l = net.layers[-2]
M = l.weight.shape[0]
w_test = np.eye(11)
w_test = data_loader.copy_to_gpu(w_test)
if hasattr(net, 'stat') == False:
net.stat = dict()
for epoch in range(num_epoch):
train_cases = 0
train_cost = 0
train_correct = 0
train_error = 0
test_cases = 0
test_cost = 0
test_correct = 0
test_error = 0
N = len(train_batches)
order = range(N)
np.random.shuffle(order)
for i in range(N):
batch = train_batches[order[i]]
net.train_batch(batch.data, batch.labels, TRAIN)
cost, correct, num_case = net.get_batch_information()
train_cases += num_case
train_correct += correct * num_case
train_cost += cost * num_case
if l.epsW > 0:
normalize_conf_matrix(net)
for batch in test_batches:
w_noisy = l.weight
l.weight = w_test
net.train_batch(batch.data, batch.labels, TEST)
l.weight = w_noisy
cost, correct, num_case = net.get_batch_information()
test_cases += num_case
test_correct += correct * num_case
test_cost += cost * num_case
if train_cases > 0:
train_error = (1. - 1.0*train_correct/train_cases)
train_cost = (1.0*train_cost/train_cases)
if test_cases > 0:
test_error = (1. - 1.0*test_correct/test_cases)
test_cost = (1.0*test_cost/test_cases)
print '%d %0.3f %0.3f %0.3f %0.3f' % (epoch, train_cost, train_error, \
test_cost, test_error)
if net.stat.has_key('test-error') == False:
net.stat['test-error'] = list()
net.stat['test-error'].append(test_error)
def train_single(net, num_epoch, train_batches, test_batches):
l = net.layers[-2]
M = l.weight.shape[0]
w_test = np.eye(11)
w_test = data_loader.copy_to_gpu(w_test)
if hasattr(net, 'stat') == False:
net.stat = dict()
for epoch in range(num_epoch):
train_cases = 0
train_cost = 0
train_correct = 0
train_error = 0
test_cases = 0
test_cost = 0
test_correct = 0
test_error = 0
N = len(train_batches)
order = range(N)
np.random.shuffle(order)
for i in range(N):
batch = train_batches[order[i]]
net.train_batch(batch.data, batch.labels, TRAIN)
cost, correct, num_case = net.get_batch_information()
train_cases += num_case
train_correct += correct * num_case
train_cost += cost * num_case
if l.epsW > 0:
normalize_conf_matrix(net)
for batch in test_batches:
w_noisy = l.weight
l.weight = w_test
net.train_batch(batch.data, batch.labels, TEST)
l.weight = w_noisy
cost, correct, num_case = net.get_batch_information()
test_cases += num_case
test_correct += correct * num_case
test_cost += cost * num_case
if train_cases > 0:
train_error = (1. - 1.0 * train_correct / train_cases)
train_cost = (1.0 * train_cost / train_cases)
if test_cases > 0:
test_error = (1. - 1.0 * test_correct / test_cases)
test_cost = (1.0 * test_cost / test_cases)
print '%d %0.3f %0.3f %0.3f %0.3f' % (epoch, train_cost, train_error, \
test_cost, test_error)
if net.stat.has_key('test-error') == False:
net.stat['test-error'] = list()
net.stat['test-error'].append(test_error)
def get_scores(net, batches): batch_size = batches[0].data.shape[1] N = len(batches) * batch_size score = np.zeros(N) i = 0 for batch in batches: net.train_batch(batch.data, batch.labels, TEST) score[i:i+batch_size] = -1 * data_loader.copy_to_cpu(net.layers[-1].cost).reshape(batch_size) i += batch_size return score
def get_scores(net, batches):
batch_size = batches[0].data.shape[1]
N = len(batches) * batch_size
score = np.zeros(N)
i = 0
for batch in batches:
net.train_batch(batch.data, batch.labels, TEST)
score[i:i + batch_size] = -1 * data_loader.copy_to_cpu(
net.layers[-1].cost).reshape(batch_size)
i += batch_size
return score
def train(net, num_epoch, train_batches, test_batches):
for epoch in range(num_epoch):
time_sta = time.time()
for batch in train_batches:
net.train_batch(batch.data, batch.labels, TRAIN)
(cost, correct, n) = net.get_batch_information()
train_error = 1 - correct
train_cost = cost
for batch in test_batches:
net.train_batch(batch.data, batch.labels, TEST)
(cost, correct, n) = net.get_batch_information()
test_error = 1 - correct
test_cost = cost
print '%d %0.3f %0.3f %0.3f %0.3f' % (epoch, train_cost, train_error,
test_cost, test_error),
print(time.time() - time_sta)
def train(net, num_epoch, train_batches, test_batches):
for epoch in range(num_epoch):
time_sta = time.time()
for batch in train_batches:
net.train_batch(batch.data, batch.labels, TRAIN)
(cost, correct, n) = net.get_batch_information()
train_error = 1 - correct
train_cost = cost
for batch in test_batches:
net.train_batch(batch.data, batch.labels, TEST)
(cost, correct, n) = net.get_batch_information()
test_error = 1 - correct
test_cost = cost
print '%d %0.3f %0.3f %0.3f %0.3f' % (epoch, train_cost, train_error, test_cost, test_error),
print (time.time() - time_sta)
# show_stat(net, test_batches)
def train(net, num_epoch, train_batches, test_batches):
for epoch in range(num_epoch):
total_cases = 0
total_correct = 0
for batch in train_batches:
net.train_batch(batch.data, batch.labels, TRAIN)
cost, correct, num_case = net.get_batch_information()
total_cases += num_case
total_correct += correct * num_case
train_error = (1. - 1.0*total_correct/total_cases)
total_cases = 0
total_correct = 0
for batch in test_batches:
net.train_batch(batch.data, batch.labels, TEST)
cost, correct, num_case = net.get_batch_information()
total_cases += num_case
total_correct += correct * num_case
test_error = (1. - 1.0*total_correct/total_cases)
print 'epoch:', epoch, 'train-error:', train_error, \
'test-error:', test_error
N = int(np.floor(train_data.shape[1] / batch_size) * batch_size) order = range(N) np.random.shuffle(order) train_data = train_data[:,order].copy() train_labels = train_labels[order].copy() is_noise = is_noise[order] train_batches, test_batches = prepare_batches(train_data, train_labels, test_data, test_labels, batch_size) train(net, num_epoch, train_batches, test_batches) while True: N = int(np.floor(train_data.shape[1] / batch_size) * batch_size) cost = np.zeros(N) i = 0 for batch in train_batches: net.train_batch(batch.data, batch.labels, TEST) cost[i:i+batch_size] = copy_to_cpu(net.layers[-1].cost).reshape(batch_size) i += batch_size order = np.argsort(cost) order = order[0:70000] train_data2 = train_data[:,order] train_labels2 = train_labels[order] print '# num of noisy labels:', np.sum(is_noise[order]) N = int(np.floor(train_data2.shape[1] / batch_size) * batch_size) order = range(N) np.random.shuffle(order) train_data2 = train_data2[:,order].copy() train_labels2 = train_labels2[order].copy() train_batches2, test_batches2 = prepare_batches(train_data2, train_labels2, test_data, test_labels, batch_size) train(net, num_epoch, train_batches2, test_batches2)
def train(net, num_epoch, train_batches, noisy_batches, test_batches, lrate_beta):
l = net.layers[-2]
M = l.weight.shape[0]
assert M == l.weight.shape[1]
w_pure = data_loader.copy_to_gpu(np.eye(M))
w_test = np.eye(M)
if M == 11:
w_test[:10,10] = 0.1
w_test[10,10] = 0
w_test = data_loader.copy_to_gpu(w_test)
if hasattr(net, 'stat') == False:
net.stat = dict()
for epoch in range(num_epoch):
train_cases = 0
train_cost = 0
train_correct = 0
train_error = 0
test_cases = 0
test_cost = 0
test_correct = 0
test_error = 0
noisy_cases = 0
noisy_cost = 0
noisy_correct = 0
noisy_error = 0
N = len(train_batches) + len(noisy_batches)
order = range(N)
np.random.shuffle(order)
for i in range(N):
if order[i] < len(train_batches):
batch = train_batches[order[i]]
w_noisy = l.weight
l.weight = w_pure
epsW_noisy = l.epsW
l.epsW = 0
net.train_batch(batch.data, batch.labels, TRAIN)
l.weight = w_noisy
l.epsW = epsW_noisy
cost, correct, num_case = net.get_batch_information()
train_cases += num_case
train_correct += correct * num_case
train_cost += cost * num_case
else:
batch = noisy_batches[order[i] - len(train_batches)]
net.adjust_learning_rate(lrate_beta)
net.train_batch(batch.data, batch.labels, TRAIN)
net.adjust_learning_rate(1./lrate_beta)
if l.epsW > 0:
normalize_conf_matrix(net)
cost, correct, num_case = net.get_batch_information()
noisy_cases += num_case
noisy_correct += correct * num_case
noisy_cost += cost * num_case
for batch in test_batches:
w_noisy = l.weight
l.weight = w_test
net.train_batch(batch.data, batch.labels, TEST)
l.weight = w_noisy
cost, correct, num_case = net.get_batch_information()
test_cases += num_case
test_correct += correct * num_case
test_cost += cost * num_case
if train_cases > 0:
train_error = (1. - 1.0*train_correct/train_cases)
train_cost = (1.0*train_cost/train_cases)
if noisy_cases > 0:
noisy_error = (1. - 1.0*noisy_correct/noisy_cases)
noisy_cost = (1.0*noisy_cost/noisy_cases)
if test_cases > 0:
test_error = (1. - 1.0*test_correct/test_cases)
test_cost = (1.0*test_cost/test_cases)
print '%d %0.3f %0.3f %0.3f %0.3f %0.3f %0.3f' % (epoch, train_cost, train_error, \
noisy_cost, noisy_error, test_cost, test_error)
if net.stat.has_key('test-error') == False:
net.stat['test-error'] = list()
net.stat['test-error'].append(test_error)
def train(net, num_epoch, train_batches, noisy_batches, test_batches,
lrate_beta):
l = net.layers[-2]
M = l.weight.shape[0]
assert M == l.weight.shape[1]
w_pure = data_loader.copy_to_gpu(np.eye(M))
w_test = np.eye(M)
if M == 11:
w_test[:10, 10] = 0.1
w_test[10, 10] = 0
w_test = data_loader.copy_to_gpu(w_test)
if hasattr(net, 'stat') == False:
net.stat = dict()
for epoch in range(num_epoch):
train_cases = 0
train_cost = 0
train_correct = 0
train_error = 0
test_cases = 0
test_cost = 0
test_correct = 0
test_error = 0
noisy_cases = 0
noisy_cost = 0
noisy_correct = 0
noisy_error = 0
N = len(train_batches) + len(noisy_batches)
order = range(N)
np.random.shuffle(order)
for i in range(N):
if order[i] < len(train_batches):
batch = train_batches[order[i]]
w_noisy = l.weight
l.weight = w_pure
epsW_noisy = l.epsW
l.epsW = 0
net.train_batch(batch.data, batch.labels, TRAIN)
l.weight = w_noisy
l.epsW = epsW_noisy
cost, correct, num_case = net.get_batch_information()
train_cases += num_case
train_correct += correct * num_case
train_cost += cost * num_case
else:
batch = noisy_batches[order[i] - len(train_batches)]
net.adjust_learning_rate(lrate_beta)
net.train_batch(batch.data, batch.labels, TRAIN)
net.adjust_learning_rate(1. / lrate_beta)
if l.epsW > 0:
normalize_conf_matrix(net)
cost, correct, num_case = net.get_batch_information()
noisy_cases += num_case
noisy_correct += correct * num_case
noisy_cost += cost * num_case
for batch in test_batches:
w_noisy = l.weight
l.weight = w_test
net.train_batch(batch.data, batch.labels, TEST)
l.weight = w_noisy
cost, correct, num_case = net.get_batch_information()
test_cases += num_case
test_correct += correct * num_case
test_cost += cost * num_case
if train_cases > 0:
train_error = (1. - 1.0 * train_correct / train_cases)
train_cost = (1.0 * train_cost / train_cases)
if noisy_cases > 0:
noisy_error = (1. - 1.0 * noisy_correct / noisy_cases)
noisy_cost = (1.0 * noisy_cost / noisy_cases)
if test_cases > 0:
test_error = (1. - 1.0 * test_correct / test_cases)
test_cost = (1.0 * test_cost / test_cases)
print '%d %0.3f %0.3f %0.3f %0.3f %0.3f %0.3f' % (epoch, train_cost, train_error, \
noisy_cost, noisy_error, test_cost, test_error)
if net.stat.has_key('test-error') == False:
net.stat['test-error'] = list()
net.stat['test-error'].append(test_error)