def fprop(self, input, output, train=TRAIN):
#util.log_info("%s %s", input.shape, output.shape)
gpu_copy_to(input, output)
if PFout:
#if True:
print_matrix(output, self.name)
def fprop(self, input, output, train=TRAIN):
#util.log_info("%s %s", input.shape, output.shape)
gpu_copy_to(input, output)
if PFout:
#if True:
print_matrix(output, self.name)
def bprop(self, grad, input, output, outGrad):
if self.dropRate > 0.0:
gpu_copy_to(grad * self.dropMask, grad)
gpu_copy_to(transpose(dot(transpose(grad), self.weight.wt)), outGrad)
self.weight.set_grad(dot(grad, transpose(input)))
add_row_sum_to_vec(self.bias.grad, grad, alpha=0.0)
def bprop(self, grad, input, output, outGrad):
if self.dropRate > 0.0:
gpu_copy_to(grad * self.dropMask, grad)
gpu_copy_to(transpose(dot(transpose(grad), self.weight.wt)), outGrad)
self.weight.set_grad(dot(grad, transpose(input)))
add_row_sum_to_vec(self.bias.grad, grad, alpha=0.0)
def fprop(self, input, output, train=TRAIN):
gpu_copy_to(dot(self.weight.wt, input), output)
add_vec_to_rows(output, self.bias.wt)
if train == TEST:
if self.dropRate > 0.0:
output *= (1.0 - self.dropRate)
else:
if self.dropRate > 0.0:
self.dropMask = to_gpu(np.random.uniform(0, 1, output.size).astype(np.float32).reshape(output.shape))
bigger_than_scaler(self.dropMask, self.dropRate)
gpu_copy_to(output * self.dropMask, output)
if PFout:
print_matrix(output, self.name)
def fprop(self, input, output, train=TRAIN):
gpu_copy_to(dot(self.weight.wt, input), output)
add_vec_to_rows(output, self.bias.wt)
if train == TEST:
if self.dropRate > 0.0:
output *= (1.0 - self.dropRate)
else:
if self.dropRate > 0.0:
self.dropMask = to_gpu(np.random.uniform(0, 1, output.size).astype(np.float32).reshape(output.shape))
bigger_than_scaler(self.dropMask, self.dropRate)
gpu_copy_to(output * self.dropMask, output)
if PFout:
print_matrix(output, self.name)
def bprop(self, grad, input, output, outGrad):
self.weight.grad.fill(0)
self.bias.grad.fill(0)
# bprop to next layer
cudaconv2.convImgActs(grad, self.weight.wt, outGrad, self.img_size, self.img_size,
self.outputSize, -self.padding, self.stride, self.numColor, 1, 0.0, 1.0)
# bprop weight
cudaconv2.convWeightActs(input, grad, self.weight.grad, self.img_size, self.outputSize,
self.outputSize, self.filterSize, -self.padding, self.stride, self.numColor, 1, 0, 0, 1)
# bprop bias
gpu_copy_to(grad, self.tmp)
add_row_sum_to_vec(self.bias.grad, self.tmp)
def fprop(self, data, probs, train=TRAIN):
assert len(self.outputs) > 0, 'No outputs: uninitialized network!'
input = data
for i in range(len(self.layers)):
l = self.layers[i]
#if isinstance(l, layer.DataLayer):
# self.outputs[i] = input
#else:
l.fprop(input, self.outputs[i], train)
input = self.outputs[i]
# probs.shape = self.outputs[-1].shape
gpu_copy_to(self.outputs[-1], probs)
def fprop(self, data, probs, train=TRAIN):
assert len(self.outputs) > 0, 'No outputs: uninitialized network!'
input = data
for i in range(len(self.layers)):
l = self.layers[i]
#if isinstance(l, layer.DataLayer):
# self.outputs[i] = input
#else:
l.fprop(input, self.outputs[i], train)
input = self.outputs[i]
# probs.shape = self.outputs[-1].shape
gpu_copy_to(self.outputs[-1], probs)
def bprop(self, grad, input, output, outGrad):
self.weight.grad.fill(0)
self.bias.grad.fill(0)
# bprop to next layer
cudaconv2.convImgActs(grad, self.weight.wt, outGrad, self.img_size, self.img_size,
self.outputSize, -self.padding, self.stride, self.numColor, 1, 0.0, 1.0)
# bprop weight
cudaconv2.convWeightActs(input, grad, self.weight.grad, self.img_size, self.outputSize,
self.outputSize, self.filterSize, -self.padding, self.stride, self.numColor, 1, 0, 0, 1)
# bprop bias
gpu_copy_to(grad, self.tmp)
add_row_sum_to_vec(self.bias.grad, self.tmp)
def fprop(self, input, output, train=TRAIN):
#np.save('input.arr', input.get())
#np.save('weight.arr', self.weight.wt.get())
cudaconv2.convFilterActs(input, self.weight.wt, output, self.img_size, self.outputSize,
self.outputSize, -self.padding, self.stride, self.numColor, 1)
#util.log_info('%s', output.get().mean())
self.tmp = gpuarray.empty((self.numFilter,
self.get_single_img_size() * self.batch_size / self.numFilter),
dtype=np.float32)
gpu_copy_to(output, self.tmp)
add_vec_to_rows(self.tmp, self.bias.wt)
gpu_copy_to(self.tmp, output)
if PFout:
print_matrix(output, self.name)
def fprop(self, input, output, train=TRAIN):
#np.save('input.arr', input.get())
#np.save('weight.arr', self.weight.wt.get())
cudaconv2.convFilterActs(input, self.weight.wt, output, self.img_size, self.outputSize,
self.outputSize, -self.padding, self.stride, self.numColor, 1)
#util.log_info('%s', output.get().mean())
self.tmp = gpuarray.empty((self.numFilter,
self.get_single_img_size() * self.batch_size / self.numFilter),
dtype=np.float32)
gpu_copy_to(output, self.tmp)
add_vec_to_rows(self.tmp, self.bias.wt)
gpu_copy_to(self.tmp, output)
if PFout:
print_matrix(output, self.name)
def adjust_learning_rate(self, factor):
factors = factor
train_data = self.train_data
test_data = self.test_data
train_label = self.train_label
test_label = self.test_label
weights = []
biases = []
epsW = []
epsB = []
print 'store the weight, bias and learning rate'
for layer in self.layers:
if isinstance(layer, WeightedLayer):
weight = gpuarray.empty_like(layer.weight)
gpu_copy_to(layer.weight, weight)
weights.append(weight)
epsW.append(layer.epsW)
bias = gpuarray.empty_like(layer.bias)
gpu_copy_to(layer.bias, bias)
biases.append(bias)
epsB.append(layer.epsB)
print 'find the best learning rate'
print 'the factor list is ', factors
self.prepare_for_train(train_data, train_label)
self.fprop(self.data, self.output)
self.bprop(self.data, self.label, self.output)
self.get_batch_information()
self.update()
self.train_batch(test_data, test_label, TEST)
cost, correct, numCase = self.get_batch_information()
best = (correct , 1.0)
origin = (correct, 1.0)
print 'The normal update produce the correct', correct, 'number of case is', numCase
for factor in factors:
print 'Try the factor', factor
i = 0
for layer in self.layers:
if isinstance(layer, WeightedLayer):
gpu_copy_to(weights[i], layer.weight)
gpu_copy_to(biases[i], layer.bias)
layer.epsW = epsW[i] * factor
layer.epsB = epsB[i] * factor
i += 1
self.update()
'''
for layer in self.layers:
if isinstance(layer, WeightedLayer):
print 'epsW', layer.epsW, 'epsB', layer.epsB
printMatrix(layer.weight, layer.name + 'weight')
printMatrix(layer.bias, layer.name + 'bias')
'''
self.train_batch(test_data, test_label, TEST)
cost, correct, numCase = self.get_batch_information()
print 'Applying factor', factor, ', The correct is', correct, 'number of case is', numCase
if correct > best[0]:
best = (correct, factor)
self.adjust_info.append((best[1], best[0], origin[0]))
if best[0] / origin[0] < 1.025:
best = origin
factor = best[1]
i = 0
for layer in self.layers:
if isinstance(layer, WeightedLayer):
gpu_copy_to(weights[i], layer.weight)
gpu_copy_to(biases[i], layer.bias)
layer.epsW = epsW[i] * factor
layer.epsB = epsB[i] * factor
print 'Layer', layer.name
print 'epsW is', layer.epsW, 'epsB is', layer.epsB
i += 1
def adjust_learning_rate(self, factor):
factors = factor
train_data = self.train_data
test_data = self.test_data
train_label = self.train_label
test_label = self.test_label
weights = []
biases = []
epsW = []
epsB = []
print 'store the weight, bias and learning rate'
for layer in self.layers:
if isinstance(layer, WeightedLayer):
weight = gpuarray.empty_like(layer.weight)
gpu_copy_to(layer.weight, weight)
weights.append(weight)
epsW.append(layer.epsW)
bias = gpuarray.empty_like(layer.bias)
gpu_copy_to(layer.bias, bias)
biases.append(bias)
epsB.append(layer.epsB)
print 'find the best learning rate'
print 'the factor list is ', factors
self.prepare_for_train(train_data, train_label)
self.fprop(self.data, self.output)
self.bprop(self.data, self.label, self.output)
self.get_batch_information()
self.update()
self.train_batch(test_data, test_label, TEST)
cost, correct, numCase = self.get_batch_information()
best = (correct, 1.0)
origin = (correct, 1.0)
print 'The normal update produce the correct', correct, 'number of case is', numCase
for factor in factors:
print 'Try the factor', factor
i = 0
for layer in self.layers:
if isinstance(layer, WeightedLayer):
gpu_copy_to(weights[i], layer.weight)
gpu_copy_to(biases[i], layer.bias)
layer.epsW = epsW[i] * factor
layer.epsB = epsB[i] * factor
i += 1
self.update()
'''
for layer in self.layers:
if isinstance(layer, WeightedLayer):
print 'epsW', layer.epsW, 'epsB', layer.epsB
printMatrix(layer.weight, layer.name + 'weight')
printMatrix(layer.bias, layer.name + 'bias')
'''
self.train_batch(test_data, test_label, TEST)
cost, correct, numCase = self.get_batch_information()
print 'Applying factor', factor, ', The correct is', correct, 'number of case is', numCase
if correct > best[0]:
best = (correct, factor)
self.adjust_info.append((best[1], best[0], origin[0]))
if best[0] / origin[0] < 1.025:
best = origin
factor = best[1]
i = 0
for layer in self.layers:
if isinstance(layer, WeightedLayer):
gpu_copy_to(weights[i], layer.weight)
gpu_copy_to(biases[i], layer.bias)
layer.epsW = epsW[i] * factor
layer.epsB = epsB[i] * factor
print 'Layer', layer.name
print 'epsW is', layer.epsW, 'epsB is', layer.epsB
i += 1
