def affine_backward_variables(x_shape, w_shape):
"""backward에서 업데이트를 위한 변수들의 미분값 테스트입니다."""
x = tensor.create_gauss(x_shape)
w = tensor.create_gauss(w_shape)
b = tensor.create_gauss([w_shape[-1]])
x_t = tensor.create_transpose(x)
forward_out = tensor.create_matrix_product(x, w)
forward_out = tensor.create_gauss(forward_out.shape)
standard_dw = tensor.create_zeros(w.shape, float)
new_dw = tensor.create_zeros(w.shape, float)
compare_dw = tensor.create_zeros(w.shape, bool)
standard_db = tensor.create_zeros(b.shape, float)
new_db = tensor.create_zeros(b.shape, float)
compare_db = tensor.create_zeros(b.shape, float)
#잘 알려진 방법
tensor.transpose(x, x_t)
tensor.matmul(x_t, forward_out, standard_dw)
tensor.sum_axis(forward_out, 0, standard_db)
#새로운 방법
affine.backward_variables(x.array, forward_out.array, new_dw.array,
new_db.array)
tensor.function_element_wise(standard_dw, new_dw, isSame, compare_dw)
tensor.function_element_wise(standard_db, new_db, isSame, compare_db)
print(compare_dw)
print(compare_db)
def test_conv3d_forward(data_shape, filter_shape, stride, pad, padding):
x = tensor.create_gauss(data_shape)
b = tensor.create_ones([filter_shape[0]])
filter = tensor.create_gauss(filter_shape)
out = tensor.create_zeros(
conv3d_module.create_shape(x.shape, filter.shape, stride, pad))
conv3d_module.forward(x.array, x.shape, filter.array, filter_shape,
b.array, stride, pad, padding, out.array, out.shape)
print(x)
print(filter)
print(b)
print(out)
def append_convolution(self, stride, pad, padding, *output):
"""stride(filter 이동 간격 크기), pad(빈 공간 크기), padding(빈 공간 값)과 출력 뉴런 구조를 정의하여 conv3d 레이어를 뒤에 추가합니다."""
input_shape = [1,1,1]
for c in range(len(self.input)):
input_shape[-1 -c] = self.input[-1 -c]
output_shape = [1,1,1]
for c in range(len(output)):
output_shape[-1 -c] = output[-1 -c]
filter = tensor.create_gauss([output_shape[0], input_shape[0], input_shape[1] + 2 * pad + stride - stride * output_shape[1], input_shape[2] + 2 * pad + stride - stride * output_shape[2]])
bias = tensor.create_gauss([output_shape[0],1,1])
self.input = list(output)
return self.append(nn.layer.Conv3d(filter, bias, stride, pad, padding))
def affine_forward(x_shape, w_shape):
x = tensor.create_gauss(x_shape)
w = tensor.create_gauss(w_shape)
b = tensor.create_gauss([w_shape[-1]])
standard_out = tensor.create_matrix_product(x, w)
new_out = tensor.create_zeros(standard_out.shape.copy(), float)
compare_out = tensor.create_zeros(standard_out.shape.copy(), bool)
tensor.matmul(x, w, standard_out)
tensor.add(standard_out, b, standard_out)
affine.forward(x.array, w.array, b.array, new_out.array)
tensor.function_element_wise(standard_out, new_out, isSame, compare_out)
print(compare_out)
def test_alg(data_shape, w_shape):
x = tensor.create_gauss(data_shape)
w = tensor.create_gauss(w_shape)
b = tensor.create_gauss([w_shape[-1]])
out = tensor.create_matrix_product(x, w)
out_test = out.copy()
time1 = time.time_ns()
NN.layer.computing.affine_module.forward(x.array, w.array, b.array,
out.array)
time2 = time.time_ns()
NN.layer.computing.affine_module.forward_old1(x.array, w.array, b.array,
out_test.array)
time3 = time.time_ns()
print('{0}, {1}'.format(time2 - time1, time3 - time2))
print('{0}'.format(tensor.isSame(out, out_test)))
def test_compare_alg1(data_shape, filter_shape, stride, pad, padding):
x = tensor.create_gauss(data_shape)
bias = tensor.create_ones([filter_shape[0]])
filter = tensor.create_gauss(filter_shape)
out1 = tensor.create_zeros(
conv3d_module.create_shape(x.shape, filter.shape, stride, pad))
out2 = tensor.create_zeros(
conv3d_module.create_shape(x.shape, filter.shape, stride, pad))
time1 = time.time_ns()
#conv3d_module.forward_test(x.array, x.shape, filter.array, filter.shape, bias.array,stride, pad, padding, out1.array, out1.shape)
time2 = time.time_ns()
conv3d_module.forward(x.array, x.shape, filter.array, filter.shape,
bias.array, stride, pad, padding, out2.array,
out2.shape)
time3 = time.time_ns()
print('{0}, {1}'.format(time2 - time1, time3 - time2))
print('{0}'.format(tensor.isSame(out1, out2)))
def affine_backward(x_shape, w_shape):
"""backward에서 뒷쪽 노드에 보내줄 dout을 테스트합니다."""
x = tensor.create_gauss(x_shape)
w = tensor.create_gauss(w_shape)
w_t = tensor.create_transpose(w)
forward_out = tensor.create_matrix_product(x, w)
forward_out = tensor.create_gauss(forward_out.shape)
standard_out = tensor.create_matrix_product(forward_out, w_t)
new_out = tensor.create_zeros(standard_out.shape.copy(), float)
compare_out = tensor.create_zeros(standard_out.shape.copy(), bool)
#잘 알려진 방법
tensor.transpose(w, w_t)
tensor.matmul(forward_out, w_t, standard_out)
#새로운 방법
affine.backward(forward_out.array, w.array, w.shape, new_out.array)
tensor.function_element_wise(standard_out, new_out, isSame, compare_out)
print(compare_out)
def append_affine(self, *output):
"""출력 뉴런 구조를 정의하여 affine레이어를 뒤에 추가합니다."""
tmp = self.input.pop()
while(len(self.input) != 0):
tmp *= self.input.pop()
self.input.append(tmp)
self.input.extend(output)
w = tensor.create_gauss(self.input)
self.input = list(output)
b = tensor.create_zeros(self.input)
self.input = self.input.copy()
return self.append(nn.layer.Affine(w,b))
def test_conv3d_backward(data_shape, filter_shape, stride, pad, padding):
x = tensor.create_gauss(data_shape)
dx = tensor.create_gauss(data_shape)
dx_diff = x.copy()
b = tensor.create_ones([filter_shape[0]])
db = b.copy()
db_diff = b.copy()
filter = tensor.create_gauss(filter_shape)
dfilter = tensor.create_gauss(filter_shape)
dfilter_diff = dfilter.copy()
out = tensor.create_gauss(
conv3d_module.create_shape(data_shape, filter_shape, stride, pad))
dout = tensor.create_ones(out.shape)
conv3d_module.forward(x.array, x.shape, filter.array, filter.shape,
b.array, stride, pad, padding, out.array, out.shape)
#conv3d_module.backward(x.array, dout.array, dout.shape, filter.array, filter.shape, stride, pad, padding, dfilter.array, db.array, dx.array, dx.shape)
conv3d_module.backward(dout.array, dout.shape, filter.array, filter.shape,
stride, pad, dx.array, dx.shape)
conv3d_module.backward_filter(dout.array, dout.shape, x.array, x.shape,
stride, pad, padding, dfilter.array,
dfilter.shape)
conv3d_module.backward_bias(dout.array, dout.shape, db.array)
for i in range(len(x.array)):
origen = x.array[i]
x.array[i] = origen + 0.0001
conv3d_module.forward(x.array, x.shape, filter.array, filter.shape,
b.array, stride, pad, padding, out.array,
out.shape)
out1 = 0
for j in range(len(out.array)):
out1 += out.array[j]
x.array[i] = origen - 0.0001
conv3d_module.forward(x.array, x.shape, filter.array, filter.shape,
b.array, stride, pad, padding, out.array,
out.shape)
out2 = 0
for j in range(len(out.array)):
out2 += out.array[j]
dx_diff.array[i] = (out1 - out2) / (2 * 0.0001)
x.array[i] = origen
for i in range(len(filter.array)):
origen = filter.array[i]
filter.array[i] = origen + 0.0001
conv3d_module.forward(x.array, x.shape, filter.array, filter.shape,
b.array, stride, pad, padding, out.array,
out.shape)
out1 = 0
for j in range(len(out.array)):
out1 += out.array[j]
filter.array[i] = origen - 0.0001
conv3d_module.forward(x.array, x.shape, filter.array, filter.shape,
b.array, stride, pad, padding, out.array,
out.shape)
out2 = 0
for j in range(len(out.array)):
out2 += out.array[j]
dfilter_diff.array[i] = (out1 - out2) / (2 * 0.0001)
filter.array[i] = origen
for i in range(len(b.array)):
origen = b.array[i]
b.array[i] = origen + 0.0001
conv3d_module.forward(x.array, x.shape, filter.array, filter.shape,
b.array, stride, pad, padding, out.array,
out.shape)
out1 = 0
for j in range(len(out.array)):
out1 += out.array[j]
b.array[i] = origen - 0.0001
conv3d_module.forward(x.array, x.shape, filter.array, filter.shape,
b.array, stride, pad, padding, out.array,
out.shape)
out2 = 0
for j in range(len(out.array)):
out2 += out.array[j]
db_diff.array[i] = (out1 - out2) / (2 * 0.0001)
b.array[i] = origen
#print(dx)
#print(dx_diff)
#print(dfilter)
#print(dfilter_diff)
print(tensor.isSame(dx, dx_diff))
print(tensor.isSame(dfilter, dfilter_diff))
print(tensor.isSame(db, db_diff))
def test(data_shape, w_shape):
x = tensor.create_gauss(data_shape)
w = tensor.create_gauss(w_shape)
b = tensor.create_gauss([w_shape[-1]])
out_affine = tensor.create_matrix_product(x, w)
t = tensor.create_zeros(out_affine.shape.copy(), int)
for i in range(t.shape[0]):
t.array[i * t.shape[1] + random.randint(0, t.shape[1] - 1)] = 1
out_sigmoid = out_affine.copy()
dw1 = w.copy()
dw2 = w.copy()
db1 = b.copy()
dout_sigmoid = out_affine.copy()
dout1 = x.copy()
dout2 = x.copy()
NN.layer.computing.affine_module.forward(x.array, w.array, b.array,
out_affine.array)
NN.layer.computing.sigmoid_module.forward(out_affine.array,
out_sigmoid.array)
NN.layer.computing.cross_entropy_module.backward(out_sigmoid.array,
t.array,
dout_sigmoid.array)
NN.layer.computing.sigmoid_module.backward(dout_sigmoid.array,
out_sigmoid.array)
NN.layer.computing.affine_module.backward(out_sigmoid.array, w.array,
w.shape, dout1.array)
#NN.layer.computing.affine_module.backward_variables(x.array, out_sigmoid.array, dw1.array, db1.array)
NN.layer.computing.affine_module.backward_dw(x.array, x.shape,
out_sigmoid.array, dw1.array,
dw1.shape)
for i in range(len(x.array)):
origen = x.array[i]
x.array[i] = origen + 0.0001
NN.layer.computing.affine_module.forward(x.array, w.array, b.array,
out_affine.array)
NN.layer.computing.sigmoid_module.forward(out_affine.array,
out_sigmoid.array)
out1 = NN.layer.computing.cross_entropy_module.forward(
out_sigmoid.array, t.array)
x.array[i] = origen - 0.0001
NN.layer.computing.affine_module.forward(x.array, w.array, b.array,
out_affine.array)
NN.layer.computing.sigmoid_module.forward(out_affine.array,
out_sigmoid.array)
out2 = NN.layer.computing.cross_entropy_module.forward(
out_sigmoid.array, t.array)
dout2.array[i] = (out1 - out2) / (2 * 0.0001)
x.array[i] = origen
for i in range(len(dw2.array)):
origen = w.array[i]
w.array[i] = origen + 0.0001
NN.layer.computing.affine_module.forward(x.array, w.array, b.array,
out_affine.array)
NN.layer.computing.sigmoid_module.forward(out_affine.array,
out_sigmoid.array)
out1 = NN.layer.computing.cross_entropy_module.forward(
out_sigmoid.array, t.array)
w.array[i] = origen - 0.0001
NN.layer.computing.affine_module.forward(x.array, w.array, b.array,
out_affine.array)
NN.layer.computing.sigmoid_module.forward(out_affine.array,
out_sigmoid.array)
out2 = NN.layer.computing.cross_entropy_module.forward(
out_sigmoid.array, t.array)
dw2.array[i] = (out1 - out2) / (2 * 0.0001)
w.array[i] = origen
"""
print("최종 역전파 결과")
print(dout1)
print("편미분 최정 결과")
print(dout2)
print('최종 w 역전파 결과')
print(dw1)
print('최종 w 편미분 결과')
print(dw2)
"""
print(tensor.isSame(dout1, dout2))
print(tensor.isSame(dw1, dw2))
def test2(data_shape, w_shape):
x = tensor.create_gauss(data_shape)
w = tensor.create_gauss(w_shape)
b = tensor.create_gauss([w_shape[-1]])
out_affine = tensor.create_matrix_product(x, w)
dout_affine = tensor.create_ones(out_affine.shape)
dw1 = w.copy()
dw2 = w.copy()
db1 = b.copy()
dx1 = x.copy()
dx2 = x.copy()
#NN.layer.computing.affine_module.forward(x.array, w.array, b.array, out_affine.array)
NN.layer.computing.affine_module.backward(dout_affine.array, w.array,
w.shape, dx1.array)
NN.layer.computing.affine_module.backward_variables(
x.array, dout_affine.array, dw1.array, db1.array)
for i in range(len(x.array)):
origen = x.array[i]
x.array[i] = origen + 0.0001
NN.layer.computing.affine_module.forward(x.array, w.array, b.array,
out_affine.array)
out1 = 0
for j in range(len(out_affine.array)):
out1 += out_affine.array[j]
x.array[i] = origen - 0.0001
NN.layer.computing.affine_module.forward(x.array, w.array, b.array,
out_affine.array)
out2 = 0
for j in range(len(out_affine.array)):
out2 += out_affine.array[j]
dx2.array[i] = (out1 - out2) / (2 * 0.0001)
x.array[i] = origen
for i in range(len(dw2.array)):
origen = w.array[i]
w.array[i] = origen + 0.0001
NN.layer.computing.affine_module.forward(x.array, w.array, b.array,
out_affine.array)
out1 = 0
for j in range(len(out_affine.array)):
out1 += out_affine.array[j]
w.array[i] = origen - 0.0001
NN.layer.computing.affine_module.forward(x.array, w.array, b.array,
out_affine.array)
out2 = 0
for j in range(len(out_affine.array)):
out2 += out_affine.array[j]
dw2.array[i] = (out1 - out2) / (2 * 0.0001)
w.array[i] = origen
print("최종 역전파 결과")
print(dx1)
print("편미분 최정 결과")
print(dx2)
print('최종 w 역전파 결과')
print(dw1)
print('최종 w 편미분 결과')
print(dw2)
import sys
sys.path.append(__file__.replace('\\NN\\example\\ex1_xor1.py', ''))
import NN as nn
import tensor
# xor 데이터
x = tensor.Tensor([0., 0., 0., 1., 1., 0., 1., 1.], [4, 2])
y = tensor.Tensor([0., 1., 1., 0., 1., 0., 0., 1.], [4, 2])
# 변수 초기화
w1 = tensor.create_gauss([2, 4])
b1 = tensor.create_zeros([4])
w2 = tensor.create_gauss([4, 2])
b2 = tensor.create_zeros([2])
# 네트워크 생성
network = nn.Network()
network.append(nn.layer.Affine(w1, b1))
network.append(nn.layer.Sigmoid())
network.append(nn.layer.Affine(w2, b2))
network.append(nn.layer.Softmax())
optimizer = nn.optimizer.SGD(0.05)
# 초기화
network.initForward(x)
network.initBackward(1, y)