def affine_backward_variables(x_shape, w_shape): """backward에서 업데이트를 위한 변수들의 미분값 테스트입니다.""" x = tensor.create_gauss(x_shape) w = tensor.create_gauss(w_shape) b = tensor.create_gauss([w_shape[-1]]) x_t = tensor.create_transpose(x) forward_out = tensor.create_matrix_product(x, w) forward_out = tensor.create_gauss(forward_out.shape) standard_dw = tensor.create_zeros(w.shape, float) new_dw = tensor.create_zeros(w.shape, float) compare_dw = tensor.create_zeros(w.shape, bool) standard_db = tensor.create_zeros(b.shape, float) new_db = tensor.create_zeros(b.shape, float) compare_db = tensor.create_zeros(b.shape, float) #잘 알려진 방법 tensor.transpose(x, x_t) tensor.matmul(x_t, forward_out, standard_dw) tensor.sum_axis(forward_out, 0, standard_db) #새로운 방법 affine.backward_variables(x.array, forward_out.array, new_dw.array, new_db.array) tensor.function_element_wise(standard_dw, new_dw, isSame, compare_dw) tensor.function_element_wise(standard_db, new_db, isSame, compare_db) print(compare_dw) print(compare_db)
def test_conv3d_forward(data_shape, filter_shape, stride, pad, padding): x = tensor.create_gauss(data_shape) b = tensor.create_ones([filter_shape[0]]) filter = tensor.create_gauss(filter_shape) out = tensor.create_zeros( conv3d_module.create_shape(x.shape, filter.shape, stride, pad)) conv3d_module.forward(x.array, x.shape, filter.array, filter_shape, b.array, stride, pad, padding, out.array, out.shape) print(x) print(filter) print(b) print(out)
def append_convolution(self, stride, pad, padding, *output): """stride(filter 이동 간격 크기), pad(빈 공간 크기), padding(빈 공간 값)과 출력 뉴런 구조를 정의하여 conv3d 레이어를 뒤에 추가합니다.""" input_shape = [1,1,1] for c in range(len(self.input)): input_shape[-1 -c] = self.input[-1 -c] output_shape = [1,1,1] for c in range(len(output)): output_shape[-1 -c] = output[-1 -c] filter = tensor.create_gauss([output_shape[0], input_shape[0], input_shape[1] + 2 * pad + stride - stride * output_shape[1], input_shape[2] + 2 * pad + stride - stride * output_shape[2]]) bias = tensor.create_gauss([output_shape[0],1,1]) self.input = list(output) return self.append(nn.layer.Conv3d(filter, bias, stride, pad, padding))
def affine_forward(x_shape, w_shape): x = tensor.create_gauss(x_shape) w = tensor.create_gauss(w_shape) b = tensor.create_gauss([w_shape[-1]]) standard_out = tensor.create_matrix_product(x, w) new_out = tensor.create_zeros(standard_out.shape.copy(), float) compare_out = tensor.create_zeros(standard_out.shape.copy(), bool) tensor.matmul(x, w, standard_out) tensor.add(standard_out, b, standard_out) affine.forward(x.array, w.array, b.array, new_out.array) tensor.function_element_wise(standard_out, new_out, isSame, compare_out) print(compare_out)
def test_alg(data_shape, w_shape): x = tensor.create_gauss(data_shape) w = tensor.create_gauss(w_shape) b = tensor.create_gauss([w_shape[-1]]) out = tensor.create_matrix_product(x, w) out_test = out.copy() time1 = time.time_ns() NN.layer.computing.affine_module.forward(x.array, w.array, b.array, out.array) time2 = time.time_ns() NN.layer.computing.affine_module.forward_old1(x.array, w.array, b.array, out_test.array) time3 = time.time_ns() print('{0}, {1}'.format(time2 - time1, time3 - time2)) print('{0}'.format(tensor.isSame(out, out_test)))
def test_compare_alg1(data_shape, filter_shape, stride, pad, padding): x = tensor.create_gauss(data_shape) bias = tensor.create_ones([filter_shape[0]]) filter = tensor.create_gauss(filter_shape) out1 = tensor.create_zeros( conv3d_module.create_shape(x.shape, filter.shape, stride, pad)) out2 = tensor.create_zeros( conv3d_module.create_shape(x.shape, filter.shape, stride, pad)) time1 = time.time_ns() #conv3d_module.forward_test(x.array, x.shape, filter.array, filter.shape, bias.array,stride, pad, padding, out1.array, out1.shape) time2 = time.time_ns() conv3d_module.forward(x.array, x.shape, filter.array, filter.shape, bias.array, stride, pad, padding, out2.array, out2.shape) time3 = time.time_ns() print('{0}, {1}'.format(time2 - time1, time3 - time2)) print('{0}'.format(tensor.isSame(out1, out2)))
def affine_backward(x_shape, w_shape): """backward에서 뒷쪽 노드에 보내줄 dout을 테스트합니다.""" x = tensor.create_gauss(x_shape) w = tensor.create_gauss(w_shape) w_t = tensor.create_transpose(w) forward_out = tensor.create_matrix_product(x, w) forward_out = tensor.create_gauss(forward_out.shape) standard_out = tensor.create_matrix_product(forward_out, w_t) new_out = tensor.create_zeros(standard_out.shape.copy(), float) compare_out = tensor.create_zeros(standard_out.shape.copy(), bool) #잘 알려진 방법 tensor.transpose(w, w_t) tensor.matmul(forward_out, w_t, standard_out) #새로운 방법 affine.backward(forward_out.array, w.array, w.shape, new_out.array) tensor.function_element_wise(standard_out, new_out, isSame, compare_out) print(compare_out)
def append_affine(self, *output): """출력 뉴런 구조를 정의하여 affine레이어를 뒤에 추가합니다.""" tmp = self.input.pop() while(len(self.input) != 0): tmp *= self.input.pop() self.input.append(tmp) self.input.extend(output) w = tensor.create_gauss(self.input) self.input = list(output) b = tensor.create_zeros(self.input) self.input = self.input.copy() return self.append(nn.layer.Affine(w,b))
def test_conv3d_backward(data_shape, filter_shape, stride, pad, padding): x = tensor.create_gauss(data_shape) dx = tensor.create_gauss(data_shape) dx_diff = x.copy() b = tensor.create_ones([filter_shape[0]]) db = b.copy() db_diff = b.copy() filter = tensor.create_gauss(filter_shape) dfilter = tensor.create_gauss(filter_shape) dfilter_diff = dfilter.copy() out = tensor.create_gauss( conv3d_module.create_shape(data_shape, filter_shape, stride, pad)) dout = tensor.create_ones(out.shape) conv3d_module.forward(x.array, x.shape, filter.array, filter.shape, b.array, stride, pad, padding, out.array, out.shape) #conv3d_module.backward(x.array, dout.array, dout.shape, filter.array, filter.shape, stride, pad, padding, dfilter.array, db.array, dx.array, dx.shape) conv3d_module.backward(dout.array, dout.shape, filter.array, filter.shape, stride, pad, dx.array, dx.shape) conv3d_module.backward_filter(dout.array, dout.shape, x.array, x.shape, stride, pad, padding, dfilter.array, dfilter.shape) conv3d_module.backward_bias(dout.array, dout.shape, db.array) for i in range(len(x.array)): origen = x.array[i] x.array[i] = origen + 0.0001 conv3d_module.forward(x.array, x.shape, filter.array, filter.shape, b.array, stride, pad, padding, out.array, out.shape) out1 = 0 for j in range(len(out.array)): out1 += out.array[j] x.array[i] = origen - 0.0001 conv3d_module.forward(x.array, x.shape, filter.array, filter.shape, b.array, stride, pad, padding, out.array, out.shape) out2 = 0 for j in range(len(out.array)): out2 += out.array[j] dx_diff.array[i] = (out1 - out2) / (2 * 0.0001) x.array[i] = origen for i in range(len(filter.array)): origen = filter.array[i] filter.array[i] = origen + 0.0001 conv3d_module.forward(x.array, x.shape, filter.array, filter.shape, b.array, stride, pad, padding, out.array, out.shape) out1 = 0 for j in range(len(out.array)): out1 += out.array[j] filter.array[i] = origen - 0.0001 conv3d_module.forward(x.array, x.shape, filter.array, filter.shape, b.array, stride, pad, padding, out.array, out.shape) out2 = 0 for j in range(len(out.array)): out2 += out.array[j] dfilter_diff.array[i] = (out1 - out2) / (2 * 0.0001) filter.array[i] = origen for i in range(len(b.array)): origen = b.array[i] b.array[i] = origen + 0.0001 conv3d_module.forward(x.array, x.shape, filter.array, filter.shape, b.array, stride, pad, padding, out.array, out.shape) out1 = 0 for j in range(len(out.array)): out1 += out.array[j] b.array[i] = origen - 0.0001 conv3d_module.forward(x.array, x.shape, filter.array, filter.shape, b.array, stride, pad, padding, out.array, out.shape) out2 = 0 for j in range(len(out.array)): out2 += out.array[j] db_diff.array[i] = (out1 - out2) / (2 * 0.0001) b.array[i] = origen #print(dx) #print(dx_diff) #print(dfilter) #print(dfilter_diff) print(tensor.isSame(dx, dx_diff)) print(tensor.isSame(dfilter, dfilter_diff)) print(tensor.isSame(db, db_diff))
def test(data_shape, w_shape): x = tensor.create_gauss(data_shape) w = tensor.create_gauss(w_shape) b = tensor.create_gauss([w_shape[-1]]) out_affine = tensor.create_matrix_product(x, w) t = tensor.create_zeros(out_affine.shape.copy(), int) for i in range(t.shape[0]): t.array[i * t.shape[1] + random.randint(0, t.shape[1] - 1)] = 1 out_sigmoid = out_affine.copy() dw1 = w.copy() dw2 = w.copy() db1 = b.copy() dout_sigmoid = out_affine.copy() dout1 = x.copy() dout2 = x.copy() NN.layer.computing.affine_module.forward(x.array, w.array, b.array, out_affine.array) NN.layer.computing.sigmoid_module.forward(out_affine.array, out_sigmoid.array) NN.layer.computing.cross_entropy_module.backward(out_sigmoid.array, t.array, dout_sigmoid.array) NN.layer.computing.sigmoid_module.backward(dout_sigmoid.array, out_sigmoid.array) NN.layer.computing.affine_module.backward(out_sigmoid.array, w.array, w.shape, dout1.array) #NN.layer.computing.affine_module.backward_variables(x.array, out_sigmoid.array, dw1.array, db1.array) NN.layer.computing.affine_module.backward_dw(x.array, x.shape, out_sigmoid.array, dw1.array, dw1.shape) for i in range(len(x.array)): origen = x.array[i] x.array[i] = origen + 0.0001 NN.layer.computing.affine_module.forward(x.array, w.array, b.array, out_affine.array) NN.layer.computing.sigmoid_module.forward(out_affine.array, out_sigmoid.array) out1 = NN.layer.computing.cross_entropy_module.forward( out_sigmoid.array, t.array) x.array[i] = origen - 0.0001 NN.layer.computing.affine_module.forward(x.array, w.array, b.array, out_affine.array) NN.layer.computing.sigmoid_module.forward(out_affine.array, out_sigmoid.array) out2 = NN.layer.computing.cross_entropy_module.forward( out_sigmoid.array, t.array) dout2.array[i] = (out1 - out2) / (2 * 0.0001) x.array[i] = origen for i in range(len(dw2.array)): origen = w.array[i] w.array[i] = origen + 0.0001 NN.layer.computing.affine_module.forward(x.array, w.array, b.array, out_affine.array) NN.layer.computing.sigmoid_module.forward(out_affine.array, out_sigmoid.array) out1 = NN.layer.computing.cross_entropy_module.forward( out_sigmoid.array, t.array) w.array[i] = origen - 0.0001 NN.layer.computing.affine_module.forward(x.array, w.array, b.array, out_affine.array) NN.layer.computing.sigmoid_module.forward(out_affine.array, out_sigmoid.array) out2 = NN.layer.computing.cross_entropy_module.forward( out_sigmoid.array, t.array) dw2.array[i] = (out1 - out2) / (2 * 0.0001) w.array[i] = origen """ print("최종 역전파 결과") print(dout1) print("편미분 최정 결과") print(dout2) print('최종 w 역전파 결과') print(dw1) print('최종 w 편미분 결과') print(dw2) """ print(tensor.isSame(dout1, dout2)) print(tensor.isSame(dw1, dw2))
def test2(data_shape, w_shape): x = tensor.create_gauss(data_shape) w = tensor.create_gauss(w_shape) b = tensor.create_gauss([w_shape[-1]]) out_affine = tensor.create_matrix_product(x, w) dout_affine = tensor.create_ones(out_affine.shape) dw1 = w.copy() dw2 = w.copy() db1 = b.copy() dx1 = x.copy() dx2 = x.copy() #NN.layer.computing.affine_module.forward(x.array, w.array, b.array, out_affine.array) NN.layer.computing.affine_module.backward(dout_affine.array, w.array, w.shape, dx1.array) NN.layer.computing.affine_module.backward_variables( x.array, dout_affine.array, dw1.array, db1.array) for i in range(len(x.array)): origen = x.array[i] x.array[i] = origen + 0.0001 NN.layer.computing.affine_module.forward(x.array, w.array, b.array, out_affine.array) out1 = 0 for j in range(len(out_affine.array)): out1 += out_affine.array[j] x.array[i] = origen - 0.0001 NN.layer.computing.affine_module.forward(x.array, w.array, b.array, out_affine.array) out2 = 0 for j in range(len(out_affine.array)): out2 += out_affine.array[j] dx2.array[i] = (out1 - out2) / (2 * 0.0001) x.array[i] = origen for i in range(len(dw2.array)): origen = w.array[i] w.array[i] = origen + 0.0001 NN.layer.computing.affine_module.forward(x.array, w.array, b.array, out_affine.array) out1 = 0 for j in range(len(out_affine.array)): out1 += out_affine.array[j] w.array[i] = origen - 0.0001 NN.layer.computing.affine_module.forward(x.array, w.array, b.array, out_affine.array) out2 = 0 for j in range(len(out_affine.array)): out2 += out_affine.array[j] dw2.array[i] = (out1 - out2) / (2 * 0.0001) w.array[i] = origen print("최종 역전파 결과") print(dx1) print("편미분 최정 결과") print(dx2) print('최종 w 역전파 결과') print(dw1) print('최종 w 편미분 결과') print(dw2)
import sys sys.path.append(__file__.replace('\\NN\\example\\ex1_xor1.py', '')) import NN as nn import tensor # xor 데이터 x = tensor.Tensor([0., 0., 0., 1., 1., 0., 1., 1.], [4, 2]) y = tensor.Tensor([0., 1., 1., 0., 1., 0., 0., 1.], [4, 2]) # 변수 초기화 w1 = tensor.create_gauss([2, 4]) b1 = tensor.create_zeros([4]) w2 = tensor.create_gauss([4, 2]) b2 = tensor.create_zeros([2]) # 네트워크 생성 network = nn.Network() network.append(nn.layer.Affine(w1, b1)) network.append(nn.layer.Sigmoid()) network.append(nn.layer.Affine(w2, b2)) network.append(nn.layer.Softmax()) optimizer = nn.optimizer.SGD(0.05) # 초기화 network.initForward(x) network.initBackward(1, y)