def test_grad_of_grad(): x2 = ad.Variable(name="x2") x3 = ad.Variable(name="x3") y = x2 * x2 + x2 * x3 grad_x2, grad_x3 = ad.gradients(y, [x2, x3]) grad_x2_x2, grad_x2_x3 = ad.gradients(grad_x2, [x2, x3]) executor = ad.Executor([y, grad_x2, grad_x3, grad_x2_x2, grad_x2_x3]) x2_val = 2 * np.ones(3) x3_val = 3 * np.ones(3) y_val, grad_x2_val, grad_x3_val, grad_x2_x2_val, grad_x2_x3_val = executor.run( feed_dict={ x2: x2_val, x3: x3_val }) expected_yval = x2_val * x2_val + x2_val * x3_val expected_grad_x2_val = 2 * x2_val + x3_val expected_grad_x3_val = x2_val expected_grad_x2_x2_val = 2 * np.ones_like(x2_val) expected_grad_x2_x3_val = 1 * np.ones_like(x2_val) assert isinstance(y, ad.Node) assert np.array_equal(y_val, expected_yval) assert np.array_equal(grad_x2_val, expected_grad_x2_val) assert np.array_equal(grad_x3_val, expected_grad_x3_val) assert np.array_equal(grad_x2_x2_val, expected_grad_x2_x2_val) assert np.array_equal(grad_x2_x3_val, expected_grad_x2_x3_val)
def test_full_forward_op(): inputs = ad.Variable("inputs") filters = ad.Variable("filters") y_ = ad.Variable(name="y_") #ini ctx = ndarray.gpu(0) x_val = np.linspace(0, 100, 100).reshape((5, 1, 20)) filters_val = np.ones((1, 1, 20)) * 0.001 y_val = np.zeros((5, 1)) x_val = ndarray.array(x_val, ctx) filters_val = ndarray.array(filters_val, ctx) y_val = ndarray.array(y_val, ctx) outputs = ad.convolution_1d_forward_op(inputs, filters, "NCHW", "VALID", 1) outputs_pool = ad.pooling_1d_forward_op(outputs, "NCHW", "max", 0, 1, 1) outputs_relu = ad.activation_forward_op(outputs_pool, "NCHW", "relu") outputs_f = ad.flatten_op(outputs_relu) output = ad.fullyactivation_forward_op(outputs_f, "NCHW", "relu") loss = ad.matmul_op(output, output, trans_A=True) * (1 / 5) grad_f = ad.gradients(loss, [filters]) #gra返回一个list executor = ad.Executor([grad_f[0]], ctx=ctx) g_val = executor.run(feed_dict={ inputs: x_val, filters: filters_val }) #返回一个list print("g_val:", g_val[0].asnumpy())
def test_matmul_two_vars(): x2 = ad.Variable(name="x2") x3 = ad.Variable(name="x3") y = ad.matmul_op(x2, x3) grad_x2, grad_x3 = ad.gradients(y, [x2, x3]) executor = ad.Executor([y, grad_x2, grad_x3]) x2_val = np.array([[1, 2], [3, 4], [5, 6]]) # 3x2 x3_val = np.array([[7, 8, 9], [10, 11, 12]]) # 2x3 y_val, grad_x2_val, grad_x3_val = executor.run(feed_dict={ x2: x2_val, x3: x3_val }) expected_yval = np.matmul(x2_val, x3_val) expected_grad_x2_val = np.matmul(np.ones_like(expected_yval), np.transpose(x3_val)) expected_grad_x3_val = np.matmul(np.transpose(x2_val), np.ones_like(expected_yval)) assert isinstance(y, ad.Node) assert np.array_equal(y_val, expected_yval) assert np.array_equal(grad_x2_val, expected_grad_x2_val) assert np.array_equal(grad_x3_val, expected_grad_x3_val)
def test_add_mul_mix_3(): x2 = ad.Variable(name="x2") x3 = ad.Variable(name="x3") z = x2 * x2 + x2 + x3 + 3 y = z * z + x3 grad_x2, grad_x3 = ad.gradients(y, [x2, x3]) executor = ad.Executor([y, grad_x2, grad_x3]) x2_val = 2 * np.ones(3) x3_val = 3 * np.ones(3) y_val, grad_x2_val, grad_x3_val = executor.run(feed_dict={ x2: x2_val, x3: x3_val }) z_val = x2_val * x2_val + x2_val + x3_val + 3 expected_yval = z_val * z_val + x3_val expected_grad_x2_val = 2 * (x2_val * x2_val + x2_val + x3_val + 3) * (2 * x2_val + 1) expected_grad_x3_val = 2 * (x2_val * x2_val + x2_val + x3_val + 3) + 1 assert isinstance(y, ad.Node) assert np.array_equal(y_val, expected_yval) assert np.array_equal(grad_x2_val, expected_grad_x2_val) assert np.array_equal(grad_x3_val, expected_grad_x3_val)
def test_add_mul_mix_2(): x1 = ad.Variable(name="x1") x2 = ad.Variable(name="x2") x3 = ad.Variable(name="x3") x4 = ad.Variable(name="x4") y = x1 + x2 * x3 * x4 grad_x1, grad_x2, grad_x3, grad_x4 = ad.gradients(y, [x1, x2, x3, x4]) executor = ad.Executor([y, grad_x1, grad_x2, grad_x3, grad_x4]) x1_val = 1 * np.ones(3) x2_val = 2 * np.ones(3) x3_val = 3 * np.ones(3) x4_val = 4 * np.ones(3) y_val, grad_x1_val, grad_x2_val, grad_x3_val, grad_x4_val = executor.run( feed_dict={ x1: x1_val, x2: x2_val, x3: x3_val, x4: x4_val }) assert isinstance(y, ad.Node) assert np.array_equal(y_val, x1_val + x2_val * x3_val * x4_val) assert np.array_equal(grad_x1_val, np.ones_like(x1_val)) assert np.array_equal(grad_x2_val, x3_val * x4_val) assert np.array_equal(grad_x3_val, x2_val * x4_val) assert np.array_equal(grad_x4_val, x2_val * x3_val)
def block_reduction_a(inputs, input_size, blockname=None): filter2_1_0 = ad.Variable(blockname + "filter2_1_0") filter2_1_1a = ad.Variable(blockname + "filter2_1_1a") filter2_1_1b = ad.Variable(blockname + "filter2_1_1b") filter2_1_1c = ad.Variable(blockname + "filter2_1_1c") rand = np.random.RandomState(seed=123) filter2_1_0_val = rand.normal(scale=0.1, size=(384, input_size, 3, 3)) filter2_1_1_vala = rand.normal(scale=0.1, size=(192, input_size, 1, 1)) filter2_1_1_valb = rand.normal(scale=0.1, size=(224, 192, 3, 3)) filter2_1_1_valc = rand.normal(scale=0.1, size=(256, 224, 3, 3)) # branch_0 incep2_1_0 = ad.convolution_2d_forward_op(inputs, filter2_1_0, "NCHW", "VALID", 2, 2) # branch 1 incep2_1_1a = ad.convolution_2d_forward_op(inputs, filter2_1_1a, "NCHW", "SAME", 1, 1) incep2_1_1b = ad.convolution_2d_forward_op(incep2_1_1a, filter2_1_1b, "NCHW", "SAME", 1, 1) incep2_1_1 = ad.convolution_2d_forward_op(incep2_1_1b, filter2_1_1c, "NCHW", "VALID", 2, 2) # branch 2 incep2_1_2 = ad.pooling_2d_forward_op(inputs, "NCHW", "mean", 0, 0, 2, 2, 3, 3) concat2_1a = ad.concat_forward_op(incep2_1_0, incep2_1_1) concat2_1 = ad.concat_forward_op(concat2_1a, incep2_1_2) dict = { filter2_1_0: filter2_1_0_val, filter2_1_1a: filter2_1_1_vala, filter2_1_1b: filter2_1_1_valb, filter2_1_1c: filter2_1_1_valc } return concat2_1, dict
def test_lr(): W = ad.Variable(name="W") b = ad.Variable(name="b") X = ad.Variable(name="X") y_ = ad.Variable(name="y_") ctx = ndarray.gpu(0) # ini x_val = np.linspace(0, 1, 100).reshape((100, 1)) y_val = x_val + 0.5 W_val = np.array([[0.1]]) b_val = np.array([0.1]) x_val = ndarray.array(x_val, ctx) W_val = ndarray.array(W_val, ctx) b_val = ndarray.array(b_val, ctx) y_val = ndarray.array(y_val, ctx) z = ad.matmul_op(X, W) # z.shape = (100,1) # b.shape = (1,1) y = z + ad.broadcastto_op(b, z) # y = (100,1) y = ad.fullyactivation_forward_op(y, "NCHW", "relu") loss = ad.matmul_op(y + (-1) * y_, y + (-1) * y_, trans_A=True) * (1 / 100) # loss = ad.softmaxcrossentropy_op(y, y_) grad_W, grad_b = ad.gradients(loss, [W, b]) executor = ad.Executor([loss, grad_W, grad_b], ctx) aph = 1e-6 for i in range(100): loss_val, grad_W_val, grad_b_val = executor.run(feed_dict={ X: x_val, b: b_val, W: W_val, y_: y_val }) grad_W_val = grad_W_val.asnumpy() W_val = W_val.asnumpy() W_val = W_val - aph * grad_W_val W_val = ndarray.array(W_val, ctx) grad_b_val = grad_b_val.asnumpy() b_val = b_val.asnumpy() b_val = b_val - aph * grad_b_val b_val = ndarray.array(b_val, ctx) print(W_val.asnumpy(), b_val.asnumpy())
def test_l1_l2_regular(): inputs = ad.Variable("inputs") filters = ad.Variable("filters") y_ = ad.Variable(name="y_") # ini ctx = ndarray.gpu(0) x_val = np.ones((5, 2)) * 0.5 x_val = ndarray.array(x_val, ctx) loss = ad.l2regular_op(inputs) # loss = ad.l1regular_op(inputs) grad_f = ad.gradients(loss, [inputs]) # gra返回一个list executor = ad.Executor([loss, grad_f[0]], ctx=ctx) g_val = executor.run(feed_dict={inputs: x_val}) # 返回一个list print("g_val:", g_val[0].asnumpy()) print("g_val:", g_val[1].asnumpy())
def convolutional_block(inputs, kernel_size, in_filter, out_filters, block_name, stride=1): f1, f2, f3 = out_filters W1 = ad.Variable(block_name + "W1") W2 = ad.Variable(block_name + "W2") W3 = ad.Variable(block_name + "W3") W_shortcut = ad.Variable(block_name + "W_shortcut") rand = np.random.RandomState(seed=123) W1_val = rand.normal(scale=0.1, size=(f1, in_filter, 1, 1)) W2_val = rand.normal(scale=0.1, size=(f2, f1, kernel_size, kernel_size)) W3_val = rand.normal(scale=0.1, size=(f3, f2, 1, 1)) W_shortcut_val = rand.normal(scale=0.1, size=(in_filter, f3, 1, 1)) # conv1 conv1 = ad.convolution_2d_forward_op(inputs, W1, "NCHW", "SAME", stride, stride) bn1 = ad.bn_forward_op(conv1, "NCHW", "pre_activation") act1 = ad.activation_forward_op(bn1, "NCHW", "relu") # conv2 conv2 = ad.convolution_2d_forward_op(act1, W2, "NCHW", "SAME", stride, stride) bn2 = ad.bn_forward_op(conv2, "NCHW", "pre_activation") act2 = ad.activation_forward_op(bn2, "NCHW", "relu") # conv3 conv3 = ad.convolution_2d_forward_op(act2, W3, "NCHW", "VALID", stride, stride) bn3 = ad.bn_forward_op(conv3, "NCHW", "pre_activation") #shortcut_path conv4 = ad.convolution_2d_forward_op(inputs, W_shortcut, "NCHW", "VALID", stride, stride) shortcut = ad.bn_forward_op(conv4, "NCHW", "pre_activation") # shortcut add = ad.add_op(bn3, shortcut) act4 = ad.activation_forward_op(add, "NCHW", "relu") dict = {W1: W1_val, W2: W2_val, W3: W3_val, W_shortcut: W_shortcut_val} return act4, dict
def test_mul_two_vars(): x2 = ad.Variable(name="x2") x3 = ad.Variable(name="x3") y = x2 * x3 grad_x2, grad_x3 = ad.gradients(y, [x2, x3]) executor = ad.Executor([y, grad_x2, grad_x3]) x2_val = 2 * np.ones(3) x3_val = 3 * np.ones(3) y_val, grad_x2_val, grad_x3_val = executor.run(feed_dict={ x2: x2_val, x3: x3_val }) assert isinstance(y, ad.Node) assert np.array_equal(y_val, x2_val * x3_val) assert np.array_equal(grad_x2_val, x3_val) assert np.array_equal(grad_x3_val, x2_val)
def test_exp_grad(): x = ad.Variable("x") y = ad.exp_op(x) x_grad, = ad.gradients(y, [x]) executor = ad.Executor([y, x_grad]) x_val = 1 y_val, x_grad_val = executor.run(feed_dict={x: x_val}) print(y_val) print(x_grad_val)
def test_l1_l2_cross_loss(): inputs = ad.Variable("inputs") filters = ad.Variable("filters") y_ = ad.Variable(name="y_") # ini ctx = ndarray.gpu(0) x_val = np.ones((5, 2)) * 0.5 filters_val = np.ones((2, 2, 10)) * 0.001 y_val = np.ones((5, 2)) x_val = ndarray.array(x_val, ctx) filters_val = ndarray.array(filters_val, ctx) y_val = ndarray.array(y_val, ctx) # loss = ad.crossEntropy_op(inputs, y_) loss = ad.l1loss_op(inputs, y_) grad_f = ad.gradients(loss, [inputs, y_]) # gra返回一个list executor = ad.Executor([loss, grad_f[0], grad_f[1]], ctx=ctx) g_val = executor.run(feed_dict={inputs: x_val, y_: y_val}) # 返回一个list print("g_val:", g_val[0].asnumpy()) print("g_val:", g_val[1].asnumpy()) print("g_val:", g_val[2].asnumpy())
def test_convolution_3d_forward_op(): inputs = ad.Variable("inputs") filters = ad.Variable("filters") y_ = ad.Variable(name="y_") #ini ctx = ndarray.gpu(0) x_val = np.linspace(0, 100, 135).reshape((5, 1, 3, 3, 3)) filters_val = np.ones((1, 1, 2, 2, 2)) * 0.001 y_val = np.zeros((5, 1)) x_val = ndarray.array(x_val, ctx) filters_val = ndarray.array(filters_val, ctx) y_val = ndarray.array(y_val, ctx) outputs = ad.convolution_3d_forward_op(inputs, filters, "NCHW", "VALID", 1, 1, 1) outputs_pool = ad.pooling_3d_forward_op(outputs, "NCHW", "max", 0, 0, 0, 1, 1, 1, 2, 2, 2) outputs_relu = ad.activation_forward_op(outputs_pool, "NCHW", "relu") outputs_dro = ad.dropout_forward_op(outputs_relu, "NCHW", 0.5, 0) outputs_f = ad.flatten_op(outputs_dro) loss = ad.matmul_op(outputs_f, outputs_f, trans_A=True) * (1 / 5) grad_inputs, grad_f = ad.gradients(loss, [inputs, filters]) executor = ad.Executor([loss, grad_f], ctx=ctx) aph = 1.0e-6 for i in range(20): loss_val, filters_grad_val = executor.run(feed_dict={ inputs: x_val, filters: filters_val }) filters_val = filters_val.asnumpy() filters_grad_val = filters_grad_val.asnumpy() filters_val = filters_val - aph * filters_grad_val filters_val = ndarray.array(filters_val, ctx) print("loss_val:", loss_val.asnumpy()) print("filters_val:", filters_val.asnumpy())
def test_exp(): x1 = ad.Variable("x1") x2 = ad.exp_op(x1) x3 = x2 + 1 x4 = x2 * x3 x1_grad, = ad.gradients(x4, [x1]) executor = ad.Executor([x4]) x1_val = 1 x4_val, x1_grad = executor.run(feed_dict={x1: x1_val}) print(x4_val) print(x1_grad)
def test_identity(): x2 = ad.Variable(name="x2") y = x2 grad_x2, = ad.gradients(y, [x2]) executor = ad.Executor([y, grad_x2]) x2_val = 2 * np.ones(3) y_val, grad_x2_val = executor.run(feed_dict={x2: x2_val}) assert isinstance(y, ad.Node) assert np.array_equal(y_val, x2_val) assert np.array_equal(grad_x2_val, np.ones_like(x2_val))
def block_inception_a(inputs, blockname=None): filter1_1_0 = ad.Variable(blockname + "filter1_1_0") filter1_1_1a = ad.Variable(blockname + "filter1_1_1a") filter1_1_1b = ad.Variable(blockname + "filter1_1_1b") filter1_1_2a = ad.Variable(blockname + "filter1_1_2a") filter1_1_2b = ad.Variable(blockname + "filter1_1_2b") filter1_1_2c = ad.Variable(blockname + "filter1_1_2c") filter1_1_3 = ad.Variable(blockname + "filter1_1_3a") rand = np.random.RandomState(seed=123) filter1_1_0_val = rand.normal(scale=0.1, size=(96, 384, 1, 1)) filter1_1_1_vala = rand.normal(scale=0.1, size=(64, 384, 1, 1)) filter1_1_1_valb = rand.normal(scale=0.1, size=(96, 64, 3, 3)) filter1_1_2_vala = rand.normal(scale=0.1, size=(64, 384, 1, 1)) filter1_1_2_valb = rand.normal(scale=0.1, size=(96, 64, 3, 3)) filter1_1_2_valc = rand.normal(scale=0.1, size=(96, 96, 3, 3)) filter1_1_3_val = rand.normal(scale=0.1, size=(96, 384, 1, 1)) # branch_0 incep1_1_0 = ad.convolution_2d_forward_op(inputs, filter1_1_0, "NCHW", "SAME", 1, 1) # branch 1 incep1_1_1a = ad.convolution_2d_forward_op(inputs, filter1_1_1a, "NCHW", "SAME", 1, 1) incep1_1_1 = ad.convolution_2d_forward_op(incep1_1_1a, filter1_1_1b, "NCHW", "SAME", 1, 1) # branch 2 incep1_1_2a = ad.convolution_2d_forward_op(inputs, filter1_1_2a, "NCHW", "SAME", 1, 1) incep1_1_2b = ad.convolution_2d_forward_op(incep1_1_2a, filter1_1_2b, "NCHW", "SAME", 1, 1) incep1_1_2 = ad.convolution_2d_forward_op(incep1_1_2b, filter1_1_2c, "NCHW", "SAME", 1, 1) # branch 3 incep1_1_3a = ad.pooling_2d_forward_op(inputs, "NCHW", "mean", 1, 1, 1, 1, 3, 3) incep1_1_3 = ad.convolution_2d_forward_op(incep1_1_3a, filter1_1_3, "NCHW", "SAME", 1, 1) concat1_1a = ad.concat_forward_op(incep1_1_0, incep1_1_1) concat1_1b = ad.concat_forward_op(concat1_1a, incep1_1_2) concat1_1 = ad.concat_forward_op(concat1_1b, incep1_1_3) dict = { filter1_1_0: filter1_1_0_val, filter1_1_1a: filter1_1_1_vala, filter1_1_1b: filter1_1_1_valb, filter1_1_2a: filter1_1_2_vala, filter1_1_2b: filter1_1_2_valb, filter1_1_2c: filter1_1_2_valc, filter1_1_3: filter1_1_3_val } return concat1_1, dict
def test_sigmoid_conv_1d(): inputs = ad.Variable("inputs") filters = ad.Variable("filters") y_ = ad.Variable(name="y_") # ini ctx = ndarray.gpu(0) x_val = np.linspace(0, 100, 80).reshape((5, 1, 4, 4)) filters_val = np.ones((1, 1, 3, 3)) * 0.001 y_val = np.zeros((5, 1)) x_val = ndarray.array(x_val, ctx) filters_val = ndarray.array(filters_val, ctx) y_val = ndarray.array(y_val, ctx) outputs = ad.convolution_2d_forward_op(inputs, filters, "NCHW", "VALID", 1, 1) # outputs_pool = ad.pooling_2d_forward_op(outputs, "NCHW", "max", 0, 0, 1, 1, 2, 2) outputs_relu = ad.activation_forward_op(outputs, "NCHW", "relu") executor = ad.Executor([outputs_relu], ctx=ctx) loss_val = executor.run(feed_dict={inputs: x_val, filters: filters_val}) print("loss_val:", loss_val[0].asnumpy())
def test_exp_log_reverse_pow(): inputs = ad.Variable("inputs") filters = ad.Variable("filters") y_ = ad.Variable(name="y_") # ini ctx = ndarray.gpu(0) x_val = np.linspace(0, 100, 80).reshape((5, 1, 4, 4)) filters_val = np.ones((1, 1, 3, 3)) * 0.001 y_val = np.zeros((5, 1)) x_val = ndarray.array(x_val, ctx) filters_val = ndarray.array(filters_val, ctx) y_val = ndarray.array(y_val, ctx) #outputs = ad.exp_op(inputs) #outputs = ad.log_op(inputs) #outputs = ad.reverse_op(inputs) outputs = ad.pow_op(inputs, 2) grad_out = ad.gradients(outputs, [inputs]) executor = ad.Executor([outputs, grad_out[0]], ctx=ctx) result = executor.run(feed_dict={inputs: filters_val}) print(result[0].asnumpy()) print(result[1].asnumpy())
def test_reduce_mean(): inputs = ad.Variable("inputs") ctx = ndarray.gpu(0) shape = (2, 2, 3) x = np.random.uniform(0, 20, shape).astype(np.float32) arr_x = ndarray.array(x, ctx=ctx) outputs = ad.reduce_mean_op(inputs, 1) f_out = ad.pow_op(outputs, 2) grad_out = ad.gradients(f_out, [inputs]) executor = ad.Executor([outputs, f_out, grad_out[0]], ctx=ctx) result = executor.run(feed_dict={inputs: arr_x}) print(arr_x.asnumpy()) print(result[0].asnumpy()) print(result[1].asnumpy()) print(result[2].asnumpy())
def block_reduction_b(inputs, input_size, blockname=None): filter3_1_0a = ad.Variable(blockname + "filter3_1_0a") filter3_1_0b = ad.Variable(blockname + "filter3_1_0b") filter3_1_1a = ad.Variable(blockname + "filter3_1_1a") filter3_1_1b = ad.Variable(blockname + "filter3_1_1b") filter3_1_1c = ad.Variable(blockname + "filter3_1_1c") filter3_1_1d = ad.Variable(blockname + "filter3_1_1d") rand = np.random.RandomState(seed=123) filter3_1_0_vala = rand.normal(scale=0.1, size=(192, input_size, 1, 1)) filter3_1_0_valb = rand.normal(scale=0.1, size=(192, 192, 3, 3)) filter3_1_1_vala = rand.normal(scale=0.1, size=(256, input_size, 1, 1)) filter3_1_1_valb = rand.normal(scale=0.1, size=(256, 256, 1, 7)) filter3_1_1_valc = rand.normal(scale=0.1, size=(320, 256, 7, 1)) filter3_1_1_vald = rand.normal(scale=0.1, size=(320, 320, 3, 3)) # branch_0 incep3_1_0a = ad.convolution_2d_forward_op(inputs, filter3_1_0a, "NCHW", "SAME", 1, 1) incep3_1_0 = ad.convolution_2d_forward_op(incep3_1_0a, filter3_1_0b, "NCHW", "VALID", 2, 2) # branch 1 incep3_1_1a = ad.convolution_2d_forward_op(inputs, filter3_1_1a, "NCHW", "SAME", 1, 1) incep3_1_1b = ad.convolution_2d_forward_op(incep3_1_1a, filter3_1_1b, "NCHW", "SAME", 1, 1) incep3_1_1c = ad.convolution_2d_forward_op(incep3_1_1b, filter3_1_1c, "NCHW", "SAME", 1, 1) incep3_1_1 = ad.convolution_2d_forward_op(incep3_1_1c, filter3_1_1d, "NCHW", "VALID", 2, 2) # branch 2 incep3_1_2 = ad.pooling_2d_forward_op(inputs, "NCHW", "mean", 0, 0, 2, 2, 3, 3) concat3_1a = ad.concat_forward_op(incep3_1_0, incep3_1_1) concat3_1 = ad.concat_forward_op(concat3_1a, incep3_1_2) dict = { filter3_1_0a: filter3_1_0_vala, filter3_1_0b: filter3_1_0_valb, filter3_1_1a: filter3_1_1_vala, filter3_1_1b: filter3_1_1_valb, filter3_1_1c: filter3_1_1_valc, filter3_1_1d: filter3_1_1_vald } return concat3_1, dict
def inception_v3(): X = ad.Placeholder("inputs") y_ = ad.Placeholder("y_") inputs = ad.Variable("inputs") filterb_1 = ad.Variable("filterb_1") filterb_2 = ad.Variable("filterb_2") filterb_3 = ad.Variable("filterb_3") filterb_4 = ad.Variable("filterb_4") filterb_5 = ad.Variable("filterb_5") ctx = ndarray.gpu(0) filtersb_val1 = np.ones((32, 3, 3, 3)) * 0.01 filtersb_val2 = np.ones((32, 32, 3, 3)) * 0.01 filtersb_val3 = np.ones((64, 32, 3, 3)) * 0.01 filtersb_val4 = np.ones((80, 64, 1, 1)) * 0.01 filtersb_val5 = np.ones((192, 80, 3, 3)) * 0.01 y_val = np.zeros((147, 147, 64)) #inception前 covb_1 = ad.convolution_2d_forward_op(inputs, filterb_1, "NCHW", "VALID", 2, 2) covb_2 = ad.convolution_2d_forward_op(covb_1, filterb_2, "NCHW", "VALID", 1, 1) covb_3 = ad.convolution_2d_forward_op(covb_2, filterb_3, "NCHW", "SAME", 1, 1) poolb = ad.pooling_2d_forward_op(covb_3, "NCHW", "max", 0, 0, 2, 2, 3, 3) covb_4 = ad.convolution_2d_forward_op(poolb, filterb_4, "NCHW", "VALID", 1, 1) covb_5 = ad.convolution_2d_forward_op(covb_4, filterb_5, "NCHW", "VALID", 1, 1) covb = ad.pooling_2d_forward_op(covb_5, "NCHW", "max", 0, 0, 2, 2, 3, 3) # inception_moudle1 #inception_moudle1_1 filter1_1_0 = ad.Variable("filter1_1_0") filter1_1_1a = ad.Variable("filter1_1_1a") filter1_1_1b = ad.Variable("filter1_1_1b") filter1_1_2a = ad.Variable("filter1_1_2a") filter1_1_2b = ad.Variable("filter1_1_2b") filter1_1_2c = ad.Variable("filter1_1_2c") filter1_1_3 = ad.Variable("filter1_1_3a") filter1_1_0_val = np.ones((64, 192, 1, 1)) * 0.01 filter1_1_1_vala = np.ones((48, 192, 1, 1)) * 0.01 filter1_1_1_valb = np.ones((64, 48, 5, 5)) * 0.01 filter1_1_2_vala = np.ones((64, 192, 1, 1)) * 0.01 filter1_1_2_valb = np.ones((96, 64, 3, 3)) * 0.01 filter1_1_2_valc = np.ones((96, 96, 3, 3)) * 0.01 filter1_1_3_val = np.ones((32, 192, 1, 1)) * 0.01 # branch_0 incep1_1_0 = ad.convolution_2d_forward_op(covb, filter1_1_0, "NCHW", "SAME", 1, 1) # branch 1 incep1_1_1a = ad.convolution_2d_forward_op(covb, filter1_1_1a, "NCHW", "SAME", 1, 1) incep1_1_1 = ad.convolution_2d_forward_op(incep1_1_1a, filter1_1_1b, "NCHW", "SAME", 1, 1) # branch 2 incep1_1_2a = ad.convolution_2d_forward_op(covb, filter1_1_2a, "NCHW", "SAME", 1, 1) incep1_1_2b = ad.convolution_2d_forward_op(incep1_1_2a, filter1_1_2b, "NCHW", "SAME", 1, 1) incep1_1_2 = ad.convolution_2d_forward_op(incep1_1_2b, filter1_1_2c, "NCHW", "SAME", 1, 1) # branch 3 incep1_1_3a = ad.pooling_2d_forward_op(covb, "NCHW", "mean", 1, 1, 1, 1, 3, 3) incep1_1_3 = ad.convolution_2d_forward_op(incep1_1_3a, filter1_1_3, "NCHW", "SAME", 1, 1) concat1_1a = ad.concat_forward_op(incep1_1_0, incep1_1_1) concat1_1b = ad.concat_forward_op(concat1_1a, incep1_1_2) concat1_1 = ad.concat_forward_op(concat1_1b, incep1_1_3) #inception_moudle1_2 filter1_2_0 = ad.Variable("filter1_2_0") filter1_2_1a = ad.Variable("filter1_2_1a") filter1_2_1b = ad.Variable("filter1_2_1b") filter1_2_2a = ad.Variable("filter1_2_2a") filter1_2_2b = ad.Variable("filter1_2_2b") filter1_2_2c = ad.Variable("filter1_2_2c") filter1_2_3 = ad.Variable("filter1_2_3a") filter1_2_0_val = np.ones((64, 256, 1, 1)) * 0.01 filter1_2_1_vala = np.ones((48, 256, 1, 1)) * 0.01 filter1_2_1_valb = np.ones((64, 48, 5, 5)) * 0.01 filter1_2_2_vala = np.ones((64, 256, 1, 1)) * 0.01 filter1_2_2_valb = np.ones((96, 64, 3, 3)) * 0.01 filter1_2_2_valc = np.ones((96, 96, 3, 3)) * 0.01 filter1_2_3_val = np.ones((64, 256, 1, 1)) * 0.01 # branch_0 incep1_2_0 = ad.convolution_2d_forward_op(concat1_1, filter1_2_0, "NCHW", "SAME", 1, 1) # branch 1 incep1_2_1a = ad.convolution_2d_forward_op(concat1_1, filter1_2_1a, "NCHW", "SAME", 1, 1) incep1_2_1 = ad.convolution_2d_forward_op(incep1_2_1a, filter1_2_1b, "NCHW", "SAME", 1, 1) # branch 2 incep1_2_2a = ad.convolution_2d_forward_op(concat1_1, filter1_2_2a, "NCHW", "SAME", 1, 1) incep1_2_2b = ad.convolution_2d_forward_op(incep1_2_2a, filter1_2_2b, "NCHW", "SAME", 1, 1) incep1_2_2 = ad.convolution_2d_forward_op(incep1_2_2b, filter1_2_2c, "NCHW", "SAME", 1, 1) # branch 3 incep1_2_3a = ad.pooling_2d_forward_op(concat1_1, "NCHW", "mean", 1, 1, 1, 1, 3, 3) incep1_2_3 = ad.convolution_2d_forward_op(incep1_2_3a, filter1_2_3, "NCHW", "SAME", 1, 1) concat1_2a = ad.concat_forward_op(incep1_2_0, incep1_2_1) concat1_2b = ad.concat_forward_op(concat1_2a, incep1_2_2) concat1_2 = ad.concat_forward_op(concat1_2b, incep1_2_3) # inception_moudle1_3 filter1_3_0 = ad.Variable("filter1_3_0") filter1_3_1a = ad.Variable("filter1_3_1a") filter1_3_1b = ad.Variable("filter1_3_1b") filter1_3_2a = ad.Variable("filter1_3_2a") filter1_3_2b = ad.Variable("filter1_3_2b") filter1_3_2c = ad.Variable("filter1_3_2c") filter1_3_3 = ad.Variable("filter1_3_3a") filter1_3_0_val = np.ones((64, 288, 1, 1)) * 0.01 filter1_3_1_vala = np.ones((48, 288, 1, 1)) * 0.01 filter1_3_1_valb = np.ones((64, 48, 5, 5)) * 0.01 filter1_3_2_vala = np.ones((64, 288, 1, 1)) * 0.01 filter1_3_2_valb = np.ones((96, 64, 3, 3)) * 0.01 filter1_3_2_valc = np.ones((96, 96, 3, 3)) * 0.01 filter1_3_3_val = np.ones((64, 288, 1, 1)) * 0.01 # branch_0 incep1_3_0 = ad.convolution_2d_forward_op(concat1_2, filter1_3_0, "NCHW", "SAME", 1, 1) # branch 1 incep1_3_1a = ad.convolution_2d_forward_op(concat1_2, filter1_3_1a, "NCHW", "SAME", 1, 1) incep1_3_1 = ad.convolution_2d_forward_op(incep1_3_1a, filter1_3_1b, "NCHW", "SAME", 1, 1) # branch 2 incep1_3_2a = ad.convolution_2d_forward_op(concat1_2, filter1_3_2a, "NCHW", "SAME", 1, 1) incep1_3_2b = ad.convolution_2d_forward_op(incep1_3_2a, filter1_3_2b, "NCHW", "SAME", 1, 1) incep1_3_2 = ad.convolution_2d_forward_op(incep1_3_2b, filter1_3_2c, "NCHW", "SAME", 1, 1) # branch 3 incep1_3_3a = ad.pooling_2d_forward_op(concat1_2, "NCHW", "mean", 1, 1, 1, 1, 3, 3) incep1_3_3 = ad.convolution_2d_forward_op(incep1_3_3a, filter1_3_3, "NCHW", "SAME", 1, 1) concat1_3a = ad.concat_forward_op(incep1_3_0, incep1_3_1) concat1_3b = ad.concat_forward_op(concat1_3a, incep1_3_2) concat1_3 = ad.concat_forward_op(concat1_3b, incep1_3_3) # # # # # # inception_moudle2 # inception_moudle2_1 filter2_1_0 = ad.Variable("filter2_1_0") filter2_1_1a = ad.Variable("filter2_1_1a") filter2_1_1b = ad.Variable("filter2_1_1b") filter2_1_1c = ad.Variable("filter2_1_1c") filter2_1_0_val = np.ones((384, 288, 3, 3)) * 0.01 filter2_1_1_vala = np.ones((64, 288, 1, 1)) * 0.01 filter2_1_1_valb = np.ones((96, 64, 3, 3)) * 0.01 filter2_1_1_valc = np.ones((96, 96, 3, 3)) * 0.01 # branch_0 incep2_1_0 = ad.convolution_2d_forward_op(concat1_3, filter2_1_0, "NCHW", "VALID", 2, 2) # branch 1 incep2_1_1a = ad.convolution_2d_forward_op(concat1_3, filter2_1_1a, "NCHW", "SAME", 1, 1) incep2_1_1b = ad.convolution_2d_forward_op(incep2_1_1a, filter2_1_1b, "NCHW", "SAME", 1, 1) incep2_1_1 = ad.convolution_2d_forward_op(incep2_1_1b, filter2_1_1c, "NCHW", "VALID", 2, 2) # branch 2 incep2_1_2 = ad.pooling_2d_forward_op(concat1_3, "NCHW", "mean", 0, 0, 2, 2, 3, 3) concat2_1a = ad.concat_forward_op(incep2_1_0, incep2_1_1) concat2_1 = ad.concat_forward_op(concat2_1a, incep2_1_2) # inception_moudle2_2 filter2_2_0 = ad.Variable("filter2_2_0") filter2_2_1a = ad.Variable("filter2_2_1a") filter2_2_1b = ad.Variable("filter2_2_1b") filter2_2_1c = ad.Variable("filter2_2_1c") filter2_2_2a = ad.Variable("filter2_2_2a") filter2_2_2b = ad.Variable("filter2_2_2b") filter2_2_2c = ad.Variable("filter2_2_2c") filter2_2_2d = ad.Variable("filter2_2_2d") filter2_2_2e = ad.Variable("filter2_2_2e") filter2_2_3 = ad.Variable("filter2_2_3a") filter2_2_0_val = np.ones((192, 768, 1, 1)) * 0.01 filter2_2_1_vala = np.ones((128, 768, 1, 1)) * 0.01 filter2_2_1_valb = np.ones((128, 128, 1, 7)) * 0.01 filter2_2_1_valc = np.ones((192, 128, 7, 1)) * 0.01 filter2_2_2_vala = np.ones((128, 768, 1, 1)) * 0.01 filter2_2_2_valb = np.ones((128, 128, 7, 1)) * 0.01 filter2_2_2_valc = np.ones((128, 128, 1, 7)) * 0.01 filter2_2_2_vald = np.ones((128, 128, 7, 1)) * 0.01 filter2_2_2_vale = np.ones((192, 128, 1, 7)) * 0.01 filter2_2_3_val = np.ones((192, 768, 1, 1)) * 0.01 # branch_0 incep2_2_0 = ad.convolution_2d_forward_op(concat2_1, filter2_2_0, "NCHW", "SAME", 1, 1) # branch 1 incep2_2_1a = ad.convolution_2d_forward_op(concat2_1, filter2_2_1a, "NCHW", "SAME", 1, 1) incep2_2_1b = ad.convolution_2d_forward_op(incep2_2_1a, filter2_2_1b, "NCHW", "SAME", 1, 1) incep2_2_1 = ad.convolution_2d_forward_op(incep2_2_1b, filter2_2_1c, "NCHW", "SAME", 1, 1) # branch 2 incep2_2_2a = ad.convolution_2d_forward_op(concat2_1, filter2_2_2a, "NCHW", "SAME", 1, 1) incep2_2_2b = ad.convolution_2d_forward_op(incep2_2_2a, filter2_2_2b, "NCHW", "SAME", 1, 1) incep2_2_2c = ad.convolution_2d_forward_op(incep2_2_2b, filter2_2_2c, "NCHW", "SAME", 1, 1) incep2_2_2d = ad.convolution_2d_forward_op(incep2_2_2c, filter2_2_2d, "NCHW", "SAME", 1, 1) incep2_2_2 = ad.convolution_2d_forward_op(incep2_2_2d, filter2_2_2e, "NCHW", "SAME", 1, 1) # branch 3 incep2_2_3a = ad.pooling_2d_forward_op(concat2_1, "NCHW", "mean", 1, 1, 1, 1, 3, 3) incep2_2_3 = ad.convolution_2d_forward_op(incep2_2_3a, filter2_2_3, "NCHW", "SAME", 1, 1) concat2_2a = ad.concat_forward_op(incep2_2_0, incep2_2_1) concat2_2b = ad.concat_forward_op(concat2_2a, incep2_2_2) concat2_2 = ad.concat_forward_op(concat2_2b, incep2_2_3) # inception_moudle2_3 # filter2_3_0 = ad.Variable("filter2_3_0") # filter2_3_1a = ad.Variable("filter2_3_1a") # filter2_3_1b = ad.Variable("filter2_3_1b") # filter2_3_1c = ad.Variable("filter2_3_1c") # filter2_3_2a = ad.Variable("filter2_3_2a") # filter2_3_2b = ad.Variable("filter2_3_2b") # filter2_3_2c = ad.Variable("filter2_3_2c") # filter2_3_2d = ad.Variable("filter2_3_2d") # filter2_3_2e = ad.Variable("filter2_3_2e") # filter2_3_3 = ad.Variable("filter2_3_3a") # # filter2_3_0_val = np.ones((192, 768, 1, 1)) * 0.01 # filter2_3_1_vala = np.ones((160, 768, 1, 1)) * 0.01 # filter2_3_1_valb = np.ones((160, 160, 1, 7)) * 0.01 # filter2_3_1_valc = np.ones((192, 160, 7, 1)) * 0.01 # filter2_3_2_vala = np.ones((160, 768, 1, 1)) * 0.01 # filter2_3_2_valb = np.ones((160, 160, 7,1)) * 0.01 # filter2_3_2_valc = np.ones((160, 160, 1, 7)) * 0.01 # filter2_3_2_vald = np.ones((160, 160, 7, 1)) * 0.01 # filter2_3_2_vale = np.ones((192, 160, 1, 7)) * 0.01 # filter2_3_3_val = np.ones((192, 768, 1, 1)) * 0.01 # # # branch_0 # incep2_3_0 = ad.convolution_2d_forward_op(concat2_2, filter2_3_0, "NCHW", "SAME", 1, 1) # # branch 1 # incep2_3_1a = ad.convolution_2d_forward_op(concat2_2, filter2_3_1a, "NCHW", "SAME", 1, 1) # incep2_3_1b = ad.convolution_2d_forward_op(incep2_3_1a, filter2_3_1b, "NCHW", "SAME", 1, 1) # incep2_3_1 = ad.convolution_2d_forward_op(incep2_3_1b, filter2_3_1c, "NCHW", "SAME", 1, 1) # # branch 2 # incep2_3_2a = ad.convolution_2d_forward_op(concat2_2, filter2_3_2a, "NCHW", "SAME", 1, 1) # incep2_3_2b = ad.convolution_2d_forward_op(incep2_3_2a, filter2_3_2b, "NCHW", "SAME", 1, 1) # incep2_3_2c = ad.convolution_2d_forward_op(incep2_3_2b, filter2_3_2c, "NCHW", "SAME", 1, 1) # incep2_3_2d = ad.convolution_2d_forward_op(incep2_3_2c, filter2_3_2d, "NCHW", "SAME", 1, 1) # incep2_3_2 = ad.convolution_2d_forward_op(incep2_3_2d, filter2_3_2e, "NCHW", "SAME", 1, 1) # # branch 3 # incep2_3_3a = ad.pooling_2d_forward_op(concat2_2, "NCHW", "mean", 1, 1, 1, 1, 3, 3) # incep2_3_3 = ad.convolution_2d_forward_op(incep2_3_3a, filter2_3_3, "NCHW", "SAME", 1, 1) # # concat2_3a = ad.concat_forward_op(incep2_3_0, incep2_3_1) # concat2_3b = ad.concat_forward_op(concat2_3a, incep2_3_2) # concat2_3 = ad.concat_forward_op(concat2_3b, incep2_3_3) # inception_moudle2_4 # filter2_4_0 = ad.Variable("filter2_4_0") # filter2_4_1a = ad.Variable("filter2_4_1a") # filter2_4_1b = ad.Variable("filter2_4_1b") # filter2_4_1c = ad.Variable("filter2_4_1c") # filter2_4_2a = ad.Variable("filter2_4_2a") # filter2_4_2b = ad.Variable("filter2_4_2b") # filter2_4_2c = ad.Variable("filter2_4_2c") # filter2_4_2d = ad.Variable("filter2_4_2d") # filter2_4_2e = ad.Variable("filter2_4_2e") # filter2_4_3 = ad.Variable("filter2_4_3a") # # filter2_4_0_val = np.ones((192, 768, 1, 1)) * 0.01 # filter2_4_1_vala = np.ones((160, 768, 1, 1)) * 0.01 # filter2_4_1_valb = np.ones((160, 160, 1, 7)) * 0.01 # filter2_4_1_valc = np.ones((192, 160, 7, 1)) * 0.01 # filter2_4_2_vala = np.ones((160, 768, 1, 1)) * 0.01 # filter2_4_2_valb = np.ones((160, 160, 7, 1)) * 0.01 # filter2_4_2_valc = np.ones((160, 160, 1, 7)) * 0.01 # filter2_4_2_vald = np.ones((160, 160, 7, 1)) * 0.01 # filter2_4_2_vale = np.ones((192, 160, 1, 7)) * 0.01 # filter2_4_3_val = np.ones((192, 768, 1, 1)) * 0.01 # # # branch_0 # incep2_4_0 = ad.convolution_2d_forward_op(concat2_3, filter2_4_0, "NCHW", "SAME", 1, 1) # # branch 1 # incep2_4_1a = ad.convolution_2d_forward_op(concat2_3, filter2_4_1a, "NCHW", "SAME", 1, 1) # incep2_4_1b = ad.convolution_2d_forward_op(incep2_4_1a, filter2_4_1b, "NCHW", "SAME", 1, 1) # incep2_4_1 = ad.convolution_2d_forward_op(incep2_4_1b, filter2_4_1c, "NCHW", "SAME", 1, 1) # # branch 2 # incep2_4_2a = ad.convolution_2d_forward_op(concat2_3, filter2_4_2a, "NCHW", "SAME", 1, 1) # incep2_4_2b = ad.convolution_2d_forward_op(incep2_4_2a, filter2_4_2b, "NCHW", "SAME", 1, 1) # incep2_4_2c = ad.convolution_2d_forward_op(incep2_4_2b, filter2_4_2c, "NCHW", "SAME", 1, 1) # incep2_4_2d = ad.convolution_2d_forward_op(incep2_4_2c, filter2_4_2d, "NCHW", "SAME", 1, 1) # incep2_4_2 = ad.convolution_2d_forward_op(incep2_4_2d, filter2_4_2e, "NCHW", "SAME", 1, 1) # # branch 3 # incep2_4_3a = ad.pooling_2d_forward_op(concat2_3, "NCHW", "mean", 1, 1, 1, 1, 3, 3) # incep2_4_3 = ad.convolution_2d_forward_op(incep2_4_3a, filter2_4_3, "NCHW", "SAME", 1, 1) # # concat2_4a = ad.concat_forward_op(incep2_4_0, incep2_4_1) # concat2_4b = ad.concat_forward_op(concat2_4a, incep2_4_2) # concat2_4 = ad.concat_forward_op(concat2_4b, incep2_4_3) # inception_moudle2_5 # filter2_5_0 = ad.Variable("filter2_5_0") # filter2_5_1a = ad.Variable("filter2_5_1a") # filter2_5_1b = ad.Variable("filter2_5_1b") # filter2_5_1c = ad.Variable("filter2_5_1c") # filter2_5_2a = ad.Variable("filter2_5_2a") # filter2_5_2b = ad.Variable("filter2_5_2b") # filter2_5_2c = ad.Variable("filter2_5_2c") # filter2_5_2d = ad.Variable("filter2_5_2d") # filter2_5_2e = ad.Variable("filter2_5_2e") # filter2_5_3 = ad.Variable("filter2_5_3a") # # filter2_5_0_val = np.ones((192, 768, 1, 1)) * 0.01 # filter2_5_1_vala = np.ones((160, 768, 1, 1)) * 0.01 # filter2_5_1_valb = np.ones((160, 160, 1, 7)) * 0.01 # filter2_5_1_valc = np.ones((192, 160, 7, 1)) * 0.01 # filter2_5_2_vala = np.ones((160, 768, 1, 1)) * 0.01 # filter2_5_2_valb = np.ones((160, 160, 7, 1)) * 0.01 # filter2_5_2_valc = np.ones((160, 160, 1, 7)) * 0.01 # filter2_5_2_vald = np.ones((160, 160, 7, 1)) * 0.01 # filter2_5_2_vale = np.ones((192, 160, 1, 7)) * 0.01 # filter2_5_3_val = np.ones((192, 768, 1, 1)) * 0.01 # # # branch_0 # incep2_5_0 = ad.convolution_2d_forward_op(concat2_4, filter2_5_0, "NCHW", "SAME", 1, 1) # # branch 1 # incep2_5_1a = ad.convolution_2d_forward_op(concat2_4, filter2_5_1a, "NCHW", "SAME", 1, 1) # incep2_5_1b = ad.convolution_2d_forward_op(incep2_5_1a, filter2_5_1b, "NCHW", "SAME", 1, 1) # incep2_5_1 = ad.convolution_2d_forward_op(incep2_5_1b, filter2_5_1c, "NCHW", "SAME", 1, 1) # # branch 2 # incep2_5_2a = ad.convolution_2d_forward_op(concat2_4, filter2_5_2a, "NCHW", "SAME", 1, 1) # incep2_5_2b = ad.convolution_2d_forward_op(incep2_5_2a, filter2_5_2b, "NCHW", "SAME", 1, 1) # incep2_5_2c = ad.convolution_2d_forward_op(incep2_5_2b, filter2_5_2c, "NCHW", "SAME", 1, 1) # incep2_5_2d = ad.convolution_2d_forward_op(incep2_5_2c, filter2_5_2d, "NCHW", "SAME", 1, 1) # incep2_5_2 = ad.convolution_2d_forward_op(incep2_5_2d, filter2_5_2e, "NCHW", "SAME", 1, 1) # # branch 3 # incep2_5_3a = ad.pooling_2d_forward_op(concat2_4, "NCHW", "mean", 1, 1, 1, 1, 3, 3) # incep2_5_3 = ad.convolution_2d_forward_op(incep2_5_3a, filter2_5_3, "NCHW", "SAME", 1, 1) # # concat2_5a = ad.concat_forward_op(incep2_5_0, incep2_5_1) # concat2_5b = ad.concat_forward_op(concat2_5a, incep2_5_2) # concat2_5 = ad.concat_forward_op(concat2_5b, incep2_5_3) # # inception_moudle3 # inception_moudle3_1 filter3_1_0a = ad.Variable("filter3_1_0a") filter3_1_0b = ad.Variable("filter3_1_0b") filter3_1_1a = ad.Variable("filter3_1_1a") filter3_1_1b = ad.Variable("filter3_1_1b") filter3_1_1c = ad.Variable("filter3_1_1c") filter3_1_1d = ad.Variable("filter3_1_1d") filter3_1_0_vala = np.ones((192, 768, 1, 1)) * 0.01 filter3_1_0_valb = np.ones((320, 192, 3, 3)) * 0.01 filter3_1_1_vala = np.ones((192, 768, 1, 1)) * 0.01 filter3_1_1_valb = np.ones((192, 192, 1, 7)) * 0.01 filter3_1_1_valc = np.ones((192, 192, 7, 1)) * 0.01 filter3_1_1_vald = np.ones((192, 192, 3, 3)) * 0.01 # branch_0 incep3_1_0a = ad.convolution_2d_forward_op(concat2_2, filter3_1_0a, "NCHW", "SAME", 1, 1) incep3_1_0 = ad.convolution_2d_forward_op(incep3_1_0a, filter3_1_0b, "NCHW", "VALID", 2, 2) # branch 1 incep3_1_1a = ad.convolution_2d_forward_op(concat2_2, filter3_1_1a, "NCHW", "SAME", 1, 1) incep3_1_1b = ad.convolution_2d_forward_op(incep3_1_1a, filter3_1_1b, "NCHW", "SAME", 1, 1) incep3_1_1c = ad.convolution_2d_forward_op(incep3_1_1b, filter3_1_1c, "NCHW", "SAME", 1, 1) incep3_1_1 = ad.convolution_2d_forward_op(incep3_1_1c, filter3_1_1d, "NCHW", "VALID", 2, 2) # branch 2 incep3_1_2 = ad.pooling_2d_forward_op(concat2_2, "NCHW", "mean", 0, 0, 2, 2, 3, 3) concat3_1a = ad.concat_forward_op(incep3_1_0, incep3_1_1) concat3_1 = ad.concat_forward_op(concat3_1a, incep3_1_2) # inception_moudle3_2 filter3_2_0 = ad.Variable("filter3_2_0") filter3_2_1a = ad.Variable("filter3_2_1a") filter3_2_1b = ad.Variable("filter3_2_1b") filter3_2_1c = ad.Variable("filter3_2_1c") filter3_2_2a = ad.Variable("filter3_2_2a") filter3_2_2b = ad.Variable("filter3_2_2b") filter3_2_2c = ad.Variable("filter3_2_2c") filter3_2_2d = ad.Variable("filter3_2_2d") filter3_2_3 = ad.Variable("filter3_2_3a") filter3_2_0_val = np.ones((320, 1280, 1, 1)) * 0.01 filter3_2_1_vala = np.ones((384, 1280, 1, 1)) * 0.01 filter3_2_1_valb = np.ones((384, 384, 1, 3)) * 0.01 filter3_2_1_valc = np.ones((384, 384, 3, 1)) * 0.01 filter3_2_2_vala = np.ones((448, 1280, 1, 1)) * 0.01 filter3_2_2_valb = np.ones((384, 448, 3, 3)) * 0.01 filter3_2_2_valc = np.ones((384, 384, 1, 3)) * 0.01 filter3_2_2_vald = np.ones((384, 384, 3, 1)) * 0.01 filter3_2_3_val = np.ones((192, 1280, 1, 1)) * 0.01 # branch_0 incep3_2_0 = ad.convolution_2d_forward_op(concat3_1, filter3_2_0, "NCHW", "SAME", 1, 1) # branch 1 incep3_2_1a = ad.convolution_2d_forward_op(concat3_1, filter3_2_1a, "NCHW", "SAME", 1, 1) incep3_2_1b = ad.convolution_2d_forward_op(incep3_2_1a, filter3_2_1b, "NCHW", "SAME", 1, 1) incep3_2_1c = ad.convolution_2d_forward_op(incep3_2_1a, filter3_2_1c, "NCHW", "SAME", 1, 1) incep3_2_1 = ad.concat_forward_op(incep3_2_1b, incep3_2_1c) # branch 2 incep3_2_2a = ad.convolution_2d_forward_op(concat3_1, filter3_2_2a, "NCHW", "SAME", 1, 1) incep3_2_2b = ad.convolution_2d_forward_op(incep3_2_2a, filter3_2_2b, "NCHW", "SAME", 1, 1) incep3_2_2c = ad.convolution_2d_forward_op(incep3_2_2b, filter3_2_2c, "NCHW", "SAME", 1, 1) incep3_2_2d = ad.convolution_2d_forward_op(incep3_2_2b, filter3_2_2d, "NCHW", "SAME", 1, 1) incep3_2_2 = ad.concat_forward_op(incep3_2_2c, incep3_2_2d) # branch 3 incep3_2_3a = ad.pooling_2d_forward_op(concat3_1, "NCHW", "mean", 1, 1, 1, 1, 3, 3) incep3_2_3 = ad.convolution_2d_forward_op(incep3_2_3a, filter3_2_3, "NCHW", "SAME", 1, 1) concat3_2a = ad.concat_forward_op(incep3_2_0, incep3_2_1) concat3_2b = ad.concat_forward_op(concat3_2a, incep3_2_2) concat3_2 = ad.concat_forward_op(concat3_2b, incep3_2_3) # # inception_moudle3_3 # filter3_3_0 = ad.Variable("filter3_3_0") # filter3_3_1a = ad.Variable("filter3_3_1a") # filter3_3_1b = ad.Variable("filter3_3_1b") # filter3_3_1c = ad.Variable("filter3_3_1c") # filter3_3_2a = ad.Variable("filter3_3_2a") # filter3_3_2b = ad.Variable("filter3_3_2b") # filter3_3_2c = ad.Variable("filter3_3_2c") # filter3_3_2d = ad.Variable("filter3_3_2d") # filter3_3_3 = ad.Variable("filter3_3_3a") # # filter3_3_0_val = np.ones((320, 2048, 1, 1)) * 0.01 # filter3_3_1_vala = np.ones((384, 2048, 1, 1)) * 0.01 # filter3_3_1_valb = np.ones((384, 384, 1, 3)) * 0.01 # filter3_3_1_valc = np.ones((384, 384, 3, 1)) * 0.01 # filter3_3_2_vala = np.ones((448, 2048, 1, 1)) * 0.01 # filter3_3_2_valb = np.ones((384, 448, 3, 3)) * 0.01 # filter3_3_2_valc = np.ones((384, 384, 1, 3)) * 0.01 # filter3_3_2_vald = np.ones((384, 384, 3, 1)) * 0.01 # filter3_3_3_val = np.ones((192, 2048, 1, 1)) * 0.01 # # # branch_0 # incep3_3_0 = ad.convolution_2d_forward_op(concat3_2, filter3_3_0, "NCHW", "SAME", 1, 1) # # branch 1 # incep3_3_1a = ad.convolution_2d_forward_op(concat3_2, filter3_3_1a, "NCHW", "SAME", 1, 1) # incep3_3_1b = ad.convolution_2d_forward_op(incep3_3_1a, filter3_3_1b, "NCHW", "SAME", 1, 1) # incep3_3_1c = ad.convolution_2d_forward_op(incep3_3_1a, filter3_3_1c, "NCHW", "SAME", 1, 1) # incep3_3_1 = ad.concat_forward_op(incep3_3_1b, incep3_3_1c) # # # branch 2 # incep3_3_2a = ad.convolution_2d_forward_op(concat3_2, filter3_3_2a, "NCHW", "SAME", 1, 1) # incep3_3_2b = ad.convolution_2d_forward_op(incep3_3_2a, filter3_3_2b, "NCHW", "SAME", 1, 1) # incep3_3_2c = ad.convolution_2d_forward_op(incep3_3_2b, filter3_3_2c, "NCHW", "SAME", 1, 1) # incep3_3_2d = ad.convolution_2d_forward_op(incep3_3_2b, filter3_3_2d, "NCHW", "SAME", 1, 1) # incep3_3_2 = ad.concat_forward_op(incep3_3_2c, incep3_3_2d) # # branch 3 # incep3_3_3a = ad.pooling_2d_forward_op(concat3_2, "NCHW", "mean", 1, 1, 1, 1, 3, 3) # incep3_3_3 = ad.convolution_2d_forward_op(incep3_3_3a, filter3_3_3, "NCHW", "SAME", 1, 1) # # concat3_3a = ad.concat_forward_op(incep3_3_0, incep3_3_1) # concat3_3b = ad.concat_forward_op(concat3_3a, incep3_3_2) # concat3_3 = ad.concat_forward_op(concat3_3b, incep3_3_3) filtera1 = ad.Variable("filtera1") filtera1val = np.ones((1000, 2048, 1, 1)) * 0.01 filtersmul = ad.Variable("filtersmul") filtersmulval = np.ones((1000, n_class)) * 0.01 biases = ad.Variable("biases") biasesval = np.ones((batch_size, n_class)) * 0.01 poollast = ad.pooling_2d_forward_op(concat3_2, "NCHW", "mean", 0, 0, 1, 1, 8, 8) dropout = ad.dropout_forward_op(poollast, "NCHW", 0.8) convlast = ad.convolution_2d_forward_op(dropout, filtera1, "NCHW", "SAME", 1, 1) squeeze = ad.squeeze_op(convlast) # fc8 mul8 = ad.matmul_op(squeeze, filtersmul) add8 = ad.add_op(mul8, biases) fc8 = ad.fullyactivation_forward_op(add8, "NCHW", "softmax") loss = ad.softmaxcrossentropy_op(fc8, y_) X_val = np.empty(shape=(batch_size, 3, 32, 32), dtype=np.float32) y_val = np.empty(shape=(batch_size, n_class), dtype=np.float32) aph = 0.001 t = train.Adam_minimize(loss, aph) t.init_Variable({ filterb_1: filtersb_val1, filterb_2: filtersb_val2, filterb_3: filtersb_val3, filterb_4: filtersb_val4, filterb_5: filtersb_val5, filter1_1_0: filter1_1_0_val, filter1_1_1a: filter1_1_1_vala, filter1_1_1b: filter1_1_1_valb, filter1_1_2a: filter1_1_2_vala, filter1_1_2b: filter1_1_2_valb, filter1_1_2c: filter1_1_2_valc, filter1_1_3: filter1_1_3_val, filter1_2_0: filter1_2_0_val, filter1_2_1a: filter1_2_1_vala, filter1_2_1b: filter1_2_1_valb, filter1_2_2a: filter1_2_2_vala, filter1_2_2b: filter1_2_2_valb, filter1_2_2c: filter1_2_2_valc, filter1_2_3: filter1_2_3_val, filter1_3_0: filter1_3_0_val, filter1_3_1a: filter1_3_1_vala, filter1_3_1b: filter1_3_1_valb, filter1_3_2a: filter1_3_2_vala, filter1_3_2b: filter1_3_2_valb, filter1_3_2c: filter1_3_2_valc, filter1_3_3: filter1_3_3_val, filter2_1_0: filter2_1_0_val, filter2_1_1a: filter2_1_1_vala, filter2_1_1b: filter2_1_1_valb, filter2_1_1c: filter2_1_1_valc, filter2_2_0: filter2_2_0_val, filter2_2_1a: filter2_2_1_vala, filter2_2_1b: filter2_2_1_valb, filter2_2_1c: filter2_2_1_valc, filter2_2_2a: filter2_2_2_vala, filter2_2_2b: filter2_2_2_valb, filter2_2_2c: filter2_2_2_valc, filter2_2_2d: filter2_2_2_vald, filter2_2_2e: filter2_2_2_vale, filter2_2_3: filter2_2_3_val # , filter2_3_0: filter2_3_0_val, filter2_3_1a: filter2_3_1_vala, filter2_3_1b: filter2_3_1_valb, # filter2_3_1c: filter2_3_1_valc, # filter2_3_2a: filter2_3_2_vala, filter2_3_2b: filter2_3_2_valb, # filter2_3_2c: filter2_3_2_valc, filter2_3_2d: filter2_3_2_vald, # filter2_3_2e: filter2_3_2_vale, filter2_3_3: filter2_3_3_val # , filter2_4_0: filter2_4_0_val, filter2_4_1a: filter2_4_1_vala, filter2_4_1b: filter2_4_1_valb, # filter2_4_1c: filter2_4_1_valc, # filter2_4_2a: filter2_4_2_vala, filter2_4_2b: filter2_4_2_valb, # filter2_4_2c: filter2_4_2_valc, filter2_4_2d: filter2_4_2_vald, # filter2_4_2e: filter2_4_2_vale, filter2_4_3: filter2_4_3_val # , filter2_5_0: filter2_5_0_val, filter2_5_1a: filter2_5_1_vala, filter2_5_1b: filter2_5_1_valb, # filter2_5_1c: filter2_5_1_valc, # filter2_5_2a: filter2_5_2_vala, filter2_5_2b: filter2_5_2_valb, # filter2_5_2c: filter2_5_2_valc, filter2_5_2d: filter2_5_2_vald, # filter2_5_2e: filter2_5_2_vale, filter2_5_3: filter2_5_3_val , filter3_1_0a: filter3_1_0_vala, filter3_1_0b: filter3_1_0_valb, filter3_1_1a: filter3_1_1_vala, filter3_1_1b: filter3_1_1_valb, filter3_1_1c: filter3_1_1_valc, filter3_1_1d: filter3_1_1_vald, filter3_2_0: filter3_2_0_val, filter3_2_1a: filter3_2_1_vala, filter3_2_1b: filter3_2_1_valb, filter3_2_1c: filter3_2_1_valc, filter3_2_2a: filter3_2_2_vala, filter3_2_2b: filter3_2_2_valb, filter3_2_2c: filter3_2_2_valc, filter3_2_2d: filter3_2_2_vald, filter3_2_3: filter3_2_3_val # , filter3_3_0: filter3_3_0_val, filter3_3_1a: filter3_3_1_vala, # filter3_3_1b: filter3_3_1_valb, # filter3_3_1c: filter3_3_1_valc, filter3_3_2a: filter3_3_2_vala, # filter3_3_2b: filter3_3_2_valb, # filter3_3_2c: filter3_3_2_valc, filter3_3_2d: filter3_3_2_vald, # filter3_3_3: filter3_3_3_val, , filtera1: filtera1val, filtersmul: filtersmulval, biases: biasesval }) train_x, train_y, test_x, test_y = prepare_data() num_epochs = 10 n_train_batches = train_x.shape[0] // batch_size n_test_batches = test_x.shape[0] // batch_size for i in range(num_epochs): print("epoch %d" % i) for minibatch_index in range(n_train_batches): minibatch_start = minibatch_index * batch_size minibatch_end = (minibatch_index + 1) * batch_size X_val[:] = train_x[minibatch_start:minibatch_end] y_val[:] = train_y[minibatch_start:minibatch_end] # print(y_val.shape) # y_val[:] = convert_to_one_hot(train_y[minibatch_start:minibatch_end]) t.run({X: X_val, y_: y_val}) # print(t.run_get_nodelist_once({X: X_val, y_: y_val}, [fc8])[fc8].asnumpy()) # print(map_to_numpy(t.get_Variable_node_to_val_map())) print("loss_val:", t.get_loss().asnumpy())
def test_inception_v4(): X = ad.Placeholder("X") Y_ = ad.Placeholder("y_") f1 = ad.Variable("f1") f2 = ad.Variable("f2") f3 = ad.Variable("f3") f4 = ad.Variable("f4") f5_1 = ad.Variable("f5_1") f5_2 = ad.Variable("f5_2") f6_1 = ad.Variable("f6_1") f6_2 = ad.Variable("f6_2") f6_3 = ad.Variable("f6_3") f6_4 = ad.Variable("f6_4") f7 = ad.Variable("f7") W = ad.Variable("W") b = ad.Variable("b") keep_prob = ad.Placeholder("keep_prob") X_val = np.random.normal(0, 0.5, (10, 3, 299, 299)) Y_val = np.random.normal(0, 0.5, (10, 1)) f1val = np.random.normal(0, 0.5, (32, 3, 3, 3)) f2val = np.random.normal(0, 0.5, (32, 32, 3, 3)) f3val = np.random.normal(0, 0.5, (64, 32, 3, 3)) f4val = np.random.normal(0, 0.5, (96, 64, 3, 3)) f5_1val = np.random.normal(0, 0.5, (64, 160, 1, 1)) f5_2val = np.random.normal(0, 0.5, (96, 64, 3, 3)) f6_1val = np.random.normal(0, 0.5, (64, 160, 1, 1)) f6_2val = np.random.normal(0, 0.5, (64, 64, 7, 1)) f6_3val = np.random.normal(0, 0.5, (64, 64, 1, 7)) f6_4val = np.random.normal(0, 0.5, (96, 64, 3, 3)) f7val = np.random.normal(0, 0.5, (192, 192, 3, 3)) # stem cov1 = ad.convolution_2d_forward_op(X, f1, "NCHW", "VALID", 2, 2) cov2 = ad.convolution_2d_forward_op(cov1, f2, "NCHW", "VALID", 1, 1) cov3 = ad.convolution_2d_forward_op(cov2, f3, "NCHW", "SAME", 1, 1) pool4 = ad.pooling_2d_forward_op(cov3, "NCHW", "max", 0, 0, 2, 2, 3, 3) cov4 = ad.convolution_2d_forward_op(cov3, f4, "NCHW", "VALID", 2, 2) concat1 = ad.concat_forward_op(pool4, cov4) cov5_1 = ad.convolution_2d_forward_op(concat1, f5_1, "NCHW", "SAME", 1, 1) cov5_2 = ad.convolution_2d_forward_op(cov5_1, f5_2, "NCHW", "VALID", 1, 1) cov6_1 = ad.convolution_2d_forward_op(concat1, f6_1, "NCHW", "SAME", 1, 1) cov6_2 = ad.convolution_2d_forward_op(cov6_1, f6_2, "NCHW", "SAME", 1, 1) cov6_3 = ad.convolution_2d_forward_op(cov6_2, f6_3, "NCHW", "SAME", 1, 1) cov6_4 = ad.convolution_2d_forward_op(cov6_3, f6_4, "NCHW", "VALID", 1, 1) concat2 = ad.concat_forward_op(cov5_2, cov6_4) cov7 = ad.convolution_2d_forward_op(concat2, f7, "NCHW", "VALID", 2, 2) pool7 = ad.pooling_2d_forward_op(concat2, "NCHW", "max", 0, 0, 2, 2, 3, 3) concat3 = ad.concat_forward_op(pool7, cov7) a1, dicta1 = block_inception_a(concat3, "a1") a2, dicta2 = block_inception_a(a1, "a2") a3, dicta3 = block_inception_a(a2, "a3") a4, dicta4 = block_inception_a(a3, "a4") ra, dictra = block_reduction_a(a4, 384, "ra") b1, dictb1 = block_inception_b(ra, 1024, "b1") # b2, dictb2 = block_inception_b(b1, 1024, "b2") # b3, dictb3 = block_inception_b(b2, 1024, "b3") # b4, dictb4 = block_inception_b(b3, 1024, "b4") # b5, dictb5 = block_inception_b(b4, 1024, "b5") # b6, dictb6 = block_inception_b(b5, 1024, "b6") # b7, dictb7 = block_inception_b(b6, 1024, "b7") # rb, dictrb = block_reduction_b(b1, 1024, "rb") c1, dictc1 = block_inception_c(rb, 1536, "c1") # c2, dictc2 = block_inception_c(c1, 1536, "c2") # c3, dictc3 = block_inception_c(c2, 1536, "c3") poollast = ad.pooling_2d_forward_op(c1, "NCHW", "mean", 0, 0, 1, 1, 8, 8) squeeze = ad.squeeze_op(poollast) drop_out = ad.fullydropout_forward_op(squeeze, "NCHW", 0.8) mul = ad.matmul_op(drop_out, W) add = ad.add_op(mul, b) loss = ad.softmaxcrossentropy_op(add, Y_) # # aph = 0.001 # t = train.Adam_minimize(loss, aph) ctx = ndarray.gpu(0) # aph = 0.001 # t = train.Adam_minimize(loss, aph) feed_dict = { X: X_val, f1: f1val, f2: f2val, f3: f3val, f4: f4val, f5_1: f5_1val, f5_2: f5_2val, f6_1: f6_1val, f6_2: f6_2val, f6_3: f6_3val, f6_4: f6_4val, f7: f7val } feed_dict.update(dicta1) feed_dict.update(dicta2) feed_dict.update(dicta3) feed_dict.update(dicta4) feed_dict.update(dictra) feed_dict.update(dictb1) # feed_dict.update(dictb2) # feed_dict.update(dictb3) # feed_dict.update(dictb4) # feed_dict.update(dictb5) # feed_dict.update(dictb6) # feed_dict.update(dictb7) feed_dict.update(dictrb) feed_dict.update(dictc1) # feed_dict.update(dictc2) # feed_dict.update(dictc3) executor = ad.Executor([drop_out], ctx=ctx) y_val = executor.run(feed_dict)
def block_inception_c(inputs, input_size, blockname=None): filter3_2_0 = ad.Variable(blockname + "filter3_2_0") filter3_2_1a = ad.Variable(blockname + "filter3_2_1a") filter3_2_1b = ad.Variable(blockname + "filter3_2_1b") filter3_2_1c = ad.Variable(blockname + "filter3_2_1c") filter3_2_2a = ad.Variable(blockname + "filter3_2_2a") filter3_2_2b = ad.Variable(blockname + "filter3_2_2b") filter3_2_2c = ad.Variable(blockname + "filter3_2_2c") filter3_2_2d = ad.Variable(blockname + "filter3_2_2d") filter3_2_2e = ad.Variable(blockname + "filter3_2_2e") filter3_2_3 = ad.Variable(blockname + "filter3_2_3a") rand = np.random.RandomState(seed=123) filter3_2_0_val = rand.normal(scale=0.1, size=(256, input_size, 1, 1)) filter3_2_1_vala = rand.normal(scale=0.1, size=(384, input_size, 1, 1)) filter3_2_1_valb = rand.normal(scale=0.1, size=(256, 384, 1, 3)) filter3_2_1_valc = rand.normal(scale=0.1, size=(256, 384, 3, 1)) filter3_2_2_vala = rand.normal(scale=0.1, size=(384, input_size, 1, 1)) filter3_2_2_valb = rand.normal(scale=0.1, size=(448, 384, 1, 3)) filter3_2_2_valc = rand.normal(scale=0.1, size=(512, 448, 3, 1)) filter3_2_2_vald = rand.normal(scale=0.1, size=(256, 512, 3, 1)) filter3_2_2_vale = rand.normal(scale=0.1, size=(256, 512, 1, 3)) filter3_2_3_val = rand.normal(scale=0.1, size=(256, input_size, 1, 1)) # branch_0 incep3_2_0 = ad.convolution_2d_forward_op(inputs, filter3_2_0, "NCHW", "SAME", 1, 1) # branch 1 incep3_2_1a = ad.convolution_2d_forward_op(inputs, filter3_2_1a, "NCHW", "SAME", 1, 1) incep3_2_1b = ad.convolution_2d_forward_op(incep3_2_1a, filter3_2_1b, "NCHW", "SAME", 1, 1) incep3_2_1c = ad.convolution_2d_forward_op(incep3_2_1a, filter3_2_1c, "NCHW", "SAME", 1, 1) incep3_2_1 = ad.concat_forward_op(incep3_2_1b, incep3_2_1c) # branch 2 incep3_2_2a = ad.convolution_2d_forward_op(inputs, filter3_2_2a, "NCHW", "SAME", 1, 1) incep3_2_2b = ad.convolution_2d_forward_op(incep3_2_2a, filter3_2_2b, "NCHW", "SAME", 1, 1) incep3_2_2c = ad.convolution_2d_forward_op(incep3_2_2b, filter3_2_2c, "NCHW", "SAME", 1, 1) incep3_2_2d = ad.convolution_2d_forward_op(incep3_2_2c, filter3_2_2d, "NCHW", "SAME", 1, 1) incep3_2_2e = ad.convolution_2d_forward_op(incep3_2_2c, filter3_2_2e, "NCHW", "SAME", 1, 1) incep3_2_2 = ad.concat_forward_op(incep3_2_2d, incep3_2_2e) # branch 3 incep3_2_3a = ad.pooling_2d_forward_op(inputs, "NCHW", "mean", 1, 1, 1, 1, 3, 3) incep3_2_3 = ad.convolution_2d_forward_op(incep3_2_3a, filter3_2_3, "NCHW", "SAME", 1, 1) concat3_2a = ad.concat_forward_op(incep3_2_0, incep3_2_1) concat3_2b = ad.concat_forward_op(concat3_2a, incep3_2_2) concat3_2 = ad.concat_forward_op(concat3_2b, incep3_2_3) dict = { filter3_2_0: filter3_2_0_val, filter3_2_1a: filter3_2_1_vala, filter3_2_1b: filter3_2_1_valb, filter3_2_1c: filter3_2_1_valc, filter3_2_2a: filter3_2_2_vala, filter3_2_2b: filter3_2_2_valb, filter3_2_2c: filter3_2_2_valc, filter3_2_2d: filter3_2_2_vald, filter3_2_2e: filter3_2_2_vale, filter3_2_3: filter3_2_3_val } return concat3_2, dict
def mnist_logreg(executor_ctx=None, num_epochs=10, print_loss_val_each_epoch=False): # 训练逻辑回归模型 print("Build logistic regression model...") W1 = ad.Variable(name="W1") b1 = ad.Variable(name="b1") X = ad.Variable(name="X") y_ = ad.Variable(name="y_") z1 = ad.matmul_op(X, W1) y = z1 + ad.broadcastto_op(b1, z1) loss = ad.softmaxcrossentropy_op(y, y_) grad_W1, grad_b1 = ad.gradients(loss, [W1, b1]) executor = ad.Executor([loss, grad_W1, grad_b1, y], ctx=executor_ctx) # Read input data datasets = load_mnist_data("mnist.pkl.gz") train_set_x, train_set_y = datasets[0] valid_set_x, valid_set_y = datasets[1] test_set_x, test_set_y = datasets[2] # Set up minibatch batch_size = 1000 n_train_batches = train_set_x.shape[0] // batch_size n_valid_batches = valid_set_x.shape[0] // batch_size print("Start training loop...") # Initialize parameters W1_val = np.zeros((784, 10)) b1_val = np.zeros((10)) X_val = np.empty(shape=(batch_size, 784), dtype=np.float32) y_val = np.empty(shape=(batch_size, 10), dtype=np.float32) valid_X_val = np.empty(shape=(batch_size, 784), dtype=np.float32) valid_y_val = np.empty(shape=(batch_size, 10), dtype=np.float32) if ndarray.is_gpu_ctx(executor_ctx): W1_val = ndarray.array(W1_val, ctx=executor_ctx) b1_val = ndarray.array(b1_val, ctx=executor_ctx) X_val = ndarray.array(X_val, ctx=executor_ctx) y_val = ndarray.array(y_val, ctx=executor_ctx) lr = 1e-3 for i in range(num_epochs): print("epoch %d" % i) for minibatch_index in range(n_train_batches): minibatch_start = minibatch_index * batch_size minibatch_end = (minibatch_index + 1) * batch_size X_val[:] = train_set_x[minibatch_start:minibatch_end] y_val[:] = convert_to_one_hot( train_set_y[minibatch_start:minibatch_end]) loss_val, grad_W1_val, grad_b1_val, _ = executor.run( feed_dict = {X: X_val, y_: y_val, W1: W1_val, b1: b1_val}) # SGD update if (executor_ctx is None): W1_val = W1_val - lr * grad_W1_val b1_val = b1_val - lr * grad_b1_val else: sgd_update_gpu(W1_val, grad_W1_val, lr) sgd_update_gpu(b1_val, grad_b1_val, lr) if print_loss_val_each_epoch: if isinstance(loss_val, ndarray.NDArray): print(loss_val.asnumpy()) else: print(loss_val) correct_predictions = [] for minibatch_index in range(n_valid_batches): minibatch_start = minibatch_index * batch_size minibatch_end = (minibatch_index + 1) * batch_size valid_X_val[:] = valid_set_x[minibatch_start:minibatch_end] valid_y_val[:] = convert_to_one_hot( valid_set_y[minibatch_start:minibatch_end]) _, _, _, valid_y_predicted = executor.run( feed_dict={ X: valid_X_val, y_: valid_y_val, W1: W1_val, b1: b1_val}, convert_to_numpy_ret_vals=True) correct_prediction = np.equal( np.argmax(valid_y_val, 1), np.argmax(valid_y_predicted, 1)).astype(np.float) correct_predictions.extend(correct_prediction) accuracy = np.mean(correct_predictions) # validation set accuracy=0.928200 print("validation set accuracy=%f" % accuracy)
def mnist_mlp(executor_ctx=None, num_epochs=10, print_loss_val_each_epoch=False): # 训练一个三层感知机模型 print("Build 3-layer MLP model...") W1 = ad.Variable(name="W1") W2 = ad.Variable(name="W2") W3 = ad.Variable(name="W3") b1 = ad.Variable(name="b1") b2 = ad.Variable(name="b2") b3 = ad.Variable(name="b3") X = ad.Variable(name="X") y_ = ad.Variable(name="y_") # 下面是三层网络的激活函数,两个relu和一个softmax # relu(X W1+b1) z1 = ad.matmul_op(X, W1) z2 = z1 + ad.broadcastto_op(b1, z1) z3 = ad.relu_op(z2) # relu(z3 W2+b2) z4 = ad.matmul_op(z3, W2) z5 = z4 + ad.broadcastto_op(b2, z4) z6 = ad.relu_op(z5) # softmax(z5 W2+b2) z7 = ad.matmul_op(z6, W3) y = z7 + ad.broadcastto_op(b3, z7) loss = ad.softmaxcrossentropy_op(y, y_) grad_W1, grad_W2, grad_W3, grad_b1, grad_b2, grad_b3 = ad.gradients( loss, [W1, W2, W3, b1, b2, b3]) # 此处向前为符号定义 # 只声明,不操作 executor = ad.Executor( [loss, grad_W1, grad_W2, grad_W3, grad_b1, grad_b2, grad_b3, y], ctx=executor_ctx) # Read input data datasets = load_mnist_data("mnist.pkl.gz") train_set_x, train_set_y = datasets[0] valid_set_x, valid_set_y = datasets[1] test_set_x, test_set_y = datasets[2] # Set up minibatch batch_size = 1000 n_train_batches = train_set_x.shape[0] // batch_size n_valid_batches = valid_set_x.shape[0] // batch_size print("Start training loop...") # Initialize parameters # 随机初始化网络中的w和b rand = np.random.RandomState(seed=123) W1_val = rand.normal(scale=0.1, size=(784, 256)) W2_val = rand.normal(scale=0.1, size=(256, 100)) W3_val = rand.normal(scale=0.1, size=(100, 10)) b1_val = rand.normal(scale=0.1, size=(256)) b2_val = rand.normal(scale=0.1, size=(100)) b3_val = rand.normal(scale=0.1, size=(10)) X_val = np.empty(shape=(batch_size, 784), dtype=np.float32) y_val = np.empty(shape=(batch_size, 10), dtype=np.float32) valid_X_val = np.empty(shape=(batch_size, 784), dtype=np.float32) valid_y_val = np.empty(shape=(batch_size, 10), dtype=np.float32) # todo 此处修改为cpu W1_val = ndarray.array(W1_val, ctx=executor_ctx_cpu) W2_val = ndarray.array(W2_val, ctx=executor_ctx_cpu) W3_val = ndarray.array(W3_val, ctx=executor_ctx_cpu) b1_val = ndarray.array(b1_val, ctx=executor_ctx_cpu) b2_val = ndarray.array(b2_val, ctx=executor_ctx_cpu) b3_val = ndarray.array(b3_val, ctx=executor_ctx_cpu) X_val = ndarray.array(X_val, ctx=executor_ctx_cpu) y_val = ndarray.array(y_val, ctx=executor_ctx_cpu) # 此处以上将数据分别转化为cpu和gpu两种格式 lr = 1.0e-3 for i in range(num_epochs): print("epoch %d" % i) for minibatch_index in range(n_train_batches): minibatch_start = minibatch_index * batch_size minibatch_end = (minibatch_index + 1) * batch_size X_val[:] = train_set_x[minibatch_start:minibatch_end] y_val[:] = convert_to_one_hot( train_set_y[minibatch_start:minibatch_end]) # 计算单步的梯度 loss_val, grad_W1_val, grad_W2_val, grad_W3_val, \ grad_b1_val, grad_b2_val, grad_b3_val, _ = executor.run( feed_dict={ X: X_val, y_: y_val, W1: W1_val, W2: W2_val, W3: W3_val, b1: b1_val, b2: b2_val, b3: b3_val}) # todo 更新sgd_update_gpu_on_cpu def sgd_update_cpu(w1, w2, w3): w1_gpu = ndarray.empty(w1.shape, executor_ctx) w1.copyto(w1_gpu) w2_gpu = ndarray.empty(w2.shape, executor_ctx) w2.copyto(w2_gpu) sgd_update_gpu(w1_gpu, w2_gpu, w3) w1_gpu.copyto(w1) w2_gpu.copyto(w2) sgd_update_cpu(W1_val, grad_W1_val, lr) sgd_update_cpu(W2_val, grad_W2_val, lr) sgd_update_cpu(W3_val, grad_W3_val, lr) sgd_update_cpu(b1_val, grad_b1_val, lr) sgd_update_cpu(b2_val, grad_b2_val, lr) sgd_update_cpu(b3_val, grad_b3_val, lr) # sgd_update_gpu(W1_val, grad_W1_val, lr) # sgd_update_gpu(W2_val, grad_W2_val, lr) # sgd_update_gpu(W3_val, grad_W3_val, lr) # sgd_update_gpu(b1_val, grad_b1_val, lr) # sgd_update_gpu(b2_val, grad_b2_val, lr) # sgd_update_gpu(b3_val, grad_b3_val, lr) if print_loss_val_each_epoch: if isinstance(loss_val, ndarray.NDArray): print(loss_val.asnumpy()) else: print(loss_val) correct_predictions = [] for minibatch_index in range(n_valid_batches): minibatch_start = minibatch_index * batch_size minibatch_end = (minibatch_index + 1) * batch_size valid_X_val[:] = valid_set_x[minibatch_start:minibatch_end] valid_y_val[:] = convert_to_one_hot( valid_set_y[minibatch_start:minibatch_end]) _, _, _, _, _, _, _, valid_y_predicted = executor.run( feed_dict={ X: valid_X_val, y_: valid_y_val, W1: W1_val, W2: W2_val, W3: W3_val, b1: b1_val, b2: b2_val, b3: b3_val}, convert_to_numpy_ret_vals=True) correct_prediction = np.equal( np.argmax(valid_y_val, 1), np.argmax(valid_y_predicted, 1)).astype(np.float) correct_predictions.extend(correct_prediction) accuracy = np.mean(correct_predictions) # validation set accuracy=0.970800 print("validation set accuracy=%f" % accuracy)
def vgg16(): n = 10 n_class = 10 inputs = ad.Variable("inputs") filters1_1 = ad.Variable("filters1_1") filters1_2 = ad.Variable("filters1_2") filters2_1 = ad.Variable("filters2_1") filters2_2 = ad.Variable("filters2_2") filters3_1 = ad.Variable("filters3_1") filters3_2 = ad.Variable("filters3_2") filters3_3 = ad.Variable("filters3_3") filters4_1 = ad.Variable("filters4_1") filters4_2 = ad.Variable("filters4_2") filters4_3 = ad.Variable("filters4_3") filters5_1 = ad.Variable("filters5_1") filters5_2 = ad.Variable("filters5_2") filters5_3 = ad.Variable("filters5_3") filters6 = ad.Variable("filters6") filters7 = ad.Variable("filters7") filters8 = ad.Variable("filters8") biases6 = ad.Variable("biases6") biases7 = ad.Variable("biases7") biases8 = ad.Variable("biases8") y_ = ad.Variable(name="y_") x_val = np.linspace(0, 0.001, 10*3*224*224).reshape((10, 3, 224, 224)) filters_val = [np.ones((64, 3, 3, 3))*0.001] filters_val.append(np.ones((64, 64, 3, 3))*0.001) filters_val.append(np.ones((128, 64, 3, 3))*0.001) filters_val.append(np.ones((128, 128, 3, 3))*0.001) filters_val.append(np.ones((256, 128, 3, 3))*0.001) filters_val.append(np.ones((256, 256, 3, 3))*0.001) filters_val.append(np.ones((256, 256, 3, 3))*0.001) filters_val.append(np.ones((512, 256, 3, 3))*0.001) filters_val.append(np.ones((512, 512, 3, 3))*0.001) filters_val.append(np.ones((512, 512, 3, 3))*0.001) filters_val.append(np.ones((512, 512, 3, 3))*0.001) filters_val.append(np.ones((512, 512, 3, 3))*0.001) filters_val.append(np.ones((512, 512, 3, 3))*0.001) filters_val.append(np.ones((512*7*7, 4096)) * 0.001) filters_val.append(np.ones((4096, 4096)) * 0.001) filters_val.append(np.ones((4096, n_class)) * 0.001) biases_val = [np.ones((1, 4096))* 0.001] biases_val.append(np.ones((1, 4096)) * 0.001) biases_val.append(np.ones((1, n_class)) * 0.001) y_val = np.zeros((10, n_class)) ctx = ndarray.gpu(0) for i in range(16): filters_val[i] = ndarray.array(filters_val[i], ctx) # conv 1 conv1_1 = ad.convolution_2d_forward_op(inputs, filters1_1, "NCHW", "SAME", 1, 1) bn1_1 = ad.bn_forward_op(conv1_1, "NCHW", "pre_activation") act1_1 = ad.activation_forward_op(bn1_1, "NCHW", "relu") conv1_2 = ad.convolution_2d_forward_op(act1_1, filters1_2, "NCHW", "SAME", 1, 1) bn1_2 = ad.bn_forward_op(conv1_2, "NCHW", "pre_activation") act1_2 = ad.activation_forward_op(bn1_2, "NCHW", "relu") pool1 = ad.pooling_2d_forward_op(act1_2, "NCHW", "max", 0, 0, 2, 2, 2, 2) # conv 2 conv2_1 = ad.convolution_2d_forward_op(pool1, filters2_1, "NCHW", "SAME", 1, 1) bn2_1 = ad.bn_forward_op(conv2_1, "NCHW", "pre_activation") act2_1 = ad.activation_forward_op(bn2_1, "NCHW", "relu") conv2_2 = ad.convolution_2d_forward_op(act2_1, filters2_2, "NCHW", "SAME", 1, 1) bn2_2 = ad.bn_forward_op(conv2_2, "NCHW", "pre_activation") act2_2 = ad.activation_forward_op(bn2_2, "NCHW", "relu") pool2 = ad.pooling_2d_forward_op(act2_2, "NCHW", "max", 0, 0, 2, 2, 2, 2) # conv 3 conv3_1 = ad.convolution_2d_forward_op(pool2, filters3_1, "NCHW", "SAME", 1, 1) bn3_1 = ad.bn_forward_op(conv3_1, "NCHW", "pre_activation") act3_1 = ad.activation_forward_op(bn3_1, "NCHW", "relu") conv3_2 = ad.convolution_2d_forward_op(act3_1, filters3_2, "NCHW", "SAME", 1, 1) bn3_2 = ad.bn_forward_op(conv3_2, "NCHW", "pre_activation") act3_2 = ad.activation_forward_op(bn3_2, "NCHW", "relu") conv3_3 = ad.convolution_2d_forward_op(act3_2, filters3_3, "NCHW", "SAME", 1, 1) bn3_3 = ad.bn_forward_op(conv3_3, "NCHW", "pre_activation") act3_3 = ad.activation_forward_op(bn3_3, "NCHW", "relu") pool3 = ad.pooling_2d_forward_op(act3_3, "NCHW", "max", 0, 0, 2, 2, 2, 2) # conv 4 conv4_1 = ad.convolution_2d_forward_op(pool3, filters4_1, "NCHW", "SAME", 1, 1) bn4_1 = ad.bn_forward_op(conv4_1, "NCHW", "pre_activation") act4_1 = ad.activation_forward_op(bn4_1, "NCHW", "relu") conv4_2 = ad.convolution_2d_forward_op(act4_1, filters4_2, "NCHW", "SAME", 1, 1) bn4_2 = ad.bn_forward_op(conv4_2, "NCHW", "pre_activation") act4_2 = ad.activation_forward_op(bn4_2, "NCHW", "relu") conv4_3 = ad.convolution_2d_forward_op(act4_2, filters4_3, "NCHW", "SAME", 1, 1) bn4_3 = ad.bn_forward_op(conv4_3, "NCHW", "pre_activation") act4_3 = ad.activation_forward_op(bn4_3, "NCHW", "relu") pool4 = ad.pooling_2d_forward_op(act4_3, "NCHW", "max", 0, 0, 2, 2, 2, 2) # conv 5 conv5_1 = ad.convolution_2d_forward_op(pool4, filters5_1, "NCHW", "SAME", 1, 1) bn5_1 = ad.bn_forward_op(conv5_1, "NCHW", "pre_activation") act5_1 = ad.activation_forward_op(bn5_1, "NCHW", "relu") conv5_2 = ad.convolution_2d_forward_op(act5_1, filters5_2, "NCHW", "SAME", 1, 1) bn5_2 = ad.bn_forward_op(conv5_2, "NCHW", "pre_activation") act5_2 = ad.activation_forward_op(bn5_2, "NCHW", "relu") conv5_3 = ad.convolution_2d_forward_op(act5_2, filters5_3, "NCHW", "SAME", 1, 1) bn5_3 = ad.bn_forward_op(conv5_3, "NCHW", "pre_activation") act5_3 = ad.activation_forward_op(bn5_3, "NCHW", "relu") pool5 = ad.pooling_2d_forward_op(act5_3, "NCHW", "max", 0, 0, 2, 2, 2, 2) # fc6 pool5_flat = ad.flatten_op(pool5) mul6 = ad.matmul_op(pool5_flat, filters6) add6 = ad.add_op(mul6, biases6) bn6 = ad.fullybn_forward_op(add6, "NCHW") fc6 = ad.fullyactivation_forward_op(bn6, "NCHW", "relu") drop6 = ad.fullydropout_forward_op(fc6, "NCHW", 0.5) # fc7 mul7 = ad.matmul_op(drop6, filters7) add7 = ad.add_op(mul7, biases7) bn7 = ad.fullybn_forward_op(add7, "NCHW") fc7 = ad.fullyactivation_forward_op(bn7, "NCHW", "relu") drop7 = ad.fullydropout_forward_op(fc7, "NCHW", 0.5) #fc8 mul8 = ad.matmul_op(drop7, filters8) add8 = ad.add_op(mul8, biases8) fc8 = ad.fullyactivation_forward_op(add8, "NCHW", "softmax") loss = ad.l2loss_op(fc8, y_) grad = ad.gradients(loss, [filters1_1, filters1_2, filters2_1, filters2_2, filters3_1, filters3_2, filters3_3 , filters4_1, filters4_2, filters4_3, filters5_1, filters5_2, filters5_3 , filters6, filters7]) executor = ad.Executor([grad[0], grad[1], grad[2], grad[3], grad[4], grad[5], grad[6], grad[7], grad[8], grad[9] , grad[10], grad[11], grad[12], grad[13], grad[14], loss, y_], ctx=ctx) aph = 1.0e-6 for i in range(20): select = random.randint(0, n-1) tmp_x_val = x_val[select] tmp_x_val = np.expand_dims(tmp_x_val, 0) tmp_y_val = y_val[select] tmp_y_val = np.expand_dims(tmp_y_val, 0) grad_val = executor.run( feed_dict={inputs: tmp_x_val, y_: tmp_y_val , filters1_1: filters_val[0], filters1_2: filters_val[1], filters2_1: filters_val[2], filters2_2: filters_val[3] , filters3_1: filters_val[4], filters3_2: filters_val[5], filters3_3: filters_val[6] , filters4_1: filters_val[7], filters4_2: filters_val[8], filters4_3: filters_val[9] , filters5_1: filters_val[10], filters5_2: filters_val[11], filters5_3: filters_val[12] , filters6: filters_val[13], filters7: filters_val[14], filters8: filters_val[15] , biases6: biases_val[0], biases7: biases_val[1], biases8: biases_val[2]}) for i in range(14): sgd_update_gpu(filters_val[i], grad_val[i], aph) print(filters_val[0].asnumpy()) return filters_val
def ResNet152(inputs, n_class): X = ad.Placeholder("X") y_ = ad.Placeholder("y_") W1 = ad.Variable("W1") W6 = ad.Variable("W6") b6 = ad.Variable("b6") W7 = ad.Variable("W7") b7 = ad.Variable("b7") keep_prob = ad.Placeholder("keep_prob") #conv1 conv1 = ad.convolution_2d_forward_op(X, W1, "NCHW", "VALID", 2, 2) bn1 = ad.bn_forward_op(conv1, "NCHW", "pre_activation") act1 = ad.activation_forward_op(bn1, "NCHW", "relu") pool1 = ad.pooling_2d_forward_op(act1, "NCHW", "max", 0, 0, 2, 2, 3, 3) #conv2_x conv2, dict2 = convolutional_block(inputs=pool1, kernel_size=3, in_filter=64, out_filters=[64, 64, 256], block_name="2a", stride=1) iden2_1, dict2_1 = identity_block(inputs=conv2, kernel_size=3, in_filter=256, out_filters=[64, 64, 256], block_name="2b", stride=1) iden2_2, dict2_2 = identity_block(iden2_1, 3, 256, [64, 64, 256], "2c", 1) #conv3_x conv3, dict3 = convolutional_block(iden2_2, 3, 256, [128, 128, 512], "3a", 1) iden3_1, dict3_1 = identity_block(conv3, 3, 512, [128, 128, 512], "3b", 1) iden3_2, dict3_2 = identity_block(iden3_1, 3, 512, [128, 128, 512], "3c", 1) iden3_3, dict3_3 = identity_block(iden3_2, 3, 512, [128, 128, 512], "3d", 1) iden3_4, dict3_4 = identity_block(iden3_3, 3, 512, [128, 128, 512], "3e", 1) iden3_5, dict3_5 = identity_block(iden3_4, 3, 512, [128, 128, 512], "3f", 1) iden3_6, dict3_6 = identity_block(iden3_5, 3, 512, [128, 128, 512], "3g", 1) iden3_7, dict3_7 = identity_block(iden3_6, 3, 512, [128, 128, 512], "3h", 1) #conv4_x conv4, dict4 = convolutional_block(iden3_7, 3, 512, [256, 256, 1024], "4a", 1) iden4_1, dict4_1 = identity_block(conv4, 3, 1024, [256, 256, 1024], "4b", 1) iden4_2, dict4_2 = identity_block(iden4_1, 3, 1024, [256, 256, 1024], "4c", 1) iden4_3, dict4_3 = identity_block(iden4_2, 3, 1024, [256, 256, 1024], "4d", 1) iden4_4, dict4_4 = identity_block(iden4_3, 3, 1024, [256, 256, 1024], "4e", 1) iden4_5, dict4_5 = identity_block(iden4_4, 3, 1024, [256, 256, 1024], "4f", 1) iden4_6, dict4_6 = identity_block(iden4_5, 3, 1024, [256, 256, 1024], "4f", 1) iden4_7, dict4_7 = identity_block(iden4_6, 3, 1024, [256, 256, 1024], "4f", 1) iden4_8, dict4_8 = identity_block(iden4_7, 3, 1024, [256, 256, 1024], "4f", 1) iden4_9, dict4_9 = identity_block(iden4_8, 3, 1024, [256, 256, 1024], "4f", 1) iden4_10, dict4_10 = identity_block(iden4_9, 3, 1024, [256, 256, 1024], "4f", 1) iden4_11, dict4_11 = identity_block(iden4_10, 3, 1024, [256, 256, 1024], "4f", 1) iden4_12, dict4_12 = identity_block(iden4_11, 3, 1024, [256, 256, 1024], "4f", 1) iden4_13, dict4_13 = identity_block(iden4_12, 3, 1024, [256, 256, 1024], "4f", 1) iden4_14, dict4_14 = identity_block(iden4_13, 3, 1024, [256, 256, 1024], "4f", 1) iden4_15, dict4_15 = identity_block(iden4_14, 3, 1024, [256, 256, 1024], "4f", 1) iden4_16, dict4_16 = identity_block(iden4_15, 3, 1024, [256, 256, 1024], "4f", 1) iden4_17, dict4_17 = identity_block(iden4_16, 3, 1024, [256, 256, 1024], "4f", 1) iden4_18, dict4_18 = identity_block(iden4_17, 3, 1024, [256, 256, 1024], "4f", 1) iden4_19, dict4_19 = identity_block(iden4_18, 3, 1024, [256, 256, 1024], "4f", 1) iden4_20, dict4_20 = identity_block(iden4_19, 3, 1024, [256, 256, 1024], "4f", 1) iden4_21, dict4_21 = identity_block(iden4_20, 3, 1024, [256, 256, 1024], "4f", 1) iden4_22, dict4_22 = identity_block(iden4_21, 3, 1024, [256, 256, 1024], "4f", 1) iden4_23, dict4_23 = identity_block(iden4_22, 3, 1024, [256, 256, 1024], "4f", 1) iden4_24, dict4_24 = identity_block(iden4_23, 3, 1024, [256, 256, 1024], "4f", 1) iden4_25, dict4_25 = identity_block(iden4_24, 3, 1024, [256, 256, 1024], "4f", 1) iden4_26, dict4_26 = identity_block(iden4_25, 3, 1024, [256, 256, 1024], "4f", 1) iden4_27, dict4_27 = identity_block(iden4_26, 3, 1024, [256, 256, 1024], "4f", 1) iden4_28, dict4_28 = identity_block(iden4_27, 3, 1024, [256, 256, 1024], "4f", 1) iden4_29, dict4_29 = identity_block(iden4_28, 3, 1024, [256, 256, 1024], "4f", 1) iden4_30, dict4_30 = identity_block(iden4_29, 3, 1024, [256, 256, 1024], "4f", 1) iden4_31, dict4_31 = identity_block(iden4_30, 3, 1024, [256, 256, 1024], "4f", 1) iden4_32, dict4_32 = identity_block(iden4_31, 3, 1024, [256, 256, 1024], "4f", 1) iden4_33, dict4_33 = identity_block(iden4_32, 3, 1024, [256, 256, 1024], "4f", 1) iden4_34, dict4_34 = identity_block(iden4_33, 3, 1024, [256, 256, 1024], "4f", 1) iden4_35, dict4_35 = identity_block(iden4_34, 3, 1024, [256, 256, 1024], "4f", 1) #conv5_x conv5, dict5 = convolutional_block(iden4_35, 3, 1024, [512, 512, 2048], "5a", 1) iden5_1, dict5_1 = identity_block(conv5, 3, 2048, [512, 512, 2048], "5b", 1) iden5_2, dict5_2 = identity_block(iden5_1, 3, 2048, [512, 512, 2048], "5c", 1) pool5 = ad.pooling_2d_forward_op(iden5_2, "NCHW", "mean", 0, 0, 1, 1, 2, 2) pool5_flat = ad.flatten_op(pool5) mul6 = ad.matmul_op(pool5_flat, W6) add6 = ad.add_op(mul6, b6) act6 = ad.fullyactivation_forward_op(add6, "NCHW", "relu") drop_out = ad.fullydropout_forward_op(act6, "NCHW", keep_prob) mul7 = ad.matmul_op(drop_out, W7) add7 = ad.add_op(mul7, b7) act7 = ad.fullyactivation_forward_op(add7, "NCHW", "softmax") loss = ad.softmaxcrossentropy_op(act7, y_) X_val = np.random.normal(0, 0.5, (10, 3, 230, 230)) W1_val = np.random.normal(0, 0.5, (64, 3, 7, 7)) W6_val = np.random.normal(0, 0.5, (7 * 7 * 2048, 50)) b6_val = np.random.normal(0, 0.5, (10, 50)) W7_val = np.random.normal(0, 0.5, (50, 6)) b7_val = np.random.normal(0, 0.5, (10, 6)) y_val = np.random.normal(0, 0.5, (10, 6)) aph = 0.001 t = train.Adam_minimize(loss, aph) feed_dict = {W1: W1_val, W6: W6_val, W7: W7_val, b6: b6_val, b7: b7_val} feed_dict.update(dict2) feed_dict.update(dict2_1) feed_dict.update(dict2_2) feed_dict.update(dict3) feed_dict.update(dict3_1) feed_dict.update(dict3_2) feed_dict.update(dict3_3) feed_dict.update(dict3_4) feed_dict.update(dict3_5) feed_dict.update(dict3_6) feed_dict.update(dict3_7) feed_dict.update(dict4) feed_dict.update(dict4_1) feed_dict.update(dict4_2) feed_dict.update(dict4_3) feed_dict.update(dict4_4) feed_dict.update(dict4_5) feed_dict.update(dict4_6) feed_dict.update(dict4_7) feed_dict.update(dict4_8) feed_dict.update(dict4_9) feed_dict.update(dict4_10) feed_dict.update(dict4_11) feed_dict.update(dict4_12) feed_dict.update(dict4_13) feed_dict.update(dict4_14) feed_dict.update(dict4_15) feed_dict.update(dict4_16) feed_dict.update(dict4_17) feed_dict.update(dict4_18) feed_dict.update(dict4_19) feed_dict.update(dict4_20) feed_dict.update(dict4_21) feed_dict.update(dict4_22) feed_dict.update(dict4_23) feed_dict.update(dict4_24) feed_dict.update(dict4_25) feed_dict.update(dict4_26) feed_dict.update(dict4_27) feed_dict.update(dict4_28) feed_dict.update(dict4_29) feed_dict.update(dict4_30) feed_dict.update(dict4_31) feed_dict.update(dict4_32) feed_dict.update(dict4_33) feed_dict.update(dict4_34) feed_dict.update(dict4_35) feed_dict.update(dict5) feed_dict.update(dict5_1) feed_dict.update(dict5_2) # t.init_Variable(feed_dict) # t.run({X: X_val, y_: y_val}) # print(t.get_Variable_node_to_val_map()[W1_val].asnumpy()) list = [] for key in feed_dict.keys(): list.append(key) executor = ad.Executor(list, ctx=ndarray.gpu(0)) executor.run(feed_dict)
def ResNet50(inputs, n_class): X = ad.Placeholder("X") y_ = ad.Placeholder("y_") W1 = ad.Variable("W1") W6 = ad.Variable("W6") b6 = ad.Variable("b6") W7 = ad.Variable("W7") b7 = ad.Variable("b7") keep_prob = ad.Placeholder("keep_prob") #conv1 conv1 = ad.convolution_2d_forward_op(X, W1, "NCHW", "VALID", 2, 2) bn1 = ad.bn_forward_op(conv1, "NCHW", "pre_activation") act1 = ad.activation_forward_op(bn1, "NCHW", "relu") pool1 = ad.pooling_2d_forward_op(act1, "NCHW", "max", 0, 0, 2, 2, 3, 3) #conv2_x conv2, dict2 = convolutional_block(inputs=pool1, kernel_size=3, in_filter=64, out_filters=[64, 64, 256], block_name="2a", stride=1) iden2_1, dict2_1 = identity_block(inputs=conv2, kernel_size=3, in_filter=256, out_filters=[64, 64, 256], block_name="2b", stride=1) iden2_2, dict2_2 = identity_block(iden2_1, 3, 256, [64, 64, 256], "2c", 1) #conv3_x conv3, dict3 = convolutional_block(iden2_2, 3, 256, [128, 128, 512], "3a", 1) iden3_1, dict3_1 = identity_block(conv3, 3, 512, [128, 128, 512], "3b", 1) iden3_2, dict3_2 = identity_block(iden3_1, 3, 512, [128, 128, 512], "3c", 1) iden3_3, dict3_3 = identity_block(iden3_2, 3, 512, [128, 128, 512], "3d", 1) #conv4_x conv4, dict4 = convolutional_block(iden3_3, 3, 512, [256, 256, 1024], "4a", 1) iden4_1, dict4_1 = identity_block(conv4, 3, 1024, [256, 256, 1024], "4b", 1) iden4_2, dict4_2 = identity_block(iden4_1, 3, 1024, [256, 256, 1024], "4c", 1) iden4_3, dict4_3 = identity_block(iden4_2, 3, 1024, [256, 256, 1024], "4d", 1) iden4_4, dict4_4 = identity_block(iden4_3, 3, 1024, [256, 256, 1024], "4e", 1) iden4_5, dict4_5 = identity_block(iden4_4, 3, 1024, [256, 256, 1024], "4f", 1) #conv5_x conv5, dict5 = convolutional_block(iden4_5, 3, 1024, [512, 512, 2048], "5a", 1) iden5_1, dict5_1 = identity_block(conv5, 3, 2048, [512, 512, 2048], "5b", 1) iden5_2, dict5_2 = identity_block(iden5_1, 3, 2048, [512, 512, 2048], "5c", 1) pool5 = ad.pooling_2d_forward_op(iden5_2, "NCHW", "mean", 0, 0, 1, 1, 2, 2) pool5_flat = ad.flatten_op(pool5) mul6 = ad.matmul_op(pool5_flat, W6) add6 = ad.add_op(mul6, b6) act6 = ad.fullyactivation_forward_op(add6, "NCHW", "relu") drop_out = ad.fullydropout_forward_op(act6, "NCHW", keep_prob) mul7 = ad.matmul_op(drop_out, W7) add7 = ad.add_op(mul7, b7) act7 = ad.fullyactivation_forward_op(add7, "NCHW", "softmax") loss = ad.softmaxcrossentropy_op(act7, y_) X_val = np.random.normal(0, 0.5, (10, 3, 230, 230)) W1_val = np.random.normal(0, 0.5, (64, 3, 7, 7)) W6_val = np.random.normal(0, 0.5, (7 * 7 * 2048, 50)) b6_val = np.random.normal(0, 0.5, (10, 50)) W7_val = np.random.normal(0, 0.5, (50, 6)) b7_val = np.random.normal(0, 0.5, (10, 6)) y_val = np.random.normal(0, 0.5, (10, 6)) feed_dict = {W1: W1_val, W6: W6_val, W7: W7_val, b6: b6_val, b7: b7_val} feed_dict.update(dict2) feed_dict.update(dict2_1) feed_dict.update(dict2_2) feed_dict.update(dict3) feed_dict.update(dict3_1) feed_dict.update(dict3_2) feed_dict.update(dict3_3) feed_dict.update(dict4) feed_dict.update(dict4_1) feed_dict.update(dict4_2) feed_dict.update(dict4_3) feed_dict.update(dict4_4) feed_dict.update(dict4_5) feed_dict.update(dict5) feed_dict.update(dict5_1) feed_dict.update(dict5_2) time.sleep(5) list = [] for key in feed_dict.keys(): list.append(key) executor = ad.Executor(list, ctx=ndarray.gpu(0)) executor.run(feed_dict) time.sleep(5)
def vgg16(num_epochs=10): n_class = 10 X = ad.Placeholder("inputs") y_ = ad.Placeholder("y_") filters1_1 = ad.Variable("filters1_1") filters1_2 = ad.Variable("filters1_2") filters2_1 = ad.Variable("filters2_1") filters2_2 = ad.Variable("filters2_2") filters3_1 = ad.Variable("filters3_1") filters3_2 = ad.Variable("filters3_2") filters3_3 = ad.Variable("filters3_3") filters4_1 = ad.Variable("filters4_1") filters4_2 = ad.Variable("filters4_2") filters4_3 = ad.Variable("filters4_3") filters5_1 = ad.Variable("filters5_1") filters5_2 = ad.Variable("filters5_2") filters5_3 = ad.Variable("filters5_3") filters6 = ad.Variable("filters6") filters7 = ad.Variable("filters7") filters8 = ad.Variable("filters8") b1_1 = ad.Variable("b1_1") b1_2 = ad.Variable("b1_2") b2_1 = ad.Variable("b2_1") b2_2 = ad.Variable("b2_2") b3_1 = ad.Variable("b3_1") b3_2 = ad.Variable("b3_2") b3_3 = ad.Variable("b3_3") b4_1 = ad.Variable("b4_1") b4_2 = ad.Variable("b4_2") b4_3 = ad.Variable("b4_3") b5_1 = ad.Variable("b5_1") b5_2 = ad.Variable("b5_2") b5_3 = ad.Variable("b5_3") b6 = ad.Variable("b6") b7 = ad.Variable("b7") b8 = ad.Variable("b8") # conv 1 conv1_1 = ad.conv2withbias(X, filters1_1, b1_1, "NCHW", "SAME", 1, 1) bn1_1 = ad.bn_forward_op(conv1_1, "NCHW", "pre_activation") act1_1 = ad.activation_forward_op(bn1_1, "NCHW", "relu") conv1_2 = ad.conv2withbias(act1_1, filters1_2, b1_2, "NCHW", "SAME", 1, 1) bn1_2 = ad.bn_forward_op(conv1_2, "NCHW", "pre_activation") act1_2 = ad.activation_forward_op(bn1_2, "NCHW", "relu") pool1 = ad.pooling_2d_forward_op(act1_2, "NCHW", "max", 0, 0, 2, 2, 2, 2) # conv 2 conv2_1 = ad.conv2withbias(pool1, filters2_1, b2_1, "NCHW", "SAME", 1, 1) bn2_1 = ad.bn_forward_op(conv2_1, "NCHW", "pre_activation") act2_1 = ad.activation_forward_op(bn2_1, "NCHW", "relu") conv2_2 = ad.conv2withbias(act2_1, filters2_2, b2_2, "NCHW", "SAME", 1, 1) bn2_2 = ad.bn_forward_op(conv2_2, "NCHW", "pre_activation") act2_2 = ad.activation_forward_op(bn2_2, "NCHW", "relu") pool2 = ad.pooling_2d_forward_op(act2_2, "NCHW", "max", 0, 0, 2, 2, 2, 2) # conv 3 conv3_1 = ad.conv2withbias(pool2, filters3_1, b3_1, "NCHW", "SAME", 1, 1) bn3_1 = ad.bn_forward_op(conv3_1, "NCHW", "pre_activation") act3_1 = ad.activation_forward_op(bn3_1, "NCHW", "relu") conv3_2 = ad.conv2withbias(act3_1, filters3_2, b3_2, "NCHW", "SAME", 1, 1) bn3_2 = ad.bn_forward_op(conv3_2, "NCHW", "pre_activation") act3_2 = ad.activation_forward_op(bn3_2, "NCHW", "relu") conv3_3 = ad.conv2withbias(act3_2, filters3_3, b3_3, "NCHW", "SAME", 1, 1) bn3_3 = ad.bn_forward_op(conv3_3, "NCHW", "pre_activation") act3_3 = ad.activation_forward_op(bn3_3, "NCHW", "relu") pool3 = ad.pooling_2d_forward_op(act3_3, "NCHW", "max", 0, 0, 2, 2, 2, 2) # conv 4 conv4_1 = ad.conv2withbias(pool3, filters4_1, b4_1, "NCHW", "SAME", 1, 1) bn4_1 = ad.bn_forward_op(conv4_1, "NCHW", "pre_activation") act4_1 = ad.activation_forward_op(bn4_1, "NCHW", "relu") conv4_2 = ad.conv2withbias(act4_1, filters4_2, b4_2, "NCHW", "SAME", 1, 1) bn4_2 = ad.bn_forward_op(conv4_2, "NCHW", "pre_activation") act4_2 = ad.activation_forward_op(bn4_2, "NCHW", "relu") conv4_3 = ad.conv2withbias(act4_2, filters4_3, b4_3, "NCHW", "SAME", 1, 1) bn4_3 = ad.bn_forward_op(conv4_3, "NCHW", "pre_activation") act4_3 = ad.activation_forward_op(bn4_3, "NCHW", "relu") pool4 = ad.pooling_2d_forward_op(act4_3, "NCHW", "max", 0, 0, 2, 2, 2, 2) # conv 5 conv5_1 = ad.conv2withbias(pool4, filters5_1, b5_1, "NCHW", "SAME", 1, 1) bn5_1 = ad.bn_forward_op(conv5_1, "NCHW", "pre_activation") act5_1 = ad.activation_forward_op(bn5_1, "NCHW", "relu") conv5_2 = ad.conv2withbias(act5_1, filters5_2, b5_2, "NCHW", "SAME", 1, 1) bn5_2 = ad.bn_forward_op(conv5_2, "NCHW", "pre_activation") act5_2 = ad.activation_forward_op(bn5_2, "NCHW", "relu") conv5_3 = ad.conv2withbias(act5_2, filters5_3, b5_3, "NCHW", "SAME", 1, 1) bn5_3 = ad.bn_forward_op(conv5_3, "NCHW", "pre_activation") act5_3 = ad.activation_forward_op(bn5_3, "NCHW", "relu") pool5 = ad.pooling_2d_forward_op(act5_3, "NCHW", "max", 0, 0, 2, 2, 2, 2) # fc6 pool5_flat = ad.flatten_op(pool5) fc6 = ad.dense(pool5_flat, filters6, b6) bn6 = ad.fullybn_forward_op(fc6, "NCHW") act6 = ad.fullyactivation_forward_op(bn6, "NCHW", "relu") drop6 = ad.fullydropout_forward_op(act6, "NCHW", 0.5) # fc7 fc7 = ad.dense(drop6, filters7, b7) bn7 = ad.fullybn_forward_op(fc7, "NCHW") act7 = ad.fullyactivation_forward_op(bn7, "NCHW", "relu") drop7 = ad.fullydropout_forward_op(act7, "NCHW", 0.5) #fc8 fc8 = ad.dense(drop7, filters8, b8) bn8 = ad.fullybn_forward_op(fc8, "NCHW") act8 = ad.fullyactivation_forward_op(bn8, "NCHW", "softmax") loss = ad.softmaxcrossentropy_op(bn8, y_) # grad = ad.gradients(loss, [filters1_1, filters1_2, filters2_1, filters2_2, filters3_1, filters3_2, filters3_3 # , filters4_1, filters4_2, filters4_3, filters5_1, filters5_2, filters5_3 # , filters6, filters7]) # executor = ad.Executor([grad[0], grad[1], grad[2], grad[3], grad[4], grad[5], grad[6], grad[7], grad[8], grad[9] # , grad[10], grad[11], grad[12], grad[13], grad[14], loss, y_], ctx=ctx) train_x, train_y, test_x, test_y = prepare_data() n_train_batches = train_x.shape[0] // batch_size n_test_batches = test_x.shape[0] // batch_size X_val = np.empty(shape=(batch_size, 3, 32, 32), dtype=np.float32) y_val = np.empty(shape=(batch_size, n_class), dtype=np.float32) filters_val = [np.random.normal(0.0, 0.1, (64, 3, 3, 3))] filters_val.append(np.random.normal(0.0, 0.1, (64, 64, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (128, 64, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (128, 128, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (256, 128, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (256, 256, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (256, 256, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (512, 256, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (512, 512, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (512, 512, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (512, 512, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (512, 512, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (512, 512, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (512 * 1 * 1, 4096))) filters_val.append(np.random.normal(0.0, 0.1, (4096, 4096)) * 0.001) filters_val.append(np.random.normal(0.0, 0.1, (4096, n_class)) * 0.001) b_val = [np.ones(64) * 0.1] b_val.append(np.ones(64) * 0.1) b_val.append(np.ones(128) * 0.1) b_val.append(np.ones(128) * 0.1) b_val.append(np.ones(256) * 0.1) b_val.append(np.ones(256) * 0.1) b_val.append(np.ones(256) * 0.1) b_val.append(np.ones(512) * 0.1) b_val.append(np.ones(512) * 0.1) b_val.append(np.ones(512) * 0.1) b_val.append(np.ones(512) * 0.1) b_val.append(np.ones(512) * 0.1) b_val.append(np.ones(512) * 0.1) b_val.append(np.ones(4096) * 0.1) b_val.append(np.ones(4096) * 0.1) b_val.append(np.ones(n_class) * 0.1) # ctx = ndarray.gpu(0) # for i in range(16): # filters_val[i] = ndarray.array(filters_val[i], ctx) # for i in range(16): # b_val[i] = ndarray.array(b_val[i], ctx) aph = 0.0001 t = train.Adam_minimize(loss, aph) t.init_Variable({ filters1_1: filters_val[0], filters1_2: filters_val[1], filters2_1: filters_val[2], filters2_2: filters_val[3], filters3_1: filters_val[4], filters3_2: filters_val[5], filters3_3: filters_val[6], filters4_1: filters_val[7], filters4_2: filters_val[8], filters4_3: filters_val[9], filters5_1: filters_val[10], filters5_2: filters_val[11], filters5_3: filters_val[12], filters6: filters_val[13], filters7: filters_val[14], filters8: filters_val[15], b1_1: b_val[0], b1_2: b_val[1], b2_1: b_val[2], b2_2: b_val[3], b3_1: b_val[4], b3_2: b_val[5], b3_3: b_val[6], b4_1: b_val[7], b4_2: b_val[8], b4_3: b_val[9], b5_1: b_val[10], b5_2: b_val[11], b5_3: b_val[12], b6: b_val[13], b7: b_val[14], b8: b_val[15] }) for i in range(num_epochs): print("epoch %d" % i) X_val = np.empty(shape=(0, 3, 32, 32), dtype=np.float32) y_val = np.empty(shape=(0, n_class), dtype=np.float32) select = random.randint(0, n_train_batches) # X_val = train_x[select] # y_val = train_y[select] for j in range(batch_size): select = random.randint(0, n_train_batches) temp_train_x = train_x[select] temp_train_x = temp_train_x[np.newaxis, :] temp_train_y = train_y[select] temp_train_y = temp_train_y[np.newaxis, :] X_val = np.append(X_val, temp_train_x, axis=0) y_val = np.append(y_val, temp_train_y, axis=0) # print("train", X_val.shape) # print("train", y_val.shape) t.run({X: X_val, y_: y_val}) if (i + 1) % 30 == 0: X_test_val = np.empty(shape=(100, 3, 32, 32), dtype=np.float32) y_test_val = np.empty(shape=(100, n_class), dtype=np.float32) correct_predictions = [] for minibatch_index in range(n_test_batches): minibatch_start = minibatch_index * batch_size minibatch_end = (minibatch_index + 1) * batch_size X_test_val[:] = test_x[minibatch_start:minibatch_end] y_test_val[:] = test_y[minibatch_start:minibatch_end] # print(X_test_val.shape) # print(y_test_val.shape) feed_dict = {X: X_test_val, y_: y_test_val} dict_Variable = t.get_Variable_node_to_val_map() feed_dict.update(dict_Variable) y_predicted = t.run_get_nodelist_once(feed_dict, [act8])[act8].asnumpy() correct_prediction = np.equal(np.argmax(y_test_val, 1), np.argmax(y_predicted, 1)).astype(np.float) correct_predictions.extend(correct_prediction) accuracy = np.mean(correct_predictions) print("validation set accuracy=%f" % accuracy) return filters_val