def test_matrix_multiply():
ctx = ndarray.gpu(0)
x = np.random.uniform(0, 10, size=(500, 700)).astype(np.float32)
y = np.random.uniform(0, 10, size=(700, 1000)).astype(np.float32)
arr_x = ndarray.array(x, ctx=ctx)
arr_y = ndarray.array(y, ctx=ctx)
arr_z = ndarray.empty((500, 1000), ctx=ctx)
gpu_op.matrix_multiply(arr_x, False, arr_y, False, arr_z)
z = arr_z.asnumpy()
np.testing.assert_allclose(np.dot(x, y), z, rtol=1e-5)
x = np.random.uniform(0, 10, size=(1000, 500)).astype(np.float32)
y = np.random.uniform(0, 10, size=(2000, 500)).astype(np.float32)
arr_x = ndarray.array(x, ctx=ctx)
arr_y = ndarray.array(y, ctx=ctx)
arr_z = ndarray.empty((1000, 2000), ctx=ctx)
gpu_op.matrix_multiply(arr_x, False, arr_y, True, arr_z)
z = arr_z.asnumpy()
np.testing.assert_allclose(np.dot(x, np.transpose(y)), z, rtol=1e-5)
x = np.random.uniform(0, 10, size=(500, 1000)).astype(np.float32)
y = np.random.uniform(0, 10, size=(2000, 500)).astype(np.float32)
arr_x = ndarray.array(x, ctx=ctx)
arr_y = ndarray.array(y, ctx=ctx)
arr_z = ndarray.empty((1000, 2000), ctx=ctx)
gpu_op.matrix_multiply(arr_x, True, arr_y, True, arr_z)
z = arr_z.asnumpy()
np.testing.assert_allclose(np.dot(np.transpose(x), np.transpose(y)), z,
rtol=1e-5)
def test_full_forward_op():
inputs = ad.Variable("inputs")
filters = ad.Variable("filters")
y_ = ad.Variable(name="y_")
#ini
ctx = ndarray.gpu(0)
x_val = np.linspace(0, 100, 100).reshape((5, 1, 20))
filters_val = np.ones((1, 1, 20)) * 0.001
y_val = np.zeros((5, 1))
x_val = ndarray.array(x_val, ctx)
filters_val = ndarray.array(filters_val, ctx)
y_val = ndarray.array(y_val, ctx)
outputs = ad.convolution_1d_forward_op(inputs, filters, "NCHW", "VALID", 1)
outputs_pool = ad.pooling_1d_forward_op(outputs, "NCHW", "max", 0, 1, 1)
outputs_relu = ad.activation_forward_op(outputs_pool, "NCHW", "relu")
outputs_f = ad.flatten_op(outputs_relu)
output = ad.fullyactivation_forward_op(outputs_f, "NCHW", "relu")
loss = ad.matmul_op(output, output, trans_A=True) * (1 / 5)
grad_f = ad.gradients(loss, [filters]) #gra返回一个list
executor = ad.Executor([grad_f[0]], ctx=ctx)
g_val = executor.run(feed_dict={
inputs: x_val,
filters: filters_val
}) #返回一个list
print("g_val:", g_val[0].asnumpy())
def test_softmax():
ctx = ndarray.gpu(0)
shape = (400, 1000)
x = np.random.uniform(-5, 5, shape).astype(np.float32)
arr_x = ndarray.array(x, ctx=ctx)
arr_y = ndarray.empty(shape, ctx=ctx)
gpu_op.softmax(arr_x, arr_y)
y = arr_y.asnumpy()
np.testing.assert_allclose(autodiff.softmax_func(x), y, rtol=1e-5)
def test_reshape():
ctx = ndarray.gpu(0)
in_shape = (5, 2)
input_arr_np = np.arange(10).reshape(in_shape)
arr_in = ndarray.array(input_arr_np, ctx=ctx)
print(arr_in.asnumpy())
arr_in = arr_in.reshape((5,1,2))
print(arr_in.asnumpy())
print(len(arr_in.shape))
def test_relu():
shape = (2000, 2500)
ctx = ndarray.gpu(0)
x = np.random.uniform(-1, 1, shape).astype(np.float32)
arr_x = ndarray.array(x, ctx=ctx)
arr_y = ndarray.empty(shape, ctx=ctx)
gpu_op.relu(arr_x, arr_y)
y = arr_y.asnumpy()
np.testing.assert_allclose(np.maximum(x, 0).astype(np.float32), y)
def test_matrix_elementwise_multiply_by_const():
shape = (2000, 3000)
ctx = ndarray.gpu(0)
x = np.random.uniform(0, 10, size=shape).astype(np.float32)
val = np.random.uniform(-5, 5)
arr_x = ndarray.array(x, ctx=ctx)
arr_y = ndarray.empty(shape, ctx=ctx)
gpu_op.matrix_elementwise_multiply_by_const(arr_x, val, arr_y)
y = arr_y.asnumpy()
np.testing.assert_allclose(x * val, y, rtol=1e-5)
def test_reduce_sum_axis_n():
ctx = ndarray.gpu(0)
shape = (5)
to_shape = (1,)
x = np.random.uniform(0, 20, shape).astype(np.float32)
arr_x = ndarray.array(x, ctx=ctx)
arr_y = ndarray.empty(to_shape, ctx=ctx)
gpu_op.reduce_sum(arr_x, arr_y)
print(arr_x.shape[0])
print(arr_y.asnumpy())
def test_matrix_elementwise_multiply():
ctx = ndarray.gpu(0)
shape = (500, 200)
x = np.random.uniform(0, 10, size=shape).astype(np.float32)
y = np.random.uniform(0, 10, size=shape).astype(np.float32)
arr_x = ndarray.array(x, ctx=ctx)
arr_y = ndarray.array(y, ctx=ctx)
arr_z = ndarray.empty(shape, ctx=ctx)
gpu_op.matrix_elementwise_multiply(arr_x, arr_y, arr_z)
z = arr_z.asnumpy()
np.testing.assert_allclose(x * y, z, rtol=1e-5)
def test_relu_gradient():
shape = (2000, 2500)
ctx = ndarray.gpu(0)
x = np.random.uniform(-1, 1, shape).astype(np.float32)
grad_x = np.random.uniform(-5, 5, shape).astype(np.float32)
arr_x = ndarray.array(x, ctx=ctx)
arr_grad_x = ndarray.array(grad_x, ctx=ctx)
arr_y = ndarray.empty(shape, ctx=ctx)
gpu_op.relu_gradient(arr_x, arr_grad_x, arr_y)
y = arr_y.asnumpy()
np.testing.assert_allclose(((x > 0) * grad_x).astype(np.float32), y)
def test_dropout():
ctx = ndarray.gpu(0)
in_shape = (5, 1, 2)
input_arr_np = np.arange(10).reshape(in_shape)
arr_in = ndarray.array(input_arr_np, ctx=ctx)
arr_out = ndarray.empty(in_shape,ctx=ctx)
r = gpu_op.dropout_forward(arr_in,arr_out,"NCHW",0.2,1)
print(arr_in.asnumpy())
print(arr_out.asnumpy())
gpu_op.dropout_backward(arr_out, arr_in, "NCHW", 0.2,1, r)
print(arr_in.asnumpy())
def test_array_set():
ctx = ndarray.gpu(0)
shape = (500, 200)
# oneslike
arr_x = ndarray.empty(shape, ctx=ctx)
gpu_op.array_set(arr_x, 1.)
x = arr_x.asnumpy()
np.testing.assert_allclose(np.ones(shape), x)
# zeroslike
gpu_op.array_set(arr_x, 0.)
x = arr_x.asnumpy()
np.testing.assert_allclose(np.zeros(shape), x)
def test_broadcast_to():
ctx = ndarray.gpu(0)
shape = (2, 3)
to_shape = ( 5,2,3)
x = np.random.uniform(-1, 1, shape).astype(np.float32)
arr_x = ndarray.array(x, ctx=ctx)
arr_y = ndarray.empty(to_shape, ctx=ctx)
gpu_op.broadcast_to(arr_x, arr_y)
y = arr_y.asnumpy()
print(arr_x.asnumpy())
print(y)
np.testing.assert_allclose(np.broadcast_to(x, to_shape), y)
def test_softmax_cross_entropy():
ctx = ndarray.gpu(0)
shape = (400, 1000)
y = np.random.uniform(-5, 5, shape).astype(np.float32)
y_ = np.random.uniform(-5, 5, shape).astype(np.float32)
arr_y = ndarray.array(y, ctx=ctx)
arr_y_ = ndarray.array(y_, ctx=ctx)
arr_out = ndarray.empty((1,), ctx=ctx)
gpu_op.softmax_cross_entropy(arr_y, arr_y_, arr_out)
out = arr_out.asnumpy()
# numpy calculation
cross_entropy = np.mean(
-np.sum(y_ * np.log(autodiff.softmax_func(y)), axis=1), keepdims=True)
np.testing.assert_allclose(cross_entropy, out, rtol=1e-5)
def test_convolution_backward():
ctx = ndarray.gpu(0)
x_val1 = np.linspace(0, 100, 100).reshape((5, 1, 20))
filters_val1 = np.ones((1, 1, 20)) * 0.001
y_val = np.array([[[0.07676768]],[[0.23838384]],[[0.40000007]],[[0.5616162 ]],[[0.7232323]]])
x_val = ndarray.array(x_val1, ctx)
d_x_val = ndarray.array(x_val1, ctx)
d_filters_val = ndarray.array(filters_val1, ctx)
filters_val = ndarray.array(filters_val1, ctx)
y_val = ndarray.array(y_val, ctx)
gpu_op.convolution_1d_backward(x_val,y_val,filters_val,d_filters_val,d_x_val,"NCHW","VALID",1)
print(1)
print(d_x_val.asnumpy())
print(d_filters_val.asnumpy())
def test_lr():
W = ad.Variable(name="W")
b = ad.Variable(name="b")
X = ad.Variable(name="X")
y_ = ad.Variable(name="y_")
ctx = ndarray.gpu(0)
# ini
x_val = np.linspace(0, 1, 100).reshape((100, 1))
y_val = x_val + 0.5
W_val = np.array([[0.1]])
b_val = np.array([0.1])
x_val = ndarray.array(x_val, ctx)
W_val = ndarray.array(W_val, ctx)
b_val = ndarray.array(b_val, ctx)
y_val = ndarray.array(y_val, ctx)
z = ad.matmul_op(X, W)
# z.shape = (100,1)
# b.shape = (1,1)
y = z + ad.broadcastto_op(b, z)
# y = (100,1)
y = ad.fullyactivation_forward_op(y, "NCHW", "relu")
loss = ad.matmul_op(y + (-1) * y_, y + (-1) * y_, trans_A=True) * (1 / 100)
# loss = ad.softmaxcrossentropy_op(y, y_)
grad_W, grad_b = ad.gradients(loss, [W, b])
executor = ad.Executor([loss, grad_W, grad_b], ctx)
aph = 1e-6
for i in range(100):
loss_val, grad_W_val, grad_b_val = executor.run(feed_dict={
X: x_val,
b: b_val,
W: W_val,
y_: y_val
})
grad_W_val = grad_W_val.asnumpy()
W_val = W_val.asnumpy()
W_val = W_val - aph * grad_W_val
W_val = ndarray.array(W_val, ctx)
grad_b_val = grad_b_val.asnumpy()
b_val = b_val.asnumpy()
b_val = b_val - aph * grad_b_val
b_val = ndarray.array(b_val, ctx)
print(W_val.asnumpy(), b_val.asnumpy())
def test_l1_l2_regular():
inputs = ad.Variable("inputs")
filters = ad.Variable("filters")
y_ = ad.Variable(name="y_")
# ini
ctx = ndarray.gpu(0)
x_val = np.ones((5, 2)) * 0.5
x_val = ndarray.array(x_val, ctx)
loss = ad.l2regular_op(inputs)
# loss = ad.l1regular_op(inputs)
grad_f = ad.gradients(loss, [inputs]) # gra返回一个list
executor = ad.Executor([loss, grad_f[0]], ctx=ctx)
g_val = executor.run(feed_dict={inputs: x_val}) # 返回一个list
print("g_val:", g_val[0].asnumpy())
print("g_val:", g_val[1].asnumpy())
def test_reduce_mean():
inputs = ad.Variable("inputs")
ctx = ndarray.gpu(0)
shape = (2, 2, 3)
x = np.random.uniform(0, 20, shape).astype(np.float32)
arr_x = ndarray.array(x, ctx=ctx)
outputs = ad.reduce_mean_op(inputs, 1)
f_out = ad.pow_op(outputs, 2)
grad_out = ad.gradients(f_out, [inputs])
executor = ad.Executor([outputs, f_out, grad_out[0]], ctx=ctx)
result = executor.run(feed_dict={inputs: arr_x})
print(arr_x.asnumpy())
print(result[0].asnumpy())
print(result[1].asnumpy())
print(result[2].asnumpy())
def test_reduce_sum_axis_zero():
ctx = ndarray.gpu(0)
shape = (5)
to_shape = (1,)
x = np.random.uniform(0, 20, shape).astype(np.float32)
arr_x = ndarray.array(x, ctx=ctx)
arr_y = ndarray.empty(to_shape, ctx=ctx)
gpu_op.reduce_sum_axis_zero(arr_x, arr_y)
print(arr_x.asnumpy())
print(arr_y.asnumpy())
y = arr_y.asnumpy()
y_ = np.sum(x, axis=0)
for index, _ in np.ndenumerate(y):
v = y[index]
v_ = y_[index]
if abs((v - v_) / v_) > 1e-4:
print(index, v, v_)
np.testing.assert_allclose(np.sum(x, axis=0), y, rtol=1e-5)
def test_l1_l2_cross_loss():
inputs = ad.Variable("inputs")
filters = ad.Variable("filters")
y_ = ad.Variable(name="y_")
# ini
ctx = ndarray.gpu(0)
x_val = np.ones((5, 2)) * 0.5
filters_val = np.ones((2, 2, 10)) * 0.001
y_val = np.ones((5, 2))
x_val = ndarray.array(x_val, ctx)
filters_val = ndarray.array(filters_val, ctx)
y_val = ndarray.array(y_val, ctx)
# loss = ad.crossEntropy_op(inputs, y_)
loss = ad.l1loss_op(inputs, y_)
grad_f = ad.gradients(loss, [inputs, y_]) # gra返回一个list
executor = ad.Executor([loss, grad_f[0], grad_f[1]], ctx=ctx)
g_val = executor.run(feed_dict={inputs: x_val, y_: y_val}) # 返回一个list
print("g_val:", g_val[0].asnumpy())
print("g_val:", g_val[1].asnumpy())
print("g_val:", g_val[2].asnumpy())
def test_convolution_forward():
ctx = ndarray.gpu(0)
in_shape = (1,1,8)
filter_shape = (1, 1, 5)
out_shape = (1, 1, 8)
input_arr_np = np.arange(8).reshape(in_shape)
dinput_arr_np = np.arange(8).reshape(in_shape)
filter_arr_np = np.arange(5).reshape(filter_shape)
dfilter_arr_np = np.arange(5).reshape(filter_shape)
dinput=ndarray.array(dinput_arr_np, ctx=ctx)
dfilter=ndarray.array(dfilter_arr_np, ctx=ctx)
arr_in = ndarray.array(input_arr_np, ctx=ctx)
arr_filter = ndarray.array(filter_arr_np, ctx=ctx)
arr_out = ndarray.empty(out_shape, ctx=ctx)
gpu_op.convolution_1d_forward(arr_in,arr_filter,arr_out,"NCHW","SAME",1)
gpu_op.convolution_1d_backward(arr_in,arr_out,arr_filter,dfilter,dinput,"NCHW","SAME",1)
print(arr_out.asnumpy())
print(dfilter.asnumpy())
print(dinput.asnumpy())
print(arr_in.asnumpy())
def test_convolution_3d_forward_op():
inputs = ad.Variable("inputs")
filters = ad.Variable("filters")
y_ = ad.Variable(name="y_")
#ini
ctx = ndarray.gpu(0)
x_val = np.linspace(0, 100, 135).reshape((5, 1, 3, 3, 3))
filters_val = np.ones((1, 1, 2, 2, 2)) * 0.001
y_val = np.zeros((5, 1))
x_val = ndarray.array(x_val, ctx)
filters_val = ndarray.array(filters_val, ctx)
y_val = ndarray.array(y_val, ctx)
outputs = ad.convolution_3d_forward_op(inputs, filters, "NCHW", "VALID", 1,
1, 1)
outputs_pool = ad.pooling_3d_forward_op(outputs, "NCHW", "max", 0, 0, 0, 1,
1, 1, 2, 2, 2)
outputs_relu = ad.activation_forward_op(outputs_pool, "NCHW", "relu")
outputs_dro = ad.dropout_forward_op(outputs_relu, "NCHW", 0.5, 0)
outputs_f = ad.flatten_op(outputs_dro)
loss = ad.matmul_op(outputs_f, outputs_f, trans_A=True) * (1 / 5)
grad_inputs, grad_f = ad.gradients(loss, [inputs, filters])
executor = ad.Executor([loss, grad_f], ctx=ctx)
aph = 1.0e-6
for i in range(20):
loss_val, filters_grad_val = executor.run(feed_dict={
inputs: x_val,
filters: filters_val
})
filters_val = filters_val.asnumpy()
filters_grad_val = filters_grad_val.asnumpy()
filters_val = filters_val - aph * filters_grad_val
filters_val = ndarray.array(filters_val, ctx)
print("loss_val:", loss_val.asnumpy())
print("filters_val:", filters_val.asnumpy())
def test_sigmoid_conv_1d():
inputs = ad.Variable("inputs")
filters = ad.Variable("filters")
y_ = ad.Variable(name="y_")
# ini
ctx = ndarray.gpu(0)
x_val = np.linspace(0, 100, 80).reshape((5, 1, 4, 4))
filters_val = np.ones((1, 1, 3, 3)) * 0.001
y_val = np.zeros((5, 1))
x_val = ndarray.array(x_val, ctx)
filters_val = ndarray.array(filters_val, ctx)
y_val = ndarray.array(y_val, ctx)
outputs = ad.convolution_2d_forward_op(inputs, filters, "NCHW", "VALID", 1,
1)
# outputs_pool = ad.pooling_2d_forward_op(outputs, "NCHW", "max", 0, 0, 1, 1, 2, 2)
outputs_relu = ad.activation_forward_op(outputs, "NCHW", "relu")
executor = ad.Executor([outputs_relu], ctx=ctx)
loss_val = executor.run(feed_dict={inputs: x_val, filters: filters_val})
print("loss_val:", loss_val[0].asnumpy())
def test_exp_log_reverse_pow():
inputs = ad.Variable("inputs")
filters = ad.Variable("filters")
y_ = ad.Variable(name="y_")
# ini
ctx = ndarray.gpu(0)
x_val = np.linspace(0, 100, 80).reshape((5, 1, 4, 4))
filters_val = np.ones((1, 1, 3, 3)) * 0.001
y_val = np.zeros((5, 1))
x_val = ndarray.array(x_val, ctx)
filters_val = ndarray.array(filters_val, ctx)
y_val = ndarray.array(y_val, ctx)
#outputs = ad.exp_op(inputs)
#outputs = ad.log_op(inputs)
#outputs = ad.reverse_op(inputs)
outputs = ad.pow_op(inputs, 2)
grad_out = ad.gradients(outputs, [inputs])
executor = ad.Executor([outputs, grad_out[0]], ctx=ctx)
result = executor.run(feed_dict={inputs: filters_val})
print(result[0].asnumpy())
print(result[1].asnumpy())
def vgg16():
n = 10
n_class = 10
inputs = ad.Variable("inputs")
filters1_1 = ad.Variable("filters1_1")
filters1_2 = ad.Variable("filters1_2")
filters2_1 = ad.Variable("filters2_1")
filters2_2 = ad.Variable("filters2_2")
filters3_1 = ad.Variable("filters3_1")
filters3_2 = ad.Variable("filters3_2")
filters3_3 = ad.Variable("filters3_3")
filters4_1 = ad.Variable("filters4_1")
filters4_2 = ad.Variable("filters4_2")
filters4_3 = ad.Variable("filters4_3")
filters5_1 = ad.Variable("filters5_1")
filters5_2 = ad.Variable("filters5_2")
filters5_3 = ad.Variable("filters5_3")
filters6 = ad.Variable("filters6")
filters7 = ad.Variable("filters7")
filters8 = ad.Variable("filters8")
biases6 = ad.Variable("biases6")
biases7 = ad.Variable("biases7")
biases8 = ad.Variable("biases8")
y_ = ad.Variable(name="y_")
x_val = np.linspace(0, 0.001, 10*3*224*224).reshape((10, 3, 224, 224))
filters_val = [np.ones((64, 3, 3, 3))*0.001]
filters_val.append(np.ones((64, 64, 3, 3))*0.001)
filters_val.append(np.ones((128, 64, 3, 3))*0.001)
filters_val.append(np.ones((128, 128, 3, 3))*0.001)
filters_val.append(np.ones((256, 128, 3, 3))*0.001)
filters_val.append(np.ones((256, 256, 3, 3))*0.001)
filters_val.append(np.ones((256, 256, 3, 3))*0.001)
filters_val.append(np.ones((512, 256, 3, 3))*0.001)
filters_val.append(np.ones((512, 512, 3, 3))*0.001)
filters_val.append(np.ones((512, 512, 3, 3))*0.001)
filters_val.append(np.ones((512, 512, 3, 3))*0.001)
filters_val.append(np.ones((512, 512, 3, 3))*0.001)
filters_val.append(np.ones((512, 512, 3, 3))*0.001)
filters_val.append(np.ones((512*7*7, 4096)) * 0.001)
filters_val.append(np.ones((4096, 4096)) * 0.001)
filters_val.append(np.ones((4096, n_class)) * 0.001)
biases_val = [np.ones((1, 4096))* 0.001]
biases_val.append(np.ones((1, 4096)) * 0.001)
biases_val.append(np.ones((1, n_class)) * 0.001)
y_val = np.zeros((10, n_class))
ctx = ndarray.gpu(0)
for i in range(16):
filters_val[i] = ndarray.array(filters_val[i], ctx)
# conv 1
conv1_1 = ad.convolution_2d_forward_op(inputs, filters1_1, "NCHW", "SAME", 1, 1)
bn1_1 = ad.bn_forward_op(conv1_1, "NCHW", "pre_activation")
act1_1 = ad.activation_forward_op(bn1_1, "NCHW", "relu")
conv1_2 = ad.convolution_2d_forward_op(act1_1, filters1_2, "NCHW", "SAME", 1, 1)
bn1_2 = ad.bn_forward_op(conv1_2, "NCHW", "pre_activation")
act1_2 = ad.activation_forward_op(bn1_2, "NCHW", "relu")
pool1 = ad.pooling_2d_forward_op(act1_2, "NCHW", "max", 0, 0, 2, 2, 2, 2)
# conv 2
conv2_1 = ad.convolution_2d_forward_op(pool1, filters2_1, "NCHW", "SAME", 1, 1)
bn2_1 = ad.bn_forward_op(conv2_1, "NCHW", "pre_activation")
act2_1 = ad.activation_forward_op(bn2_1, "NCHW", "relu")
conv2_2 = ad.convolution_2d_forward_op(act2_1, filters2_2, "NCHW", "SAME", 1, 1)
bn2_2 = ad.bn_forward_op(conv2_2, "NCHW", "pre_activation")
act2_2 = ad.activation_forward_op(bn2_2, "NCHW", "relu")
pool2 = ad.pooling_2d_forward_op(act2_2, "NCHW", "max", 0, 0, 2, 2, 2, 2)
# conv 3
conv3_1 = ad.convolution_2d_forward_op(pool2, filters3_1, "NCHW", "SAME", 1, 1)
bn3_1 = ad.bn_forward_op(conv3_1, "NCHW", "pre_activation")
act3_1 = ad.activation_forward_op(bn3_1, "NCHW", "relu")
conv3_2 = ad.convolution_2d_forward_op(act3_1, filters3_2, "NCHW", "SAME", 1, 1)
bn3_2 = ad.bn_forward_op(conv3_2, "NCHW", "pre_activation")
act3_2 = ad.activation_forward_op(bn3_2, "NCHW", "relu")
conv3_3 = ad.convolution_2d_forward_op(act3_2, filters3_3, "NCHW", "SAME", 1, 1)
bn3_3 = ad.bn_forward_op(conv3_3, "NCHW", "pre_activation")
act3_3 = ad.activation_forward_op(bn3_3, "NCHW", "relu")
pool3 = ad.pooling_2d_forward_op(act3_3, "NCHW", "max", 0, 0, 2, 2, 2, 2)
# conv 4
conv4_1 = ad.convolution_2d_forward_op(pool3, filters4_1, "NCHW", "SAME", 1, 1)
bn4_1 = ad.bn_forward_op(conv4_1, "NCHW", "pre_activation")
act4_1 = ad.activation_forward_op(bn4_1, "NCHW", "relu")
conv4_2 = ad.convolution_2d_forward_op(act4_1, filters4_2, "NCHW", "SAME", 1, 1)
bn4_2 = ad.bn_forward_op(conv4_2, "NCHW", "pre_activation")
act4_2 = ad.activation_forward_op(bn4_2, "NCHW", "relu")
conv4_3 = ad.convolution_2d_forward_op(act4_2, filters4_3, "NCHW", "SAME", 1, 1)
bn4_3 = ad.bn_forward_op(conv4_3, "NCHW", "pre_activation")
act4_3 = ad.activation_forward_op(bn4_3, "NCHW", "relu")
pool4 = ad.pooling_2d_forward_op(act4_3, "NCHW", "max", 0, 0, 2, 2, 2, 2)
# conv 5
conv5_1 = ad.convolution_2d_forward_op(pool4, filters5_1, "NCHW", "SAME", 1, 1)
bn5_1 = ad.bn_forward_op(conv5_1, "NCHW", "pre_activation")
act5_1 = ad.activation_forward_op(bn5_1, "NCHW", "relu")
conv5_2 = ad.convolution_2d_forward_op(act5_1, filters5_2, "NCHW", "SAME", 1, 1)
bn5_2 = ad.bn_forward_op(conv5_2, "NCHW", "pre_activation")
act5_2 = ad.activation_forward_op(bn5_2, "NCHW", "relu")
conv5_3 = ad.convolution_2d_forward_op(act5_2, filters5_3, "NCHW", "SAME", 1, 1)
bn5_3 = ad.bn_forward_op(conv5_3, "NCHW", "pre_activation")
act5_3 = ad.activation_forward_op(bn5_3, "NCHW", "relu")
pool5 = ad.pooling_2d_forward_op(act5_3, "NCHW", "max", 0, 0, 2, 2, 2, 2)
# fc6
pool5_flat = ad.flatten_op(pool5)
mul6 = ad.matmul_op(pool5_flat, filters6)
add6 = ad.add_op(mul6, biases6)
bn6 = ad.fullybn_forward_op(add6, "NCHW")
fc6 = ad.fullyactivation_forward_op(bn6, "NCHW", "relu")
drop6 = ad.fullydropout_forward_op(fc6, "NCHW", 0.5)
# fc7
mul7 = ad.matmul_op(drop6, filters7)
add7 = ad.add_op(mul7, biases7)
bn7 = ad.fullybn_forward_op(add7, "NCHW")
fc7 = ad.fullyactivation_forward_op(bn7, "NCHW", "relu")
drop7 = ad.fullydropout_forward_op(fc7, "NCHW", 0.5)
#fc8
mul8 = ad.matmul_op(drop7, filters8)
add8 = ad.add_op(mul8, biases8)
fc8 = ad.fullyactivation_forward_op(add8, "NCHW", "softmax")
loss = ad.l2loss_op(fc8, y_)
grad = ad.gradients(loss, [filters1_1, filters1_2, filters2_1, filters2_2, filters3_1, filters3_2, filters3_3
, filters4_1, filters4_2, filters4_3, filters5_1, filters5_2, filters5_3
, filters6, filters7])
executor = ad.Executor([grad[0], grad[1], grad[2], grad[3], grad[4], grad[5], grad[6], grad[7], grad[8], grad[9]
, grad[10], grad[11], grad[12], grad[13], grad[14], loss, y_], ctx=ctx)
aph = 1.0e-6
for i in range(20):
select = random.randint(0, n-1)
tmp_x_val = x_val[select]
tmp_x_val = np.expand_dims(tmp_x_val, 0)
tmp_y_val = y_val[select]
tmp_y_val = np.expand_dims(tmp_y_val, 0)
grad_val = executor.run(
feed_dict={inputs: tmp_x_val, y_: tmp_y_val
, filters1_1: filters_val[0], filters1_2: filters_val[1], filters2_1: filters_val[2], filters2_2: filters_val[3]
, filters3_1: filters_val[4], filters3_2: filters_val[5], filters3_3: filters_val[6]
, filters4_1: filters_val[7], filters4_2: filters_val[8], filters4_3: filters_val[9]
, filters5_1: filters_val[10], filters5_2: filters_val[11], filters5_3: filters_val[12]
, filters6: filters_val[13], filters7: filters_val[14], filters8: filters_val[15]
, biases6: biases_val[0], biases7: biases_val[1], biases8: biases_val[2]})
for i in range(14):
sgd_update_gpu(filters_val[i], grad_val[i], aph)
print(filters_val[0].asnumpy())
return filters_val
# if args.model == "logreg":
# models = [mnist_logreg]
# elif args.model == "mlp":
# models = [mnist_mlp]
# elif args.model == "all":
# models = [mnist_logreg, mnist_mlp]
#
# if args.executor_context == "numpy":
# executor_ctx = None
# elif args.executor_context == "gpu":
# # Assume only use gpu 0.
# executor_ctx = ndarray.gpu(0)
#
# if args.print_loss_val_each_epoch:
# print_loss_val_each_epoch = True
num_epochs = 20
models = [mnist_mlp]
executor_ctx = ndarray.gpu(0)
executor_ctx_cpu = ndarray.cpu(0)
print_loss_val_each_epoch = True
for m in models:
import time
tic = time.time()
m(executor_ctx, num_epochs, print_loss_val_each_epoch)
toc = time.time()
print("mode use time: " + str(toc - tic))
def inception_v3():
X = ad.Placeholder("inputs")
y_ = ad.Placeholder("y_")
inputs = ad.Variable("inputs")
filterb_1 = ad.Variable("filterb_1")
filterb_2 = ad.Variable("filterb_2")
filterb_3 = ad.Variable("filterb_3")
filterb_4 = ad.Variable("filterb_4")
filterb_5 = ad.Variable("filterb_5")
ctx = ndarray.gpu(0)
filtersb_val1 = np.ones((32, 3, 3, 3)) * 0.01
filtersb_val2 = np.ones((32, 32, 3, 3)) * 0.01
filtersb_val3 = np.ones((64, 32, 3, 3)) * 0.01
filtersb_val4 = np.ones((80, 64, 1, 1)) * 0.01
filtersb_val5 = np.ones((192, 80, 3, 3)) * 0.01
y_val = np.zeros((147, 147, 64))
#inception前
covb_1 = ad.convolution_2d_forward_op(inputs, filterb_1, "NCHW", "VALID",
2, 2)
covb_2 = ad.convolution_2d_forward_op(covb_1, filterb_2, "NCHW", "VALID",
1, 1)
covb_3 = ad.convolution_2d_forward_op(covb_2, filterb_3, "NCHW", "SAME", 1,
1)
poolb = ad.pooling_2d_forward_op(covb_3, "NCHW", "max", 0, 0, 2, 2, 3, 3)
covb_4 = ad.convolution_2d_forward_op(poolb, filterb_4, "NCHW", "VALID", 1,
1)
covb_5 = ad.convolution_2d_forward_op(covb_4, filterb_5, "NCHW", "VALID",
1, 1)
covb = ad.pooling_2d_forward_op(covb_5, "NCHW", "max", 0, 0, 2, 2, 3, 3)
# inception_moudle1
#inception_moudle1_1
filter1_1_0 = ad.Variable("filter1_1_0")
filter1_1_1a = ad.Variable("filter1_1_1a")
filter1_1_1b = ad.Variable("filter1_1_1b")
filter1_1_2a = ad.Variable("filter1_1_2a")
filter1_1_2b = ad.Variable("filter1_1_2b")
filter1_1_2c = ad.Variable("filter1_1_2c")
filter1_1_3 = ad.Variable("filter1_1_3a")
filter1_1_0_val = np.ones((64, 192, 1, 1)) * 0.01
filter1_1_1_vala = np.ones((48, 192, 1, 1)) * 0.01
filter1_1_1_valb = np.ones((64, 48, 5, 5)) * 0.01
filter1_1_2_vala = np.ones((64, 192, 1, 1)) * 0.01
filter1_1_2_valb = np.ones((96, 64, 3, 3)) * 0.01
filter1_1_2_valc = np.ones((96, 96, 3, 3)) * 0.01
filter1_1_3_val = np.ones((32, 192, 1, 1)) * 0.01
# branch_0
incep1_1_0 = ad.convolution_2d_forward_op(covb, filter1_1_0, "NCHW",
"SAME", 1, 1)
# branch 1
incep1_1_1a = ad.convolution_2d_forward_op(covb, filter1_1_1a, "NCHW",
"SAME", 1, 1)
incep1_1_1 = ad.convolution_2d_forward_op(incep1_1_1a, filter1_1_1b,
"NCHW", "SAME", 1, 1)
# branch 2
incep1_1_2a = ad.convolution_2d_forward_op(covb, filter1_1_2a, "NCHW",
"SAME", 1, 1)
incep1_1_2b = ad.convolution_2d_forward_op(incep1_1_2a, filter1_1_2b,
"NCHW", "SAME", 1, 1)
incep1_1_2 = ad.convolution_2d_forward_op(incep1_1_2b, filter1_1_2c,
"NCHW", "SAME", 1, 1)
# branch 3
incep1_1_3a = ad.pooling_2d_forward_op(covb, "NCHW", "mean", 1, 1, 1, 1, 3,
3)
incep1_1_3 = ad.convolution_2d_forward_op(incep1_1_3a, filter1_1_3, "NCHW",
"SAME", 1, 1)
concat1_1a = ad.concat_forward_op(incep1_1_0, incep1_1_1)
concat1_1b = ad.concat_forward_op(concat1_1a, incep1_1_2)
concat1_1 = ad.concat_forward_op(concat1_1b, incep1_1_3)
#inception_moudle1_2
filter1_2_0 = ad.Variable("filter1_2_0")
filter1_2_1a = ad.Variable("filter1_2_1a")
filter1_2_1b = ad.Variable("filter1_2_1b")
filter1_2_2a = ad.Variable("filter1_2_2a")
filter1_2_2b = ad.Variable("filter1_2_2b")
filter1_2_2c = ad.Variable("filter1_2_2c")
filter1_2_3 = ad.Variable("filter1_2_3a")
filter1_2_0_val = np.ones((64, 256, 1, 1)) * 0.01
filter1_2_1_vala = np.ones((48, 256, 1, 1)) * 0.01
filter1_2_1_valb = np.ones((64, 48, 5, 5)) * 0.01
filter1_2_2_vala = np.ones((64, 256, 1, 1)) * 0.01
filter1_2_2_valb = np.ones((96, 64, 3, 3)) * 0.01
filter1_2_2_valc = np.ones((96, 96, 3, 3)) * 0.01
filter1_2_3_val = np.ones((64, 256, 1, 1)) * 0.01
# branch_0
incep1_2_0 = ad.convolution_2d_forward_op(concat1_1, filter1_2_0, "NCHW",
"SAME", 1, 1)
# branch 1
incep1_2_1a = ad.convolution_2d_forward_op(concat1_1, filter1_2_1a, "NCHW",
"SAME", 1, 1)
incep1_2_1 = ad.convolution_2d_forward_op(incep1_2_1a, filter1_2_1b,
"NCHW", "SAME", 1, 1)
# branch 2
incep1_2_2a = ad.convolution_2d_forward_op(concat1_1, filter1_2_2a, "NCHW",
"SAME", 1, 1)
incep1_2_2b = ad.convolution_2d_forward_op(incep1_2_2a, filter1_2_2b,
"NCHW", "SAME", 1, 1)
incep1_2_2 = ad.convolution_2d_forward_op(incep1_2_2b, filter1_2_2c,
"NCHW", "SAME", 1, 1)
# branch 3
incep1_2_3a = ad.pooling_2d_forward_op(concat1_1, "NCHW", "mean", 1, 1, 1,
1, 3, 3)
incep1_2_3 = ad.convolution_2d_forward_op(incep1_2_3a, filter1_2_3, "NCHW",
"SAME", 1, 1)
concat1_2a = ad.concat_forward_op(incep1_2_0, incep1_2_1)
concat1_2b = ad.concat_forward_op(concat1_2a, incep1_2_2)
concat1_2 = ad.concat_forward_op(concat1_2b, incep1_2_3)
# inception_moudle1_3
filter1_3_0 = ad.Variable("filter1_3_0")
filter1_3_1a = ad.Variable("filter1_3_1a")
filter1_3_1b = ad.Variable("filter1_3_1b")
filter1_3_2a = ad.Variable("filter1_3_2a")
filter1_3_2b = ad.Variable("filter1_3_2b")
filter1_3_2c = ad.Variable("filter1_3_2c")
filter1_3_3 = ad.Variable("filter1_3_3a")
filter1_3_0_val = np.ones((64, 288, 1, 1)) * 0.01
filter1_3_1_vala = np.ones((48, 288, 1, 1)) * 0.01
filter1_3_1_valb = np.ones((64, 48, 5, 5)) * 0.01
filter1_3_2_vala = np.ones((64, 288, 1, 1)) * 0.01
filter1_3_2_valb = np.ones((96, 64, 3, 3)) * 0.01
filter1_3_2_valc = np.ones((96, 96, 3, 3)) * 0.01
filter1_3_3_val = np.ones((64, 288, 1, 1)) * 0.01
# branch_0
incep1_3_0 = ad.convolution_2d_forward_op(concat1_2, filter1_3_0, "NCHW",
"SAME", 1, 1)
# branch 1
incep1_3_1a = ad.convolution_2d_forward_op(concat1_2, filter1_3_1a, "NCHW",
"SAME", 1, 1)
incep1_3_1 = ad.convolution_2d_forward_op(incep1_3_1a, filter1_3_1b,
"NCHW", "SAME", 1, 1)
# branch 2
incep1_3_2a = ad.convolution_2d_forward_op(concat1_2, filter1_3_2a, "NCHW",
"SAME", 1, 1)
incep1_3_2b = ad.convolution_2d_forward_op(incep1_3_2a, filter1_3_2b,
"NCHW", "SAME", 1, 1)
incep1_3_2 = ad.convolution_2d_forward_op(incep1_3_2b, filter1_3_2c,
"NCHW", "SAME", 1, 1)
# branch 3
incep1_3_3a = ad.pooling_2d_forward_op(concat1_2, "NCHW", "mean", 1, 1, 1,
1, 3, 3)
incep1_3_3 = ad.convolution_2d_forward_op(incep1_3_3a, filter1_3_3, "NCHW",
"SAME", 1, 1)
concat1_3a = ad.concat_forward_op(incep1_3_0, incep1_3_1)
concat1_3b = ad.concat_forward_op(concat1_3a, incep1_3_2)
concat1_3 = ad.concat_forward_op(concat1_3b, incep1_3_3)
#
#
#
#
# # inception_moudle2
# inception_moudle2_1
filter2_1_0 = ad.Variable("filter2_1_0")
filter2_1_1a = ad.Variable("filter2_1_1a")
filter2_1_1b = ad.Variable("filter2_1_1b")
filter2_1_1c = ad.Variable("filter2_1_1c")
filter2_1_0_val = np.ones((384, 288, 3, 3)) * 0.01
filter2_1_1_vala = np.ones((64, 288, 1, 1)) * 0.01
filter2_1_1_valb = np.ones((96, 64, 3, 3)) * 0.01
filter2_1_1_valc = np.ones((96, 96, 3, 3)) * 0.01
# branch_0
incep2_1_0 = ad.convolution_2d_forward_op(concat1_3, filter2_1_0, "NCHW",
"VALID", 2, 2)
# branch 1
incep2_1_1a = ad.convolution_2d_forward_op(concat1_3, filter2_1_1a, "NCHW",
"SAME", 1, 1)
incep2_1_1b = ad.convolution_2d_forward_op(incep2_1_1a, filter2_1_1b,
"NCHW", "SAME", 1, 1)
incep2_1_1 = ad.convolution_2d_forward_op(incep2_1_1b, filter2_1_1c,
"NCHW", "VALID", 2, 2)
# branch 2
incep2_1_2 = ad.pooling_2d_forward_op(concat1_3, "NCHW", "mean", 0, 0, 2,
2, 3, 3)
concat2_1a = ad.concat_forward_op(incep2_1_0, incep2_1_1)
concat2_1 = ad.concat_forward_op(concat2_1a, incep2_1_2)
# inception_moudle2_2
filter2_2_0 = ad.Variable("filter2_2_0")
filter2_2_1a = ad.Variable("filter2_2_1a")
filter2_2_1b = ad.Variable("filter2_2_1b")
filter2_2_1c = ad.Variable("filter2_2_1c")
filter2_2_2a = ad.Variable("filter2_2_2a")
filter2_2_2b = ad.Variable("filter2_2_2b")
filter2_2_2c = ad.Variable("filter2_2_2c")
filter2_2_2d = ad.Variable("filter2_2_2d")
filter2_2_2e = ad.Variable("filter2_2_2e")
filter2_2_3 = ad.Variable("filter2_2_3a")
filter2_2_0_val = np.ones((192, 768, 1, 1)) * 0.01
filter2_2_1_vala = np.ones((128, 768, 1, 1)) * 0.01
filter2_2_1_valb = np.ones((128, 128, 1, 7)) * 0.01
filter2_2_1_valc = np.ones((192, 128, 7, 1)) * 0.01
filter2_2_2_vala = np.ones((128, 768, 1, 1)) * 0.01
filter2_2_2_valb = np.ones((128, 128, 7, 1)) * 0.01
filter2_2_2_valc = np.ones((128, 128, 1, 7)) * 0.01
filter2_2_2_vald = np.ones((128, 128, 7, 1)) * 0.01
filter2_2_2_vale = np.ones((192, 128, 1, 7)) * 0.01
filter2_2_3_val = np.ones((192, 768, 1, 1)) * 0.01
# branch_0
incep2_2_0 = ad.convolution_2d_forward_op(concat2_1, filter2_2_0, "NCHW",
"SAME", 1, 1)
# branch 1
incep2_2_1a = ad.convolution_2d_forward_op(concat2_1, filter2_2_1a, "NCHW",
"SAME", 1, 1)
incep2_2_1b = ad.convolution_2d_forward_op(incep2_2_1a, filter2_2_1b,
"NCHW", "SAME", 1, 1)
incep2_2_1 = ad.convolution_2d_forward_op(incep2_2_1b, filter2_2_1c,
"NCHW", "SAME", 1, 1)
# branch 2
incep2_2_2a = ad.convolution_2d_forward_op(concat2_1, filter2_2_2a, "NCHW",
"SAME", 1, 1)
incep2_2_2b = ad.convolution_2d_forward_op(incep2_2_2a, filter2_2_2b,
"NCHW", "SAME", 1, 1)
incep2_2_2c = ad.convolution_2d_forward_op(incep2_2_2b, filter2_2_2c,
"NCHW", "SAME", 1, 1)
incep2_2_2d = ad.convolution_2d_forward_op(incep2_2_2c, filter2_2_2d,
"NCHW", "SAME", 1, 1)
incep2_2_2 = ad.convolution_2d_forward_op(incep2_2_2d, filter2_2_2e,
"NCHW", "SAME", 1, 1)
# branch 3
incep2_2_3a = ad.pooling_2d_forward_op(concat2_1, "NCHW", "mean", 1, 1, 1,
1, 3, 3)
incep2_2_3 = ad.convolution_2d_forward_op(incep2_2_3a, filter2_2_3, "NCHW",
"SAME", 1, 1)
concat2_2a = ad.concat_forward_op(incep2_2_0, incep2_2_1)
concat2_2b = ad.concat_forward_op(concat2_2a, incep2_2_2)
concat2_2 = ad.concat_forward_op(concat2_2b, incep2_2_3)
# inception_moudle2_3
# filter2_3_0 = ad.Variable("filter2_3_0")
# filter2_3_1a = ad.Variable("filter2_3_1a")
# filter2_3_1b = ad.Variable("filter2_3_1b")
# filter2_3_1c = ad.Variable("filter2_3_1c")
# filter2_3_2a = ad.Variable("filter2_3_2a")
# filter2_3_2b = ad.Variable("filter2_3_2b")
# filter2_3_2c = ad.Variable("filter2_3_2c")
# filter2_3_2d = ad.Variable("filter2_3_2d")
# filter2_3_2e = ad.Variable("filter2_3_2e")
# filter2_3_3 = ad.Variable("filter2_3_3a")
#
# filter2_3_0_val = np.ones((192, 768, 1, 1)) * 0.01
# filter2_3_1_vala = np.ones((160, 768, 1, 1)) * 0.01
# filter2_3_1_valb = np.ones((160, 160, 1, 7)) * 0.01
# filter2_3_1_valc = np.ones((192, 160, 7, 1)) * 0.01
# filter2_3_2_vala = np.ones((160, 768, 1, 1)) * 0.01
# filter2_3_2_valb = np.ones((160, 160, 7,1)) * 0.01
# filter2_3_2_valc = np.ones((160, 160, 1, 7)) * 0.01
# filter2_3_2_vald = np.ones((160, 160, 7, 1)) * 0.01
# filter2_3_2_vale = np.ones((192, 160, 1, 7)) * 0.01
# filter2_3_3_val = np.ones((192, 768, 1, 1)) * 0.01
#
# # branch_0
# incep2_3_0 = ad.convolution_2d_forward_op(concat2_2, filter2_3_0, "NCHW", "SAME", 1, 1)
# # branch 1
# incep2_3_1a = ad.convolution_2d_forward_op(concat2_2, filter2_3_1a, "NCHW", "SAME", 1, 1)
# incep2_3_1b = ad.convolution_2d_forward_op(incep2_3_1a, filter2_3_1b, "NCHW", "SAME", 1, 1)
# incep2_3_1 = ad.convolution_2d_forward_op(incep2_3_1b, filter2_3_1c, "NCHW", "SAME", 1, 1)
# # branch 2
# incep2_3_2a = ad.convolution_2d_forward_op(concat2_2, filter2_3_2a, "NCHW", "SAME", 1, 1)
# incep2_3_2b = ad.convolution_2d_forward_op(incep2_3_2a, filter2_3_2b, "NCHW", "SAME", 1, 1)
# incep2_3_2c = ad.convolution_2d_forward_op(incep2_3_2b, filter2_3_2c, "NCHW", "SAME", 1, 1)
# incep2_3_2d = ad.convolution_2d_forward_op(incep2_3_2c, filter2_3_2d, "NCHW", "SAME", 1, 1)
# incep2_3_2 = ad.convolution_2d_forward_op(incep2_3_2d, filter2_3_2e, "NCHW", "SAME", 1, 1)
# # branch 3
# incep2_3_3a = ad.pooling_2d_forward_op(concat2_2, "NCHW", "mean", 1, 1, 1, 1, 3, 3)
# incep2_3_3 = ad.convolution_2d_forward_op(incep2_3_3a, filter2_3_3, "NCHW", "SAME", 1, 1)
#
# concat2_3a = ad.concat_forward_op(incep2_3_0, incep2_3_1)
# concat2_3b = ad.concat_forward_op(concat2_3a, incep2_3_2)
# concat2_3 = ad.concat_forward_op(concat2_3b, incep2_3_3)
# inception_moudle2_4
# filter2_4_0 = ad.Variable("filter2_4_0")
# filter2_4_1a = ad.Variable("filter2_4_1a")
# filter2_4_1b = ad.Variable("filter2_4_1b")
# filter2_4_1c = ad.Variable("filter2_4_1c")
# filter2_4_2a = ad.Variable("filter2_4_2a")
# filter2_4_2b = ad.Variable("filter2_4_2b")
# filter2_4_2c = ad.Variable("filter2_4_2c")
# filter2_4_2d = ad.Variable("filter2_4_2d")
# filter2_4_2e = ad.Variable("filter2_4_2e")
# filter2_4_3 = ad.Variable("filter2_4_3a")
#
# filter2_4_0_val = np.ones((192, 768, 1, 1)) * 0.01
# filter2_4_1_vala = np.ones((160, 768, 1, 1)) * 0.01
# filter2_4_1_valb = np.ones((160, 160, 1, 7)) * 0.01
# filter2_4_1_valc = np.ones((192, 160, 7, 1)) * 0.01
# filter2_4_2_vala = np.ones((160, 768, 1, 1)) * 0.01
# filter2_4_2_valb = np.ones((160, 160, 7, 1)) * 0.01
# filter2_4_2_valc = np.ones((160, 160, 1, 7)) * 0.01
# filter2_4_2_vald = np.ones((160, 160, 7, 1)) * 0.01
# filter2_4_2_vale = np.ones((192, 160, 1, 7)) * 0.01
# filter2_4_3_val = np.ones((192, 768, 1, 1)) * 0.01
#
# # branch_0
# incep2_4_0 = ad.convolution_2d_forward_op(concat2_3, filter2_4_0, "NCHW", "SAME", 1, 1)
# # branch 1
# incep2_4_1a = ad.convolution_2d_forward_op(concat2_3, filter2_4_1a, "NCHW", "SAME", 1, 1)
# incep2_4_1b = ad.convolution_2d_forward_op(incep2_4_1a, filter2_4_1b, "NCHW", "SAME", 1, 1)
# incep2_4_1 = ad.convolution_2d_forward_op(incep2_4_1b, filter2_4_1c, "NCHW", "SAME", 1, 1)
# # branch 2
# incep2_4_2a = ad.convolution_2d_forward_op(concat2_3, filter2_4_2a, "NCHW", "SAME", 1, 1)
# incep2_4_2b = ad.convolution_2d_forward_op(incep2_4_2a, filter2_4_2b, "NCHW", "SAME", 1, 1)
# incep2_4_2c = ad.convolution_2d_forward_op(incep2_4_2b, filter2_4_2c, "NCHW", "SAME", 1, 1)
# incep2_4_2d = ad.convolution_2d_forward_op(incep2_4_2c, filter2_4_2d, "NCHW", "SAME", 1, 1)
# incep2_4_2 = ad.convolution_2d_forward_op(incep2_4_2d, filter2_4_2e, "NCHW", "SAME", 1, 1)
# # branch 3
# incep2_4_3a = ad.pooling_2d_forward_op(concat2_3, "NCHW", "mean", 1, 1, 1, 1, 3, 3)
# incep2_4_3 = ad.convolution_2d_forward_op(incep2_4_3a, filter2_4_3, "NCHW", "SAME", 1, 1)
#
# concat2_4a = ad.concat_forward_op(incep2_4_0, incep2_4_1)
# concat2_4b = ad.concat_forward_op(concat2_4a, incep2_4_2)
# concat2_4 = ad.concat_forward_op(concat2_4b, incep2_4_3)
# inception_moudle2_5
# filter2_5_0 = ad.Variable("filter2_5_0")
# filter2_5_1a = ad.Variable("filter2_5_1a")
# filter2_5_1b = ad.Variable("filter2_5_1b")
# filter2_5_1c = ad.Variable("filter2_5_1c")
# filter2_5_2a = ad.Variable("filter2_5_2a")
# filter2_5_2b = ad.Variable("filter2_5_2b")
# filter2_5_2c = ad.Variable("filter2_5_2c")
# filter2_5_2d = ad.Variable("filter2_5_2d")
# filter2_5_2e = ad.Variable("filter2_5_2e")
# filter2_5_3 = ad.Variable("filter2_5_3a")
#
# filter2_5_0_val = np.ones((192, 768, 1, 1)) * 0.01
# filter2_5_1_vala = np.ones((160, 768, 1, 1)) * 0.01
# filter2_5_1_valb = np.ones((160, 160, 1, 7)) * 0.01
# filter2_5_1_valc = np.ones((192, 160, 7, 1)) * 0.01
# filter2_5_2_vala = np.ones((160, 768, 1, 1)) * 0.01
# filter2_5_2_valb = np.ones((160, 160, 7, 1)) * 0.01
# filter2_5_2_valc = np.ones((160, 160, 1, 7)) * 0.01
# filter2_5_2_vald = np.ones((160, 160, 7, 1)) * 0.01
# filter2_5_2_vale = np.ones((192, 160, 1, 7)) * 0.01
# filter2_5_3_val = np.ones((192, 768, 1, 1)) * 0.01
#
# # branch_0
# incep2_5_0 = ad.convolution_2d_forward_op(concat2_4, filter2_5_0, "NCHW", "SAME", 1, 1)
# # branch 1
# incep2_5_1a = ad.convolution_2d_forward_op(concat2_4, filter2_5_1a, "NCHW", "SAME", 1, 1)
# incep2_5_1b = ad.convolution_2d_forward_op(incep2_5_1a, filter2_5_1b, "NCHW", "SAME", 1, 1)
# incep2_5_1 = ad.convolution_2d_forward_op(incep2_5_1b, filter2_5_1c, "NCHW", "SAME", 1, 1)
# # branch 2
# incep2_5_2a = ad.convolution_2d_forward_op(concat2_4, filter2_5_2a, "NCHW", "SAME", 1, 1)
# incep2_5_2b = ad.convolution_2d_forward_op(incep2_5_2a, filter2_5_2b, "NCHW", "SAME", 1, 1)
# incep2_5_2c = ad.convolution_2d_forward_op(incep2_5_2b, filter2_5_2c, "NCHW", "SAME", 1, 1)
# incep2_5_2d = ad.convolution_2d_forward_op(incep2_5_2c, filter2_5_2d, "NCHW", "SAME", 1, 1)
# incep2_5_2 = ad.convolution_2d_forward_op(incep2_5_2d, filter2_5_2e, "NCHW", "SAME", 1, 1)
# # branch 3
# incep2_5_3a = ad.pooling_2d_forward_op(concat2_4, "NCHW", "mean", 1, 1, 1, 1, 3, 3)
# incep2_5_3 = ad.convolution_2d_forward_op(incep2_5_3a, filter2_5_3, "NCHW", "SAME", 1, 1)
#
# concat2_5a = ad.concat_forward_op(incep2_5_0, incep2_5_1)
# concat2_5b = ad.concat_forward_op(concat2_5a, incep2_5_2)
# concat2_5 = ad.concat_forward_op(concat2_5b, incep2_5_3)
# # inception_moudle3
# inception_moudle3_1
filter3_1_0a = ad.Variable("filter3_1_0a")
filter3_1_0b = ad.Variable("filter3_1_0b")
filter3_1_1a = ad.Variable("filter3_1_1a")
filter3_1_1b = ad.Variable("filter3_1_1b")
filter3_1_1c = ad.Variable("filter3_1_1c")
filter3_1_1d = ad.Variable("filter3_1_1d")
filter3_1_0_vala = np.ones((192, 768, 1, 1)) * 0.01
filter3_1_0_valb = np.ones((320, 192, 3, 3)) * 0.01
filter3_1_1_vala = np.ones((192, 768, 1, 1)) * 0.01
filter3_1_1_valb = np.ones((192, 192, 1, 7)) * 0.01
filter3_1_1_valc = np.ones((192, 192, 7, 1)) * 0.01
filter3_1_1_vald = np.ones((192, 192, 3, 3)) * 0.01
# branch_0
incep3_1_0a = ad.convolution_2d_forward_op(concat2_2, filter3_1_0a, "NCHW",
"SAME", 1, 1)
incep3_1_0 = ad.convolution_2d_forward_op(incep3_1_0a, filter3_1_0b,
"NCHW", "VALID", 2, 2)
# branch 1
incep3_1_1a = ad.convolution_2d_forward_op(concat2_2, filter3_1_1a, "NCHW",
"SAME", 1, 1)
incep3_1_1b = ad.convolution_2d_forward_op(incep3_1_1a, filter3_1_1b,
"NCHW", "SAME", 1, 1)
incep3_1_1c = ad.convolution_2d_forward_op(incep3_1_1b, filter3_1_1c,
"NCHW", "SAME", 1, 1)
incep3_1_1 = ad.convolution_2d_forward_op(incep3_1_1c, filter3_1_1d,
"NCHW", "VALID", 2, 2)
# branch 2
incep3_1_2 = ad.pooling_2d_forward_op(concat2_2, "NCHW", "mean", 0, 0, 2,
2, 3, 3)
concat3_1a = ad.concat_forward_op(incep3_1_0, incep3_1_1)
concat3_1 = ad.concat_forward_op(concat3_1a, incep3_1_2)
# inception_moudle3_2
filter3_2_0 = ad.Variable("filter3_2_0")
filter3_2_1a = ad.Variable("filter3_2_1a")
filter3_2_1b = ad.Variable("filter3_2_1b")
filter3_2_1c = ad.Variable("filter3_2_1c")
filter3_2_2a = ad.Variable("filter3_2_2a")
filter3_2_2b = ad.Variable("filter3_2_2b")
filter3_2_2c = ad.Variable("filter3_2_2c")
filter3_2_2d = ad.Variable("filter3_2_2d")
filter3_2_3 = ad.Variable("filter3_2_3a")
filter3_2_0_val = np.ones((320, 1280, 1, 1)) * 0.01
filter3_2_1_vala = np.ones((384, 1280, 1, 1)) * 0.01
filter3_2_1_valb = np.ones((384, 384, 1, 3)) * 0.01
filter3_2_1_valc = np.ones((384, 384, 3, 1)) * 0.01
filter3_2_2_vala = np.ones((448, 1280, 1, 1)) * 0.01
filter3_2_2_valb = np.ones((384, 448, 3, 3)) * 0.01
filter3_2_2_valc = np.ones((384, 384, 1, 3)) * 0.01
filter3_2_2_vald = np.ones((384, 384, 3, 1)) * 0.01
filter3_2_3_val = np.ones((192, 1280, 1, 1)) * 0.01
# branch_0
incep3_2_0 = ad.convolution_2d_forward_op(concat3_1, filter3_2_0, "NCHW",
"SAME", 1, 1)
# branch 1
incep3_2_1a = ad.convolution_2d_forward_op(concat3_1, filter3_2_1a, "NCHW",
"SAME", 1, 1)
incep3_2_1b = ad.convolution_2d_forward_op(incep3_2_1a, filter3_2_1b,
"NCHW", "SAME", 1, 1)
incep3_2_1c = ad.convolution_2d_forward_op(incep3_2_1a, filter3_2_1c,
"NCHW", "SAME", 1, 1)
incep3_2_1 = ad.concat_forward_op(incep3_2_1b, incep3_2_1c)
# branch 2
incep3_2_2a = ad.convolution_2d_forward_op(concat3_1, filter3_2_2a, "NCHW",
"SAME", 1, 1)
incep3_2_2b = ad.convolution_2d_forward_op(incep3_2_2a, filter3_2_2b,
"NCHW", "SAME", 1, 1)
incep3_2_2c = ad.convolution_2d_forward_op(incep3_2_2b, filter3_2_2c,
"NCHW", "SAME", 1, 1)
incep3_2_2d = ad.convolution_2d_forward_op(incep3_2_2b, filter3_2_2d,
"NCHW", "SAME", 1, 1)
incep3_2_2 = ad.concat_forward_op(incep3_2_2c, incep3_2_2d)
# branch 3
incep3_2_3a = ad.pooling_2d_forward_op(concat3_1, "NCHW", "mean", 1, 1, 1,
1, 3, 3)
incep3_2_3 = ad.convolution_2d_forward_op(incep3_2_3a, filter3_2_3, "NCHW",
"SAME", 1, 1)
concat3_2a = ad.concat_forward_op(incep3_2_0, incep3_2_1)
concat3_2b = ad.concat_forward_op(concat3_2a, incep3_2_2)
concat3_2 = ad.concat_forward_op(concat3_2b, incep3_2_3)
# # inception_moudle3_3
# filter3_3_0 = ad.Variable("filter3_3_0")
# filter3_3_1a = ad.Variable("filter3_3_1a")
# filter3_3_1b = ad.Variable("filter3_3_1b")
# filter3_3_1c = ad.Variable("filter3_3_1c")
# filter3_3_2a = ad.Variable("filter3_3_2a")
# filter3_3_2b = ad.Variable("filter3_3_2b")
# filter3_3_2c = ad.Variable("filter3_3_2c")
# filter3_3_2d = ad.Variable("filter3_3_2d")
# filter3_3_3 = ad.Variable("filter3_3_3a")
#
# filter3_3_0_val = np.ones((320, 2048, 1, 1)) * 0.01
# filter3_3_1_vala = np.ones((384, 2048, 1, 1)) * 0.01
# filter3_3_1_valb = np.ones((384, 384, 1, 3)) * 0.01
# filter3_3_1_valc = np.ones((384, 384, 3, 1)) * 0.01
# filter3_3_2_vala = np.ones((448, 2048, 1, 1)) * 0.01
# filter3_3_2_valb = np.ones((384, 448, 3, 3)) * 0.01
# filter3_3_2_valc = np.ones((384, 384, 1, 3)) * 0.01
# filter3_3_2_vald = np.ones((384, 384, 3, 1)) * 0.01
# filter3_3_3_val = np.ones((192, 2048, 1, 1)) * 0.01
#
# # branch_0
# incep3_3_0 = ad.convolution_2d_forward_op(concat3_2, filter3_3_0, "NCHW", "SAME", 1, 1)
# # branch 1
# incep3_3_1a = ad.convolution_2d_forward_op(concat3_2, filter3_3_1a, "NCHW", "SAME", 1, 1)
# incep3_3_1b = ad.convolution_2d_forward_op(incep3_3_1a, filter3_3_1b, "NCHW", "SAME", 1, 1)
# incep3_3_1c = ad.convolution_2d_forward_op(incep3_3_1a, filter3_3_1c, "NCHW", "SAME", 1, 1)
# incep3_3_1 = ad.concat_forward_op(incep3_3_1b, incep3_3_1c)
#
# # branch 2
# incep3_3_2a = ad.convolution_2d_forward_op(concat3_2, filter3_3_2a, "NCHW", "SAME", 1, 1)
# incep3_3_2b = ad.convolution_2d_forward_op(incep3_3_2a, filter3_3_2b, "NCHW", "SAME", 1, 1)
# incep3_3_2c = ad.convolution_2d_forward_op(incep3_3_2b, filter3_3_2c, "NCHW", "SAME", 1, 1)
# incep3_3_2d = ad.convolution_2d_forward_op(incep3_3_2b, filter3_3_2d, "NCHW", "SAME", 1, 1)
# incep3_3_2 = ad.concat_forward_op(incep3_3_2c, incep3_3_2d)
# # branch 3
# incep3_3_3a = ad.pooling_2d_forward_op(concat3_2, "NCHW", "mean", 1, 1, 1, 1, 3, 3)
# incep3_3_3 = ad.convolution_2d_forward_op(incep3_3_3a, filter3_3_3, "NCHW", "SAME", 1, 1)
#
# concat3_3a = ad.concat_forward_op(incep3_3_0, incep3_3_1)
# concat3_3b = ad.concat_forward_op(concat3_3a, incep3_3_2)
# concat3_3 = ad.concat_forward_op(concat3_3b, incep3_3_3)
filtera1 = ad.Variable("filtera1")
filtera1val = np.ones((1000, 2048, 1, 1)) * 0.01
filtersmul = ad.Variable("filtersmul")
filtersmulval = np.ones((1000, n_class)) * 0.01
biases = ad.Variable("biases")
biasesval = np.ones((batch_size, n_class)) * 0.01
poollast = ad.pooling_2d_forward_op(concat3_2, "NCHW", "mean", 0, 0, 1, 1,
8, 8)
dropout = ad.dropout_forward_op(poollast, "NCHW", 0.8)
convlast = ad.convolution_2d_forward_op(dropout, filtera1, "NCHW", "SAME",
1, 1)
squeeze = ad.squeeze_op(convlast)
# fc8
mul8 = ad.matmul_op(squeeze, filtersmul)
add8 = ad.add_op(mul8, biases)
fc8 = ad.fullyactivation_forward_op(add8, "NCHW", "softmax")
loss = ad.softmaxcrossentropy_op(fc8, y_)
X_val = np.empty(shape=(batch_size, 3, 32, 32), dtype=np.float32)
y_val = np.empty(shape=(batch_size, n_class), dtype=np.float32)
aph = 0.001
t = train.Adam_minimize(loss, aph)
t.init_Variable({
filterb_1: filtersb_val1,
filterb_2: filtersb_val2,
filterb_3: filtersb_val3,
filterb_4: filtersb_val4,
filterb_5: filtersb_val5,
filter1_1_0: filter1_1_0_val,
filter1_1_1a: filter1_1_1_vala,
filter1_1_1b: filter1_1_1_valb,
filter1_1_2a: filter1_1_2_vala,
filter1_1_2b: filter1_1_2_valb,
filter1_1_2c: filter1_1_2_valc,
filter1_1_3: filter1_1_3_val,
filter1_2_0: filter1_2_0_val,
filter1_2_1a: filter1_2_1_vala,
filter1_2_1b: filter1_2_1_valb,
filter1_2_2a: filter1_2_2_vala,
filter1_2_2b: filter1_2_2_valb,
filter1_2_2c: filter1_2_2_valc,
filter1_2_3: filter1_2_3_val,
filter1_3_0: filter1_3_0_val,
filter1_3_1a: filter1_3_1_vala,
filter1_3_1b: filter1_3_1_valb,
filter1_3_2a: filter1_3_2_vala,
filter1_3_2b: filter1_3_2_valb,
filter1_3_2c: filter1_3_2_valc,
filter1_3_3: filter1_3_3_val,
filter2_1_0: filter2_1_0_val,
filter2_1_1a: filter2_1_1_vala,
filter2_1_1b: filter2_1_1_valb,
filter2_1_1c: filter2_1_1_valc,
filter2_2_0: filter2_2_0_val,
filter2_2_1a: filter2_2_1_vala,
filter2_2_1b: filter2_2_1_valb,
filter2_2_1c: filter2_2_1_valc,
filter2_2_2a: filter2_2_2_vala,
filter2_2_2b: filter2_2_2_valb,
filter2_2_2c: filter2_2_2_valc,
filter2_2_2d: filter2_2_2_vald,
filter2_2_2e: filter2_2_2_vale,
filter2_2_3: filter2_2_3_val
# , filter2_3_0: filter2_3_0_val, filter2_3_1a: filter2_3_1_vala, filter2_3_1b: filter2_3_1_valb,
# filter2_3_1c: filter2_3_1_valc,
# filter2_3_2a: filter2_3_2_vala, filter2_3_2b: filter2_3_2_valb,
# filter2_3_2c: filter2_3_2_valc, filter2_3_2d: filter2_3_2_vald,
# filter2_3_2e: filter2_3_2_vale, filter2_3_3: filter2_3_3_val
# , filter2_4_0: filter2_4_0_val, filter2_4_1a: filter2_4_1_vala, filter2_4_1b: filter2_4_1_valb,
# filter2_4_1c: filter2_4_1_valc,
# filter2_4_2a: filter2_4_2_vala, filter2_4_2b: filter2_4_2_valb,
# filter2_4_2c: filter2_4_2_valc, filter2_4_2d: filter2_4_2_vald,
# filter2_4_2e: filter2_4_2_vale, filter2_4_3: filter2_4_3_val
# , filter2_5_0: filter2_5_0_val, filter2_5_1a: filter2_5_1_vala, filter2_5_1b: filter2_5_1_valb,
# filter2_5_1c: filter2_5_1_valc,
# filter2_5_2a: filter2_5_2_vala, filter2_5_2b: filter2_5_2_valb,
# filter2_5_2c: filter2_5_2_valc, filter2_5_2d: filter2_5_2_vald,
# filter2_5_2e: filter2_5_2_vale, filter2_5_3: filter2_5_3_val
,
filter3_1_0a: filter3_1_0_vala,
filter3_1_0b: filter3_1_0_valb,
filter3_1_1a: filter3_1_1_vala,
filter3_1_1b: filter3_1_1_valb,
filter3_1_1c: filter3_1_1_valc,
filter3_1_1d: filter3_1_1_vald,
filter3_2_0: filter3_2_0_val,
filter3_2_1a: filter3_2_1_vala,
filter3_2_1b: filter3_2_1_valb,
filter3_2_1c: filter3_2_1_valc,
filter3_2_2a: filter3_2_2_vala,
filter3_2_2b: filter3_2_2_valb,
filter3_2_2c: filter3_2_2_valc,
filter3_2_2d: filter3_2_2_vald,
filter3_2_3: filter3_2_3_val
# , filter3_3_0: filter3_3_0_val, filter3_3_1a: filter3_3_1_vala,
# filter3_3_1b: filter3_3_1_valb,
# filter3_3_1c: filter3_3_1_valc, filter3_3_2a: filter3_3_2_vala,
# filter3_3_2b: filter3_3_2_valb,
# filter3_3_2c: filter3_3_2_valc, filter3_3_2d: filter3_3_2_vald,
# filter3_3_3: filter3_3_3_val,
,
filtera1: filtera1val,
filtersmul: filtersmulval,
biases: biasesval
})
train_x, train_y, test_x, test_y = prepare_data()
num_epochs = 10
n_train_batches = train_x.shape[0] // batch_size
n_test_batches = test_x.shape[0] // batch_size
for i in range(num_epochs):
print("epoch %d" % i)
for minibatch_index in range(n_train_batches):
minibatch_start = minibatch_index * batch_size
minibatch_end = (minibatch_index + 1) * batch_size
X_val[:] = train_x[minibatch_start:minibatch_end]
y_val[:] = train_y[minibatch_start:minibatch_end]
# print(y_val.shape)
# y_val[:] = convert_to_one_hot(train_y[minibatch_start:minibatch_end])
t.run({X: X_val, y_: y_val})
# print(t.run_get_nodelist_once({X: X_val, y_: y_val}, [fc8])[fc8].asnumpy())
# print(map_to_numpy(t.get_Variable_node_to_val_map()))
print("loss_val:", t.get_loss().asnumpy())
def test_inception_v4():
X = ad.Placeholder("X")
Y_ = ad.Placeholder("y_")
f1 = ad.Variable("f1")
f2 = ad.Variable("f2")
f3 = ad.Variable("f3")
f4 = ad.Variable("f4")
f5_1 = ad.Variable("f5_1")
f5_2 = ad.Variable("f5_2")
f6_1 = ad.Variable("f6_1")
f6_2 = ad.Variable("f6_2")
f6_3 = ad.Variable("f6_3")
f6_4 = ad.Variable("f6_4")
f7 = ad.Variable("f7")
W = ad.Variable("W")
b = ad.Variable("b")
keep_prob = ad.Placeholder("keep_prob")
X_val = np.random.normal(0, 0.5, (10, 3, 299, 299))
Y_val = np.random.normal(0, 0.5, (10, 1))
f1val = np.random.normal(0, 0.5, (32, 3, 3, 3))
f2val = np.random.normal(0, 0.5, (32, 32, 3, 3))
f3val = np.random.normal(0, 0.5, (64, 32, 3, 3))
f4val = np.random.normal(0, 0.5, (96, 64, 3, 3))
f5_1val = np.random.normal(0, 0.5, (64, 160, 1, 1))
f5_2val = np.random.normal(0, 0.5, (96, 64, 3, 3))
f6_1val = np.random.normal(0, 0.5, (64, 160, 1, 1))
f6_2val = np.random.normal(0, 0.5, (64, 64, 7, 1))
f6_3val = np.random.normal(0, 0.5, (64, 64, 1, 7))
f6_4val = np.random.normal(0, 0.5, (96, 64, 3, 3))
f7val = np.random.normal(0, 0.5, (192, 192, 3, 3))
# stem
cov1 = ad.convolution_2d_forward_op(X, f1, "NCHW", "VALID", 2, 2)
cov2 = ad.convolution_2d_forward_op(cov1, f2, "NCHW", "VALID", 1, 1)
cov3 = ad.convolution_2d_forward_op(cov2, f3, "NCHW", "SAME", 1, 1)
pool4 = ad.pooling_2d_forward_op(cov3, "NCHW", "max", 0, 0, 2, 2, 3, 3)
cov4 = ad.convolution_2d_forward_op(cov3, f4, "NCHW", "VALID", 2, 2)
concat1 = ad.concat_forward_op(pool4, cov4)
cov5_1 = ad.convolution_2d_forward_op(concat1, f5_1, "NCHW", "SAME", 1, 1)
cov5_2 = ad.convolution_2d_forward_op(cov5_1, f5_2, "NCHW", "VALID", 1, 1)
cov6_1 = ad.convolution_2d_forward_op(concat1, f6_1, "NCHW", "SAME", 1, 1)
cov6_2 = ad.convolution_2d_forward_op(cov6_1, f6_2, "NCHW", "SAME", 1, 1)
cov6_3 = ad.convolution_2d_forward_op(cov6_2, f6_3, "NCHW", "SAME", 1, 1)
cov6_4 = ad.convolution_2d_forward_op(cov6_3, f6_4, "NCHW", "VALID", 1, 1)
concat2 = ad.concat_forward_op(cov5_2, cov6_4)
cov7 = ad.convolution_2d_forward_op(concat2, f7, "NCHW", "VALID", 2, 2)
pool7 = ad.pooling_2d_forward_op(concat2, "NCHW", "max", 0, 0, 2, 2, 3, 3)
concat3 = ad.concat_forward_op(pool7, cov7)
a1, dicta1 = block_inception_a(concat3, "a1")
a2, dicta2 = block_inception_a(a1, "a2")
a3, dicta3 = block_inception_a(a2, "a3")
a4, dicta4 = block_inception_a(a3, "a4")
ra, dictra = block_reduction_a(a4, 384, "ra")
b1, dictb1 = block_inception_b(ra, 1024, "b1")
# b2, dictb2 = block_inception_b(b1, 1024, "b2")
# b3, dictb3 = block_inception_b(b2, 1024, "b3")
# b4, dictb4 = block_inception_b(b3, 1024, "b4")
# b5, dictb5 = block_inception_b(b4, 1024, "b5")
# b6, dictb6 = block_inception_b(b5, 1024, "b6")
# b7, dictb7 = block_inception_b(b6, 1024, "b7")
#
rb, dictrb = block_reduction_b(b1, 1024, "rb")
c1, dictc1 = block_inception_c(rb, 1536, "c1")
# c2, dictc2 = block_inception_c(c1, 1536, "c2")
# c3, dictc3 = block_inception_c(c2, 1536, "c3")
poollast = ad.pooling_2d_forward_op(c1, "NCHW", "mean", 0, 0, 1, 1, 8, 8)
squeeze = ad.squeeze_op(poollast)
drop_out = ad.fullydropout_forward_op(squeeze, "NCHW", 0.8)
mul = ad.matmul_op(drop_out, W)
add = ad.add_op(mul, b)
loss = ad.softmaxcrossentropy_op(add, Y_)
#
# aph = 0.001
# t = train.Adam_minimize(loss, aph)
ctx = ndarray.gpu(0)
# aph = 0.001
# t = train.Adam_minimize(loss, aph)
feed_dict = {
X: X_val,
f1: f1val,
f2: f2val,
f3: f3val,
f4: f4val,
f5_1: f5_1val,
f5_2: f5_2val,
f6_1: f6_1val,
f6_2: f6_2val,
f6_3: f6_3val,
f6_4: f6_4val,
f7: f7val
}
feed_dict.update(dicta1)
feed_dict.update(dicta2)
feed_dict.update(dicta3)
feed_dict.update(dicta4)
feed_dict.update(dictra)
feed_dict.update(dictb1)
# feed_dict.update(dictb2)
# feed_dict.update(dictb3)
# feed_dict.update(dictb4)
# feed_dict.update(dictb5)
# feed_dict.update(dictb6)
# feed_dict.update(dictb7)
feed_dict.update(dictrb)
feed_dict.update(dictc1)
# feed_dict.update(dictc2)
# feed_dict.update(dictc3)
executor = ad.Executor([drop_out], ctx=ctx)
y_val = executor.run(feed_dict)
def ResNet50(inputs, n_class):
X = ad.Placeholder("X")
y_ = ad.Placeholder("y_")
W1 = ad.Variable("W1")
W6 = ad.Variable("W6")
b6 = ad.Variable("b6")
W7 = ad.Variable("W7")
b7 = ad.Variable("b7")
keep_prob = ad.Placeholder("keep_prob")
#conv1
conv1 = ad.convolution_2d_forward_op(X, W1, "NCHW", "VALID", 2, 2)
bn1 = ad.bn_forward_op(conv1, "NCHW", "pre_activation")
act1 = ad.activation_forward_op(bn1, "NCHW", "relu")
pool1 = ad.pooling_2d_forward_op(act1, "NCHW", "max", 0, 0, 2, 2, 3, 3)
#conv2_x
conv2, dict2 = convolutional_block(inputs=pool1,
kernel_size=3,
in_filter=64,
out_filters=[64, 64, 256],
block_name="2a",
stride=1)
iden2_1, dict2_1 = identity_block(inputs=conv2,
kernel_size=3,
in_filter=256,
out_filters=[64, 64, 256],
block_name="2b",
stride=1)
iden2_2, dict2_2 = identity_block(iden2_1, 3, 256, [64, 64, 256], "2c", 1)
#conv3_x
conv3, dict3 = convolutional_block(iden2_2, 3, 256, [128, 128, 512], "3a",
1)
iden3_1, dict3_1 = identity_block(conv3, 3, 512, [128, 128, 512], "3b", 1)
iden3_2, dict3_2 = identity_block(iden3_1, 3, 512, [128, 128, 512], "3c",
1)
iden3_3, dict3_3 = identity_block(iden3_2, 3, 512, [128, 128, 512], "3d",
1)
#conv4_x
conv4, dict4 = convolutional_block(iden3_3, 3, 512, [256, 256, 1024], "4a",
1)
iden4_1, dict4_1 = identity_block(conv4, 3, 1024, [256, 256, 1024], "4b",
1)
iden4_2, dict4_2 = identity_block(iden4_1, 3, 1024, [256, 256, 1024], "4c",
1)
iden4_3, dict4_3 = identity_block(iden4_2, 3, 1024, [256, 256, 1024], "4d",
1)
iden4_4, dict4_4 = identity_block(iden4_3, 3, 1024, [256, 256, 1024], "4e",
1)
iden4_5, dict4_5 = identity_block(iden4_4, 3, 1024, [256, 256, 1024], "4f",
1)
#conv5_x
conv5, dict5 = convolutional_block(iden4_5, 3, 1024, [512, 512, 2048],
"5a", 1)
iden5_1, dict5_1 = identity_block(conv5, 3, 2048, [512, 512, 2048], "5b",
1)
iden5_2, dict5_2 = identity_block(iden5_1, 3, 2048, [512, 512, 2048], "5c",
1)
pool5 = ad.pooling_2d_forward_op(iden5_2, "NCHW", "mean", 0, 0, 1, 1, 2, 2)
pool5_flat = ad.flatten_op(pool5)
mul6 = ad.matmul_op(pool5_flat, W6)
add6 = ad.add_op(mul6, b6)
act6 = ad.fullyactivation_forward_op(add6, "NCHW", "relu")
drop_out = ad.fullydropout_forward_op(act6, "NCHW", keep_prob)
mul7 = ad.matmul_op(drop_out, W7)
add7 = ad.add_op(mul7, b7)
act7 = ad.fullyactivation_forward_op(add7, "NCHW", "softmax")
loss = ad.softmaxcrossentropy_op(act7, y_)
X_val = np.random.normal(0, 0.5, (10, 3, 230, 230))
W1_val = np.random.normal(0, 0.5, (64, 3, 7, 7))
W6_val = np.random.normal(0, 0.5, (7 * 7 * 2048, 50))
b6_val = np.random.normal(0, 0.5, (10, 50))
W7_val = np.random.normal(0, 0.5, (50, 6))
b7_val = np.random.normal(0, 0.5, (10, 6))
y_val = np.random.normal(0, 0.5, (10, 6))
feed_dict = {W1: W1_val, W6: W6_val, W7: W7_val, b6: b6_val, b7: b7_val}
feed_dict.update(dict2)
feed_dict.update(dict2_1)
feed_dict.update(dict2_2)
feed_dict.update(dict3)
feed_dict.update(dict3_1)
feed_dict.update(dict3_2)
feed_dict.update(dict3_3)
feed_dict.update(dict4)
feed_dict.update(dict4_1)
feed_dict.update(dict4_2)
feed_dict.update(dict4_3)
feed_dict.update(dict4_4)
feed_dict.update(dict4_5)
feed_dict.update(dict5)
feed_dict.update(dict5_1)
feed_dict.update(dict5_2)
time.sleep(5)
list = []
for key in feed_dict.keys():
list.append(key)
executor = ad.Executor(list, ctx=ndarray.gpu(0))
executor.run(feed_dict)
time.sleep(5)
def ResNet152(inputs, n_class):
X = ad.Placeholder("X")
y_ = ad.Placeholder("y_")
W1 = ad.Variable("W1")
W6 = ad.Variable("W6")
b6 = ad.Variable("b6")
W7 = ad.Variable("W7")
b7 = ad.Variable("b7")
keep_prob = ad.Placeholder("keep_prob")
#conv1
conv1 = ad.convolution_2d_forward_op(X, W1, "NCHW", "VALID", 2, 2)
bn1 = ad.bn_forward_op(conv1, "NCHW", "pre_activation")
act1 = ad.activation_forward_op(bn1, "NCHW", "relu")
pool1 = ad.pooling_2d_forward_op(act1, "NCHW", "max", 0, 0, 2, 2, 3, 3)
#conv2_x
conv2, dict2 = convolutional_block(inputs=pool1,
kernel_size=3,
in_filter=64,
out_filters=[64, 64, 256],
block_name="2a",
stride=1)
iden2_1, dict2_1 = identity_block(inputs=conv2,
kernel_size=3,
in_filter=256,
out_filters=[64, 64, 256],
block_name="2b",
stride=1)
iden2_2, dict2_2 = identity_block(iden2_1, 3, 256, [64, 64, 256], "2c", 1)
#conv3_x
conv3, dict3 = convolutional_block(iden2_2, 3, 256, [128, 128, 512], "3a",
1)
iden3_1, dict3_1 = identity_block(conv3, 3, 512, [128, 128, 512], "3b", 1)
iden3_2, dict3_2 = identity_block(iden3_1, 3, 512, [128, 128, 512], "3c",
1)
iden3_3, dict3_3 = identity_block(iden3_2, 3, 512, [128, 128, 512], "3d",
1)
iden3_4, dict3_4 = identity_block(iden3_3, 3, 512, [128, 128, 512], "3e",
1)
iden3_5, dict3_5 = identity_block(iden3_4, 3, 512, [128, 128, 512], "3f",
1)
iden3_6, dict3_6 = identity_block(iden3_5, 3, 512, [128, 128, 512], "3g",
1)
iden3_7, dict3_7 = identity_block(iden3_6, 3, 512, [128, 128, 512], "3h",
1)
#conv4_x
conv4, dict4 = convolutional_block(iden3_7, 3, 512, [256, 256, 1024], "4a",
1)
iden4_1, dict4_1 = identity_block(conv4, 3, 1024, [256, 256, 1024], "4b",
1)
iden4_2, dict4_2 = identity_block(iden4_1, 3, 1024, [256, 256, 1024], "4c",
1)
iden4_3, dict4_3 = identity_block(iden4_2, 3, 1024, [256, 256, 1024], "4d",
1)
iden4_4, dict4_4 = identity_block(iden4_3, 3, 1024, [256, 256, 1024], "4e",
1)
iden4_5, dict4_5 = identity_block(iden4_4, 3, 1024, [256, 256, 1024], "4f",
1)
iden4_6, dict4_6 = identity_block(iden4_5, 3, 1024, [256, 256, 1024], "4f",
1)
iden4_7, dict4_7 = identity_block(iden4_6, 3, 1024, [256, 256, 1024], "4f",
1)
iden4_8, dict4_8 = identity_block(iden4_7, 3, 1024, [256, 256, 1024], "4f",
1)
iden4_9, dict4_9 = identity_block(iden4_8, 3, 1024, [256, 256, 1024], "4f",
1)
iden4_10, dict4_10 = identity_block(iden4_9, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_11, dict4_11 = identity_block(iden4_10, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_12, dict4_12 = identity_block(iden4_11, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_13, dict4_13 = identity_block(iden4_12, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_14, dict4_14 = identity_block(iden4_13, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_15, dict4_15 = identity_block(iden4_14, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_16, dict4_16 = identity_block(iden4_15, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_17, dict4_17 = identity_block(iden4_16, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_18, dict4_18 = identity_block(iden4_17, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_19, dict4_19 = identity_block(iden4_18, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_20, dict4_20 = identity_block(iden4_19, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_21, dict4_21 = identity_block(iden4_20, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_22, dict4_22 = identity_block(iden4_21, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_23, dict4_23 = identity_block(iden4_22, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_24, dict4_24 = identity_block(iden4_23, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_25, dict4_25 = identity_block(iden4_24, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_26, dict4_26 = identity_block(iden4_25, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_27, dict4_27 = identity_block(iden4_26, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_28, dict4_28 = identity_block(iden4_27, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_29, dict4_29 = identity_block(iden4_28, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_30, dict4_30 = identity_block(iden4_29, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_31, dict4_31 = identity_block(iden4_30, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_32, dict4_32 = identity_block(iden4_31, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_33, dict4_33 = identity_block(iden4_32, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_34, dict4_34 = identity_block(iden4_33, 3, 1024, [256, 256, 1024],
"4f", 1)
iden4_35, dict4_35 = identity_block(iden4_34, 3, 1024, [256, 256, 1024],
"4f", 1)
#conv5_x
conv5, dict5 = convolutional_block(iden4_35, 3, 1024, [512, 512, 2048],
"5a", 1)
iden5_1, dict5_1 = identity_block(conv5, 3, 2048, [512, 512, 2048], "5b",
1)
iden5_2, dict5_2 = identity_block(iden5_1, 3, 2048, [512, 512, 2048], "5c",
1)
pool5 = ad.pooling_2d_forward_op(iden5_2, "NCHW", "mean", 0, 0, 1, 1, 2, 2)
pool5_flat = ad.flatten_op(pool5)
mul6 = ad.matmul_op(pool5_flat, W6)
add6 = ad.add_op(mul6, b6)
act6 = ad.fullyactivation_forward_op(add6, "NCHW", "relu")
drop_out = ad.fullydropout_forward_op(act6, "NCHW", keep_prob)
mul7 = ad.matmul_op(drop_out, W7)
add7 = ad.add_op(mul7, b7)
act7 = ad.fullyactivation_forward_op(add7, "NCHW", "softmax")
loss = ad.softmaxcrossentropy_op(act7, y_)
X_val = np.random.normal(0, 0.5, (10, 3, 230, 230))
W1_val = np.random.normal(0, 0.5, (64, 3, 7, 7))
W6_val = np.random.normal(0, 0.5, (7 * 7 * 2048, 50))
b6_val = np.random.normal(0, 0.5, (10, 50))
W7_val = np.random.normal(0, 0.5, (50, 6))
b7_val = np.random.normal(0, 0.5, (10, 6))
y_val = np.random.normal(0, 0.5, (10, 6))
aph = 0.001
t = train.Adam_minimize(loss, aph)
feed_dict = {W1: W1_val, W6: W6_val, W7: W7_val, b6: b6_val, b7: b7_val}
feed_dict.update(dict2)
feed_dict.update(dict2_1)
feed_dict.update(dict2_2)
feed_dict.update(dict3)
feed_dict.update(dict3_1)
feed_dict.update(dict3_2)
feed_dict.update(dict3_3)
feed_dict.update(dict3_4)
feed_dict.update(dict3_5)
feed_dict.update(dict3_6)
feed_dict.update(dict3_7)
feed_dict.update(dict4)
feed_dict.update(dict4_1)
feed_dict.update(dict4_2)
feed_dict.update(dict4_3)
feed_dict.update(dict4_4)
feed_dict.update(dict4_5)
feed_dict.update(dict4_6)
feed_dict.update(dict4_7)
feed_dict.update(dict4_8)
feed_dict.update(dict4_9)
feed_dict.update(dict4_10)
feed_dict.update(dict4_11)
feed_dict.update(dict4_12)
feed_dict.update(dict4_13)
feed_dict.update(dict4_14)
feed_dict.update(dict4_15)
feed_dict.update(dict4_16)
feed_dict.update(dict4_17)
feed_dict.update(dict4_18)
feed_dict.update(dict4_19)
feed_dict.update(dict4_20)
feed_dict.update(dict4_21)
feed_dict.update(dict4_22)
feed_dict.update(dict4_23)
feed_dict.update(dict4_24)
feed_dict.update(dict4_25)
feed_dict.update(dict4_26)
feed_dict.update(dict4_27)
feed_dict.update(dict4_28)
feed_dict.update(dict4_29)
feed_dict.update(dict4_30)
feed_dict.update(dict4_31)
feed_dict.update(dict4_32)
feed_dict.update(dict4_33)
feed_dict.update(dict4_34)
feed_dict.update(dict4_35)
feed_dict.update(dict5)
feed_dict.update(dict5_1)
feed_dict.update(dict5_2)
# t.init_Variable(feed_dict)
# t.run({X: X_val, y_: y_val})
# print(t.get_Variable_node_to_val_map()[W1_val].asnumpy())
list = []
for key in feed_dict.keys():
list.append(key)
executor = ad.Executor(list, ctx=ndarray.gpu(0))
executor.run(feed_dict)
