def convolutional_block(inputs, kernel_size, in_filter, out_filters, block_name, stride=1): f1, f2, f3 = out_filters W1 = ad.Variable(block_name + "W1") W2 = ad.Variable(block_name + "W2") W3 = ad.Variable(block_name + "W3") W_shortcut = ad.Variable(block_name + "W_shortcut") rand = np.random.RandomState(seed=123) W1_val = rand.normal(scale=0.1, size=(f1, in_filter, 1, 1)) W2_val = rand.normal(scale=0.1, size=(f2, f1, kernel_size, kernel_size)) W3_val = rand.normal(scale=0.1, size=(f3, f2, 1, 1)) W_shortcut_val = rand.normal(scale=0.1, size=(in_filter, f3, 1, 1)) # conv1 conv1 = ad.convolution_2d_forward_op(inputs, W1, "NCHW", "SAME", stride, stride) bn1 = ad.bn_forward_op(conv1, "NCHW", "pre_activation") act1 = ad.activation_forward_op(bn1, "NCHW", "relu") # conv2 conv2 = ad.convolution_2d_forward_op(act1, W2, "NCHW", "SAME", stride, stride) bn2 = ad.bn_forward_op(conv2, "NCHW", "pre_activation") act2 = ad.activation_forward_op(bn2, "NCHW", "relu") # conv3 conv3 = ad.convolution_2d_forward_op(act2, W3, "NCHW", "VALID", stride, stride) bn3 = ad.bn_forward_op(conv3, "NCHW", "pre_activation") #shortcut_path conv4 = ad.convolution_2d_forward_op(inputs, W_shortcut, "NCHW", "VALID", stride, stride) shortcut = ad.bn_forward_op(conv4, "NCHW", "pre_activation") # shortcut add = ad.add_op(bn3, shortcut) act4 = ad.activation_forward_op(add, "NCHW", "relu") dict = {W1: W1_val, W2: W2_val, W3: W3_val, W_shortcut: W_shortcut_val} return act4, dict
def ResNet50(inputs, n_class): X = ad.Placeholder("X") y_ = ad.Placeholder("y_") W1 = ad.Variable("W1") W6 = ad.Variable("W6") b6 = ad.Variable("b6") W7 = ad.Variable("W7") b7 = ad.Variable("b7") keep_prob = ad.Placeholder("keep_prob") #conv1 conv1 = ad.convolution_2d_forward_op(X, W1, "NCHW", "VALID", 2, 2) bn1 = ad.bn_forward_op(conv1, "NCHW", "pre_activation") act1 = ad.activation_forward_op(bn1, "NCHW", "relu") pool1 = ad.pooling_2d_forward_op(act1, "NCHW", "max", 0, 0, 2, 2, 3, 3) #conv2_x conv2, dict2 = convolutional_block(inputs=pool1, kernel_size=3, in_filter=64, out_filters=[64, 64, 256], block_name="2a", stride=1) iden2_1, dict2_1 = identity_block(inputs=conv2, kernel_size=3, in_filter=256, out_filters=[64, 64, 256], block_name="2b", stride=1) iden2_2, dict2_2 = identity_block(iden2_1, 3, 256, [64, 64, 256], "2c", 1) #conv3_x conv3, dict3 = convolutional_block(iden2_2, 3, 256, [128, 128, 512], "3a", 1) iden3_1, dict3_1 = identity_block(conv3, 3, 512, [128, 128, 512], "3b", 1) iden3_2, dict3_2 = identity_block(iden3_1, 3, 512, [128, 128, 512], "3c", 1) iden3_3, dict3_3 = identity_block(iden3_2, 3, 512, [128, 128, 512], "3d", 1) #conv4_x conv4, dict4 = convolutional_block(iden3_3, 3, 512, [256, 256, 1024], "4a", 1) iden4_1, dict4_1 = identity_block(conv4, 3, 1024, [256, 256, 1024], "4b", 1) iden4_2, dict4_2 = identity_block(iden4_1, 3, 1024, [256, 256, 1024], "4c", 1) iden4_3, dict4_3 = identity_block(iden4_2, 3, 1024, [256, 256, 1024], "4d", 1) iden4_4, dict4_4 = identity_block(iden4_3, 3, 1024, [256, 256, 1024], "4e", 1) iden4_5, dict4_5 = identity_block(iden4_4, 3, 1024, [256, 256, 1024], "4f", 1) #conv5_x conv5, dict5 = convolutional_block(iden4_5, 3, 1024, [512, 512, 2048], "5a", 1) iden5_1, dict5_1 = identity_block(conv5, 3, 2048, [512, 512, 2048], "5b", 1) iden5_2, dict5_2 = identity_block(iden5_1, 3, 2048, [512, 512, 2048], "5c", 1) pool5 = ad.pooling_2d_forward_op(iden5_2, "NCHW", "mean", 0, 0, 1, 1, 2, 2) pool5_flat = ad.flatten_op(pool5) mul6 = ad.matmul_op(pool5_flat, W6) add6 = ad.add_op(mul6, b6) act6 = ad.fullyactivation_forward_op(add6, "NCHW", "relu") drop_out = ad.fullydropout_forward_op(act6, "NCHW", keep_prob) mul7 = ad.matmul_op(drop_out, W7) add7 = ad.add_op(mul7, b7) act7 = ad.fullyactivation_forward_op(add7, "NCHW", "softmax") loss = ad.softmaxcrossentropy_op(act7, y_) X_val = np.random.normal(0, 0.5, (10, 3, 230, 230)) W1_val = np.random.normal(0, 0.5, (64, 3, 7, 7)) W6_val = np.random.normal(0, 0.5, (7 * 7 * 2048, 50)) b6_val = np.random.normal(0, 0.5, (10, 50)) W7_val = np.random.normal(0, 0.5, (50, 6)) b7_val = np.random.normal(0, 0.5, (10, 6)) y_val = np.random.normal(0, 0.5, (10, 6)) feed_dict = {W1: W1_val, W6: W6_val, W7: W7_val, b6: b6_val, b7: b7_val} feed_dict.update(dict2) feed_dict.update(dict2_1) feed_dict.update(dict2_2) feed_dict.update(dict3) feed_dict.update(dict3_1) feed_dict.update(dict3_2) feed_dict.update(dict3_3) feed_dict.update(dict4) feed_dict.update(dict4_1) feed_dict.update(dict4_2) feed_dict.update(dict4_3) feed_dict.update(dict4_4) feed_dict.update(dict4_5) feed_dict.update(dict5) feed_dict.update(dict5_1) feed_dict.update(dict5_2) time.sleep(5) list = [] for key in feed_dict.keys(): list.append(key) executor = ad.Executor(list, ctx=ndarray.gpu(0)) executor.run(feed_dict) time.sleep(5)
def ResNet152(inputs, n_class): X = ad.Placeholder("X") y_ = ad.Placeholder("y_") W1 = ad.Variable("W1") W6 = ad.Variable("W6") b6 = ad.Variable("b6") W7 = ad.Variable("W7") b7 = ad.Variable("b7") keep_prob = ad.Placeholder("keep_prob") #conv1 conv1 = ad.convolution_2d_forward_op(X, W1, "NCHW", "VALID", 2, 2) bn1 = ad.bn_forward_op(conv1, "NCHW", "pre_activation") act1 = ad.activation_forward_op(bn1, "NCHW", "relu") pool1 = ad.pooling_2d_forward_op(act1, "NCHW", "max", 0, 0, 2, 2, 3, 3) #conv2_x conv2, dict2 = convolutional_block(inputs=pool1, kernel_size=3, in_filter=64, out_filters=[64, 64, 256], block_name="2a", stride=1) iden2_1, dict2_1 = identity_block(inputs=conv2, kernel_size=3, in_filter=256, out_filters=[64, 64, 256], block_name="2b", stride=1) iden2_2, dict2_2 = identity_block(iden2_1, 3, 256, [64, 64, 256], "2c", 1) #conv3_x conv3, dict3 = convolutional_block(iden2_2, 3, 256, [128, 128, 512], "3a", 1) iden3_1, dict3_1 = identity_block(conv3, 3, 512, [128, 128, 512], "3b", 1) iden3_2, dict3_2 = identity_block(iden3_1, 3, 512, [128, 128, 512], "3c", 1) iden3_3, dict3_3 = identity_block(iden3_2, 3, 512, [128, 128, 512], "3d", 1) iden3_4, dict3_4 = identity_block(iden3_3, 3, 512, [128, 128, 512], "3e", 1) iden3_5, dict3_5 = identity_block(iden3_4, 3, 512, [128, 128, 512], "3f", 1) iden3_6, dict3_6 = identity_block(iden3_5, 3, 512, [128, 128, 512], "3g", 1) iden3_7, dict3_7 = identity_block(iden3_6, 3, 512, [128, 128, 512], "3h", 1) #conv4_x conv4, dict4 = convolutional_block(iden3_7, 3, 512, [256, 256, 1024], "4a", 1) iden4_1, dict4_1 = identity_block(conv4, 3, 1024, [256, 256, 1024], "4b", 1) iden4_2, dict4_2 = identity_block(iden4_1, 3, 1024, [256, 256, 1024], "4c", 1) iden4_3, dict4_3 = identity_block(iden4_2, 3, 1024, [256, 256, 1024], "4d", 1) iden4_4, dict4_4 = identity_block(iden4_3, 3, 1024, [256, 256, 1024], "4e", 1) iden4_5, dict4_5 = identity_block(iden4_4, 3, 1024, [256, 256, 1024], "4f", 1) iden4_6, dict4_6 = identity_block(iden4_5, 3, 1024, [256, 256, 1024], "4f", 1) iden4_7, dict4_7 = identity_block(iden4_6, 3, 1024, [256, 256, 1024], "4f", 1) iden4_8, dict4_8 = identity_block(iden4_7, 3, 1024, [256, 256, 1024], "4f", 1) iden4_9, dict4_9 = identity_block(iden4_8, 3, 1024, [256, 256, 1024], "4f", 1) iden4_10, dict4_10 = identity_block(iden4_9, 3, 1024, [256, 256, 1024], "4f", 1) iden4_11, dict4_11 = identity_block(iden4_10, 3, 1024, [256, 256, 1024], "4f", 1) iden4_12, dict4_12 = identity_block(iden4_11, 3, 1024, [256, 256, 1024], "4f", 1) iden4_13, dict4_13 = identity_block(iden4_12, 3, 1024, [256, 256, 1024], "4f", 1) iden4_14, dict4_14 = identity_block(iden4_13, 3, 1024, [256, 256, 1024], "4f", 1) iden4_15, dict4_15 = identity_block(iden4_14, 3, 1024, [256, 256, 1024], "4f", 1) iden4_16, dict4_16 = identity_block(iden4_15, 3, 1024, [256, 256, 1024], "4f", 1) iden4_17, dict4_17 = identity_block(iden4_16, 3, 1024, [256, 256, 1024], "4f", 1) iden4_18, dict4_18 = identity_block(iden4_17, 3, 1024, [256, 256, 1024], "4f", 1) iden4_19, dict4_19 = identity_block(iden4_18, 3, 1024, [256, 256, 1024], "4f", 1) iden4_20, dict4_20 = identity_block(iden4_19, 3, 1024, [256, 256, 1024], "4f", 1) iden4_21, dict4_21 = identity_block(iden4_20, 3, 1024, [256, 256, 1024], "4f", 1) iden4_22, dict4_22 = identity_block(iden4_21, 3, 1024, [256, 256, 1024], "4f", 1) iden4_23, dict4_23 = identity_block(iden4_22, 3, 1024, [256, 256, 1024], "4f", 1) iden4_24, dict4_24 = identity_block(iden4_23, 3, 1024, [256, 256, 1024], "4f", 1) iden4_25, dict4_25 = identity_block(iden4_24, 3, 1024, [256, 256, 1024], "4f", 1) iden4_26, dict4_26 = identity_block(iden4_25, 3, 1024, [256, 256, 1024], "4f", 1) iden4_27, dict4_27 = identity_block(iden4_26, 3, 1024, [256, 256, 1024], "4f", 1) iden4_28, dict4_28 = identity_block(iden4_27, 3, 1024, [256, 256, 1024], "4f", 1) iden4_29, dict4_29 = identity_block(iden4_28, 3, 1024, [256, 256, 1024], "4f", 1) iden4_30, dict4_30 = identity_block(iden4_29, 3, 1024, [256, 256, 1024], "4f", 1) iden4_31, dict4_31 = identity_block(iden4_30, 3, 1024, [256, 256, 1024], "4f", 1) iden4_32, dict4_32 = identity_block(iden4_31, 3, 1024, [256, 256, 1024], "4f", 1) iden4_33, dict4_33 = identity_block(iden4_32, 3, 1024, [256, 256, 1024], "4f", 1) iden4_34, dict4_34 = identity_block(iden4_33, 3, 1024, [256, 256, 1024], "4f", 1) iden4_35, dict4_35 = identity_block(iden4_34, 3, 1024, [256, 256, 1024], "4f", 1) #conv5_x conv5, dict5 = convolutional_block(iden4_35, 3, 1024, [512, 512, 2048], "5a", 1) iden5_1, dict5_1 = identity_block(conv5, 3, 2048, [512, 512, 2048], "5b", 1) iden5_2, dict5_2 = identity_block(iden5_1, 3, 2048, [512, 512, 2048], "5c", 1) pool5 = ad.pooling_2d_forward_op(iden5_2, "NCHW", "mean", 0, 0, 1, 1, 2, 2) pool5_flat = ad.flatten_op(pool5) mul6 = ad.matmul_op(pool5_flat, W6) add6 = ad.add_op(mul6, b6) act6 = ad.fullyactivation_forward_op(add6, "NCHW", "relu") drop_out = ad.fullydropout_forward_op(act6, "NCHW", keep_prob) mul7 = ad.matmul_op(drop_out, W7) add7 = ad.add_op(mul7, b7) act7 = ad.fullyactivation_forward_op(add7, "NCHW", "softmax") loss = ad.softmaxcrossentropy_op(act7, y_) X_val = np.random.normal(0, 0.5, (10, 3, 230, 230)) W1_val = np.random.normal(0, 0.5, (64, 3, 7, 7)) W6_val = np.random.normal(0, 0.5, (7 * 7 * 2048, 50)) b6_val = np.random.normal(0, 0.5, (10, 50)) W7_val = np.random.normal(0, 0.5, (50, 6)) b7_val = np.random.normal(0, 0.5, (10, 6)) y_val = np.random.normal(0, 0.5, (10, 6)) aph = 0.001 t = train.Adam_minimize(loss, aph) feed_dict = {W1: W1_val, W6: W6_val, W7: W7_val, b6: b6_val, b7: b7_val} feed_dict.update(dict2) feed_dict.update(dict2_1) feed_dict.update(dict2_2) feed_dict.update(dict3) feed_dict.update(dict3_1) feed_dict.update(dict3_2) feed_dict.update(dict3_3) feed_dict.update(dict3_4) feed_dict.update(dict3_5) feed_dict.update(dict3_6) feed_dict.update(dict3_7) feed_dict.update(dict4) feed_dict.update(dict4_1) feed_dict.update(dict4_2) feed_dict.update(dict4_3) feed_dict.update(dict4_4) feed_dict.update(dict4_5) feed_dict.update(dict4_6) feed_dict.update(dict4_7) feed_dict.update(dict4_8) feed_dict.update(dict4_9) feed_dict.update(dict4_10) feed_dict.update(dict4_11) feed_dict.update(dict4_12) feed_dict.update(dict4_13) feed_dict.update(dict4_14) feed_dict.update(dict4_15) feed_dict.update(dict4_16) feed_dict.update(dict4_17) feed_dict.update(dict4_18) feed_dict.update(dict4_19) feed_dict.update(dict4_20) feed_dict.update(dict4_21) feed_dict.update(dict4_22) feed_dict.update(dict4_23) feed_dict.update(dict4_24) feed_dict.update(dict4_25) feed_dict.update(dict4_26) feed_dict.update(dict4_27) feed_dict.update(dict4_28) feed_dict.update(dict4_29) feed_dict.update(dict4_30) feed_dict.update(dict4_31) feed_dict.update(dict4_32) feed_dict.update(dict4_33) feed_dict.update(dict4_34) feed_dict.update(dict4_35) feed_dict.update(dict5) feed_dict.update(dict5_1) feed_dict.update(dict5_2) # t.init_Variable(feed_dict) # t.run({X: X_val, y_: y_val}) # print(t.get_Variable_node_to_val_map()[W1_val].asnumpy()) list = [] for key in feed_dict.keys(): list.append(key) executor = ad.Executor(list, ctx=ndarray.gpu(0)) executor.run(feed_dict)
def vgg16(): n = 10 n_class = 10 inputs = ad.Variable("inputs") filters1_1 = ad.Variable("filters1_1") filters1_2 = ad.Variable("filters1_2") filters2_1 = ad.Variable("filters2_1") filters2_2 = ad.Variable("filters2_2") filters3_1 = ad.Variable("filters3_1") filters3_2 = ad.Variable("filters3_2") filters3_3 = ad.Variable("filters3_3") filters4_1 = ad.Variable("filters4_1") filters4_2 = ad.Variable("filters4_2") filters4_3 = ad.Variable("filters4_3") filters5_1 = ad.Variable("filters5_1") filters5_2 = ad.Variable("filters5_2") filters5_3 = ad.Variable("filters5_3") filters6 = ad.Variable("filters6") filters7 = ad.Variable("filters7") filters8 = ad.Variable("filters8") biases6 = ad.Variable("biases6") biases7 = ad.Variable("biases7") biases8 = ad.Variable("biases8") y_ = ad.Variable(name="y_") x_val = np.linspace(0, 0.001, 10*3*224*224).reshape((10, 3, 224, 224)) filters_val = [np.ones((64, 3, 3, 3))*0.001] filters_val.append(np.ones((64, 64, 3, 3))*0.001) filters_val.append(np.ones((128, 64, 3, 3))*0.001) filters_val.append(np.ones((128, 128, 3, 3))*0.001) filters_val.append(np.ones((256, 128, 3, 3))*0.001) filters_val.append(np.ones((256, 256, 3, 3))*0.001) filters_val.append(np.ones((256, 256, 3, 3))*0.001) filters_val.append(np.ones((512, 256, 3, 3))*0.001) filters_val.append(np.ones((512, 512, 3, 3))*0.001) filters_val.append(np.ones((512, 512, 3, 3))*0.001) filters_val.append(np.ones((512, 512, 3, 3))*0.001) filters_val.append(np.ones((512, 512, 3, 3))*0.001) filters_val.append(np.ones((512, 512, 3, 3))*0.001) filters_val.append(np.ones((512*7*7, 4096)) * 0.001) filters_val.append(np.ones((4096, 4096)) * 0.001) filters_val.append(np.ones((4096, n_class)) * 0.001) biases_val = [np.ones((1, 4096))* 0.001] biases_val.append(np.ones((1, 4096)) * 0.001) biases_val.append(np.ones((1, n_class)) * 0.001) y_val = np.zeros((10, n_class)) ctx = ndarray.gpu(0) for i in range(16): filters_val[i] = ndarray.array(filters_val[i], ctx) # conv 1 conv1_1 = ad.convolution_2d_forward_op(inputs, filters1_1, "NCHW", "SAME", 1, 1) bn1_1 = ad.bn_forward_op(conv1_1, "NCHW", "pre_activation") act1_1 = ad.activation_forward_op(bn1_1, "NCHW", "relu") conv1_2 = ad.convolution_2d_forward_op(act1_1, filters1_2, "NCHW", "SAME", 1, 1) bn1_2 = ad.bn_forward_op(conv1_2, "NCHW", "pre_activation") act1_2 = ad.activation_forward_op(bn1_2, "NCHW", "relu") pool1 = ad.pooling_2d_forward_op(act1_2, "NCHW", "max", 0, 0, 2, 2, 2, 2) # conv 2 conv2_1 = ad.convolution_2d_forward_op(pool1, filters2_1, "NCHW", "SAME", 1, 1) bn2_1 = ad.bn_forward_op(conv2_1, "NCHW", "pre_activation") act2_1 = ad.activation_forward_op(bn2_1, "NCHW", "relu") conv2_2 = ad.convolution_2d_forward_op(act2_1, filters2_2, "NCHW", "SAME", 1, 1) bn2_2 = ad.bn_forward_op(conv2_2, "NCHW", "pre_activation") act2_2 = ad.activation_forward_op(bn2_2, "NCHW", "relu") pool2 = ad.pooling_2d_forward_op(act2_2, "NCHW", "max", 0, 0, 2, 2, 2, 2) # conv 3 conv3_1 = ad.convolution_2d_forward_op(pool2, filters3_1, "NCHW", "SAME", 1, 1) bn3_1 = ad.bn_forward_op(conv3_1, "NCHW", "pre_activation") act3_1 = ad.activation_forward_op(bn3_1, "NCHW", "relu") conv3_2 = ad.convolution_2d_forward_op(act3_1, filters3_2, "NCHW", "SAME", 1, 1) bn3_2 = ad.bn_forward_op(conv3_2, "NCHW", "pre_activation") act3_2 = ad.activation_forward_op(bn3_2, "NCHW", "relu") conv3_3 = ad.convolution_2d_forward_op(act3_2, filters3_3, "NCHW", "SAME", 1, 1) bn3_3 = ad.bn_forward_op(conv3_3, "NCHW", "pre_activation") act3_3 = ad.activation_forward_op(bn3_3, "NCHW", "relu") pool3 = ad.pooling_2d_forward_op(act3_3, "NCHW", "max", 0, 0, 2, 2, 2, 2) # conv 4 conv4_1 = ad.convolution_2d_forward_op(pool3, filters4_1, "NCHW", "SAME", 1, 1) bn4_1 = ad.bn_forward_op(conv4_1, "NCHW", "pre_activation") act4_1 = ad.activation_forward_op(bn4_1, "NCHW", "relu") conv4_2 = ad.convolution_2d_forward_op(act4_1, filters4_2, "NCHW", "SAME", 1, 1) bn4_2 = ad.bn_forward_op(conv4_2, "NCHW", "pre_activation") act4_2 = ad.activation_forward_op(bn4_2, "NCHW", "relu") conv4_3 = ad.convolution_2d_forward_op(act4_2, filters4_3, "NCHW", "SAME", 1, 1) bn4_3 = ad.bn_forward_op(conv4_3, "NCHW", "pre_activation") act4_3 = ad.activation_forward_op(bn4_3, "NCHW", "relu") pool4 = ad.pooling_2d_forward_op(act4_3, "NCHW", "max", 0, 0, 2, 2, 2, 2) # conv 5 conv5_1 = ad.convolution_2d_forward_op(pool4, filters5_1, "NCHW", "SAME", 1, 1) bn5_1 = ad.bn_forward_op(conv5_1, "NCHW", "pre_activation") act5_1 = ad.activation_forward_op(bn5_1, "NCHW", "relu") conv5_2 = ad.convolution_2d_forward_op(act5_1, filters5_2, "NCHW", "SAME", 1, 1) bn5_2 = ad.bn_forward_op(conv5_2, "NCHW", "pre_activation") act5_2 = ad.activation_forward_op(bn5_2, "NCHW", "relu") conv5_3 = ad.convolution_2d_forward_op(act5_2, filters5_3, "NCHW", "SAME", 1, 1) bn5_3 = ad.bn_forward_op(conv5_3, "NCHW", "pre_activation") act5_3 = ad.activation_forward_op(bn5_3, "NCHW", "relu") pool5 = ad.pooling_2d_forward_op(act5_3, "NCHW", "max", 0, 0, 2, 2, 2, 2) # fc6 pool5_flat = ad.flatten_op(pool5) mul6 = ad.matmul_op(pool5_flat, filters6) add6 = ad.add_op(mul6, biases6) bn6 = ad.fullybn_forward_op(add6, "NCHW") fc6 = ad.fullyactivation_forward_op(bn6, "NCHW", "relu") drop6 = ad.fullydropout_forward_op(fc6, "NCHW", 0.5) # fc7 mul7 = ad.matmul_op(drop6, filters7) add7 = ad.add_op(mul7, biases7) bn7 = ad.fullybn_forward_op(add7, "NCHW") fc7 = ad.fullyactivation_forward_op(bn7, "NCHW", "relu") drop7 = ad.fullydropout_forward_op(fc7, "NCHW", 0.5) #fc8 mul8 = ad.matmul_op(drop7, filters8) add8 = ad.add_op(mul8, biases8) fc8 = ad.fullyactivation_forward_op(add8, "NCHW", "softmax") loss = ad.l2loss_op(fc8, y_) grad = ad.gradients(loss, [filters1_1, filters1_2, filters2_1, filters2_2, filters3_1, filters3_2, filters3_3 , filters4_1, filters4_2, filters4_3, filters5_1, filters5_2, filters5_3 , filters6, filters7]) executor = ad.Executor([grad[0], grad[1], grad[2], grad[3], grad[4], grad[5], grad[6], grad[7], grad[8], grad[9] , grad[10], grad[11], grad[12], grad[13], grad[14], loss, y_], ctx=ctx) aph = 1.0e-6 for i in range(20): select = random.randint(0, n-1) tmp_x_val = x_val[select] tmp_x_val = np.expand_dims(tmp_x_val, 0) tmp_y_val = y_val[select] tmp_y_val = np.expand_dims(tmp_y_val, 0) grad_val = executor.run( feed_dict={inputs: tmp_x_val, y_: tmp_y_val , filters1_1: filters_val[0], filters1_2: filters_val[1], filters2_1: filters_val[2], filters2_2: filters_val[3] , filters3_1: filters_val[4], filters3_2: filters_val[5], filters3_3: filters_val[6] , filters4_1: filters_val[7], filters4_2: filters_val[8], filters4_3: filters_val[9] , filters5_1: filters_val[10], filters5_2: filters_val[11], filters5_3: filters_val[12] , filters6: filters_val[13], filters7: filters_val[14], filters8: filters_val[15] , biases6: biases_val[0], biases7: biases_val[1], biases8: biases_val[2]}) for i in range(14): sgd_update_gpu(filters_val[i], grad_val[i], aph) print(filters_val[0].asnumpy()) return filters_val
def vgg16(num_epochs=10): n_class = 10 X = ad.Placeholder("inputs") y_ = ad.Placeholder("y_") filters1_1 = ad.Variable("filters1_1") filters1_2 = ad.Variable("filters1_2") filters2_1 = ad.Variable("filters2_1") filters2_2 = ad.Variable("filters2_2") filters3_1 = ad.Variable("filters3_1") filters3_2 = ad.Variable("filters3_2") filters3_3 = ad.Variable("filters3_3") filters4_1 = ad.Variable("filters4_1") filters4_2 = ad.Variable("filters4_2") filters4_3 = ad.Variable("filters4_3") filters5_1 = ad.Variable("filters5_1") filters5_2 = ad.Variable("filters5_2") filters5_3 = ad.Variable("filters5_3") filters6 = ad.Variable("filters6") filters7 = ad.Variable("filters7") filters8 = ad.Variable("filters8") b1_1 = ad.Variable("b1_1") b1_2 = ad.Variable("b1_2") b2_1 = ad.Variable("b2_1") b2_2 = ad.Variable("b2_2") b3_1 = ad.Variable("b3_1") b3_2 = ad.Variable("b3_2") b3_3 = ad.Variable("b3_3") b4_1 = ad.Variable("b4_1") b4_2 = ad.Variable("b4_2") b4_3 = ad.Variable("b4_3") b5_1 = ad.Variable("b5_1") b5_2 = ad.Variable("b5_2") b5_3 = ad.Variable("b5_3") b6 = ad.Variable("b6") b7 = ad.Variable("b7") b8 = ad.Variable("b8") # conv 1 conv1_1 = ad.conv2withbias(X, filters1_1, b1_1, "NCHW", "SAME", 1, 1) bn1_1 = ad.bn_forward_op(conv1_1, "NCHW", "pre_activation") act1_1 = ad.activation_forward_op(bn1_1, "NCHW", "relu") conv1_2 = ad.conv2withbias(act1_1, filters1_2, b1_2, "NCHW", "SAME", 1, 1) bn1_2 = ad.bn_forward_op(conv1_2, "NCHW", "pre_activation") act1_2 = ad.activation_forward_op(bn1_2, "NCHW", "relu") pool1 = ad.pooling_2d_forward_op(act1_2, "NCHW", "max", 0, 0, 2, 2, 2, 2) # conv 2 conv2_1 = ad.conv2withbias(pool1, filters2_1, b2_1, "NCHW", "SAME", 1, 1) bn2_1 = ad.bn_forward_op(conv2_1, "NCHW", "pre_activation") act2_1 = ad.activation_forward_op(bn2_1, "NCHW", "relu") conv2_2 = ad.conv2withbias(act2_1, filters2_2, b2_2, "NCHW", "SAME", 1, 1) bn2_2 = ad.bn_forward_op(conv2_2, "NCHW", "pre_activation") act2_2 = ad.activation_forward_op(bn2_2, "NCHW", "relu") pool2 = ad.pooling_2d_forward_op(act2_2, "NCHW", "max", 0, 0, 2, 2, 2, 2) # conv 3 conv3_1 = ad.conv2withbias(pool2, filters3_1, b3_1, "NCHW", "SAME", 1, 1) bn3_1 = ad.bn_forward_op(conv3_1, "NCHW", "pre_activation") act3_1 = ad.activation_forward_op(bn3_1, "NCHW", "relu") conv3_2 = ad.conv2withbias(act3_1, filters3_2, b3_2, "NCHW", "SAME", 1, 1) bn3_2 = ad.bn_forward_op(conv3_2, "NCHW", "pre_activation") act3_2 = ad.activation_forward_op(bn3_2, "NCHW", "relu") conv3_3 = ad.conv2withbias(act3_2, filters3_3, b3_3, "NCHW", "SAME", 1, 1) bn3_3 = ad.bn_forward_op(conv3_3, "NCHW", "pre_activation") act3_3 = ad.activation_forward_op(bn3_3, "NCHW", "relu") pool3 = ad.pooling_2d_forward_op(act3_3, "NCHW", "max", 0, 0, 2, 2, 2, 2) # conv 4 conv4_1 = ad.conv2withbias(pool3, filters4_1, b4_1, "NCHW", "SAME", 1, 1) bn4_1 = ad.bn_forward_op(conv4_1, "NCHW", "pre_activation") act4_1 = ad.activation_forward_op(bn4_1, "NCHW", "relu") conv4_2 = ad.conv2withbias(act4_1, filters4_2, b4_2, "NCHW", "SAME", 1, 1) bn4_2 = ad.bn_forward_op(conv4_2, "NCHW", "pre_activation") act4_2 = ad.activation_forward_op(bn4_2, "NCHW", "relu") conv4_3 = ad.conv2withbias(act4_2, filters4_3, b4_3, "NCHW", "SAME", 1, 1) bn4_3 = ad.bn_forward_op(conv4_3, "NCHW", "pre_activation") act4_3 = ad.activation_forward_op(bn4_3, "NCHW", "relu") pool4 = ad.pooling_2d_forward_op(act4_3, "NCHW", "max", 0, 0, 2, 2, 2, 2) # conv 5 conv5_1 = ad.conv2withbias(pool4, filters5_1, b5_1, "NCHW", "SAME", 1, 1) bn5_1 = ad.bn_forward_op(conv5_1, "NCHW", "pre_activation") act5_1 = ad.activation_forward_op(bn5_1, "NCHW", "relu") conv5_2 = ad.conv2withbias(act5_1, filters5_2, b5_2, "NCHW", "SAME", 1, 1) bn5_2 = ad.bn_forward_op(conv5_2, "NCHW", "pre_activation") act5_2 = ad.activation_forward_op(bn5_2, "NCHW", "relu") conv5_3 = ad.conv2withbias(act5_2, filters5_3, b5_3, "NCHW", "SAME", 1, 1) bn5_3 = ad.bn_forward_op(conv5_3, "NCHW", "pre_activation") act5_3 = ad.activation_forward_op(bn5_3, "NCHW", "relu") pool5 = ad.pooling_2d_forward_op(act5_3, "NCHW", "max", 0, 0, 2, 2, 2, 2) # fc6 pool5_flat = ad.flatten_op(pool5) fc6 = ad.dense(pool5_flat, filters6, b6) bn6 = ad.fullybn_forward_op(fc6, "NCHW") act6 = ad.fullyactivation_forward_op(bn6, "NCHW", "relu") drop6 = ad.fullydropout_forward_op(act6, "NCHW", 0.5) # fc7 fc7 = ad.dense(drop6, filters7, b7) bn7 = ad.fullybn_forward_op(fc7, "NCHW") act7 = ad.fullyactivation_forward_op(bn7, "NCHW", "relu") drop7 = ad.fullydropout_forward_op(act7, "NCHW", 0.5) #fc8 fc8 = ad.dense(drop7, filters8, b8) bn8 = ad.fullybn_forward_op(fc8, "NCHW") act8 = ad.fullyactivation_forward_op(bn8, "NCHW", "softmax") loss = ad.softmaxcrossentropy_op(bn8, y_) # grad = ad.gradients(loss, [filters1_1, filters1_2, filters2_1, filters2_2, filters3_1, filters3_2, filters3_3 # , filters4_1, filters4_2, filters4_3, filters5_1, filters5_2, filters5_3 # , filters6, filters7]) # executor = ad.Executor([grad[0], grad[1], grad[2], grad[3], grad[4], grad[5], grad[6], grad[7], grad[8], grad[9] # , grad[10], grad[11], grad[12], grad[13], grad[14], loss, y_], ctx=ctx) train_x, train_y, test_x, test_y = prepare_data() n_train_batches = train_x.shape[0] // batch_size n_test_batches = test_x.shape[0] // batch_size X_val = np.empty(shape=(batch_size, 3, 32, 32), dtype=np.float32) y_val = np.empty(shape=(batch_size, n_class), dtype=np.float32) filters_val = [np.random.normal(0.0, 0.1, (64, 3, 3, 3))] filters_val.append(np.random.normal(0.0, 0.1, (64, 64, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (128, 64, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (128, 128, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (256, 128, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (256, 256, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (256, 256, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (512, 256, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (512, 512, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (512, 512, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (512, 512, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (512, 512, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (512, 512, 3, 3))) filters_val.append(np.random.normal(0.0, 0.1, (512 * 1 * 1, 4096))) filters_val.append(np.random.normal(0.0, 0.1, (4096, 4096)) * 0.001) filters_val.append(np.random.normal(0.0, 0.1, (4096, n_class)) * 0.001) b_val = [np.ones(64) * 0.1] b_val.append(np.ones(64) * 0.1) b_val.append(np.ones(128) * 0.1) b_val.append(np.ones(128) * 0.1) b_val.append(np.ones(256) * 0.1) b_val.append(np.ones(256) * 0.1) b_val.append(np.ones(256) * 0.1) b_val.append(np.ones(512) * 0.1) b_val.append(np.ones(512) * 0.1) b_val.append(np.ones(512) * 0.1) b_val.append(np.ones(512) * 0.1) b_val.append(np.ones(512) * 0.1) b_val.append(np.ones(512) * 0.1) b_val.append(np.ones(4096) * 0.1) b_val.append(np.ones(4096) * 0.1) b_val.append(np.ones(n_class) * 0.1) # ctx = ndarray.gpu(0) # for i in range(16): # filters_val[i] = ndarray.array(filters_val[i], ctx) # for i in range(16): # b_val[i] = ndarray.array(b_val[i], ctx) aph = 0.0001 t = train.Adam_minimize(loss, aph) t.init_Variable({ filters1_1: filters_val[0], filters1_2: filters_val[1], filters2_1: filters_val[2], filters2_2: filters_val[3], filters3_1: filters_val[4], filters3_2: filters_val[5], filters3_3: filters_val[6], filters4_1: filters_val[7], filters4_2: filters_val[8], filters4_3: filters_val[9], filters5_1: filters_val[10], filters5_2: filters_val[11], filters5_3: filters_val[12], filters6: filters_val[13], filters7: filters_val[14], filters8: filters_val[15], b1_1: b_val[0], b1_2: b_val[1], b2_1: b_val[2], b2_2: b_val[3], b3_1: b_val[4], b3_2: b_val[5], b3_3: b_val[6], b4_1: b_val[7], b4_2: b_val[8], b4_3: b_val[9], b5_1: b_val[10], b5_2: b_val[11], b5_3: b_val[12], b6: b_val[13], b7: b_val[14], b8: b_val[15] }) for i in range(num_epochs): print("epoch %d" % i) X_val = np.empty(shape=(0, 3, 32, 32), dtype=np.float32) y_val = np.empty(shape=(0, n_class), dtype=np.float32) select = random.randint(0, n_train_batches) # X_val = train_x[select] # y_val = train_y[select] for j in range(batch_size): select = random.randint(0, n_train_batches) temp_train_x = train_x[select] temp_train_x = temp_train_x[np.newaxis, :] temp_train_y = train_y[select] temp_train_y = temp_train_y[np.newaxis, :] X_val = np.append(X_val, temp_train_x, axis=0) y_val = np.append(y_val, temp_train_y, axis=0) # print("train", X_val.shape) # print("train", y_val.shape) t.run({X: X_val, y_: y_val}) if (i + 1) % 30 == 0: X_test_val = np.empty(shape=(100, 3, 32, 32), dtype=np.float32) y_test_val = np.empty(shape=(100, n_class), dtype=np.float32) correct_predictions = [] for minibatch_index in range(n_test_batches): minibatch_start = minibatch_index * batch_size minibatch_end = (minibatch_index + 1) * batch_size X_test_val[:] = test_x[minibatch_start:minibatch_end] y_test_val[:] = test_y[minibatch_start:minibatch_end] # print(X_test_val.shape) # print(y_test_val.shape) feed_dict = {X: X_test_val, y_: y_test_val} dict_Variable = t.get_Variable_node_to_val_map() feed_dict.update(dict_Variable) y_predicted = t.run_get_nodelist_once(feed_dict, [act8])[act8].asnumpy() correct_prediction = np.equal(np.argmax(y_test_val, 1), np.argmax(y_predicted, 1)).astype(np.float) correct_predictions.extend(correct_prediction) accuracy = np.mean(correct_predictions) print("validation set accuracy=%f" % accuracy) return filters_val