Esempio n. 1
0
 def ff(self, x, phase):
     self.dropmask = owl.randb(x.shape, self.keep_ratio)
     if phase == "TRAIN":
         return ele.mult(x, self.dropmask)*self.scale
     else:
         return x
     '''
Esempio n. 2
0
 def ff(self, x, phase):
     self.dropmask = owl.randb(x.shape, self.keep_ratio)
     if phase == "TRAIN":
         return ele.mult(x, self.dropmask) * self.scale
     else:
         #return x * (1 - self.params.dropout_param.dropout_ratio)
         return x
     '''
Esempio n. 3
0
 def ff(self, x, phase):
     ''' Foward function of dropout
     
     The dropout mask will not be multiplied if under ``"TEST"`` mode.
     '''
     if phase == "TRAIN":
         self.dropmask = owl.randb(x.shape, self.keep_ratio)
         return ele.mult(x, self.dropmask) * self.scale
     else:
         return x
Esempio n. 4
0
 def ff(self, x, phase):
     ''' Foward function of dropout
     
     The dropout mask will not be multiplied if under ``"TEST"`` mode.
     '''
     self.dropmask = owl.randb(x.shape, self.keep_ratio)
     if phase == "TRAIN":
         return ele.mult(x, self.dropmask)*self.scale
     else:
         return x
Esempio n. 5
0
def train_network(model,
                  num_epochs=100,
                  minibatch_size=256,
                  dropout_rate=0.5,
                  eps_w=0.01,
                  eps_b=0.01,
                  mom=0.9,
                  wd=0.0005):
    gpu = owl.create_gpu_device(1)
    owl.set_device(gpu)
    num_layers = 20
    count = 0
    last = time.time()

    dp = ImageNetDataProvider(
        mean_file='/home/minjie/data/imagenet/imagenet_mean.binaryproto',
        train_db='/home/minjie/data/imagenet/ilsvrc12_train_lmdb',
        val_db='/home/minjie/data/imagenet/ilsvrc12_val_lmdb',
        test_db='/home/minjie/data/imagenet/ilsvrc12_test_lmdb')

    acts = [None] * num_layers
    sens = [None] * num_layers

    for i in xrange(num_epochs):
        print "---------------------Epoch #", i
        sys.stdout.flush()
        for (samples, labels) in dp.get_train_mb(minibatch_size):
            num_samples = samples.shape[0]

            acts = [None] * num_layers
            sens = [None] * num_layers

            # FF
            acts[0] = owl.from_nparray(samples).reshape(
                [227, 227, 3, num_samples])
            target = owl.from_nparray(labels)

            acts1 = conv_forward(acts[0], model.weights[0], model.bias[0],
                                 model.conv_infos[0])
            acts[1] = ele.relu(
                acts1
            )  #(conv_forward(acts[0], model.weights[0], model.bias[0], model.conv_infos[0])) # conv1
            acts[2] = pooling_forward(acts[1], model.pooling_infos[0])  # pool1
            acts3 = conv_forward(acts[2], model.weights[1], model.bias[1],
                                 model.conv_infos[1])  # conv2
            acts[3] = ele.relu(
                acts3
            )  #(conv_forward(acts[2], model.weights[1], model.bias[1], model.conv_infos[1])) # conv2
            acts[4] = pooling_forward(acts[3], model.pooling_infos[1])  # pool2
            acts5 = conv_forward(acts[4], model.weights[2], model.bias[2],
                                 model.conv_infos[2])  # conv3
            acts[5] = ele.relu(
                acts5
            )  #(conv_forward(acts[4], model.weights[2], model.bias[2], model.conv_infos[2])) # conv3
            acts6 = conv_forward(acts[5], model.weights[3], model.bias[3],
                                 model.conv_infos[3])  # conv4
            acts[6] = ele.relu(
                acts6
            )  #(conv_forward(acts[5], model.weights[3], model.bias[3], model.conv_infos[3])) # conv4
            acts7 = conv_forward(acts[6], model.weights[4], model.bias[4],
                                 model.conv_infos[4])  # conv5
            acts[7] = ele.relu(
                acts7
            )  #(conv_forward(acts[6], model.weights[4], model.bias[4], model.conv_infos[4])) # conv5
            acts[8] = pooling_forward(acts[7], model.pooling_infos[2])  # pool5
            re_acts8 = acts[8].reshape(
                [np.prod(acts[8].shape[0:3]), num_samples])
            acts9 = model.weights[5] * re_acts8 + model.bias[5]  # fc6
            acts[9] = ele.relu(
                acts9)  #(model.weights[5] * re_acts8 + model.bias[5]) # fc6
            mask6 = owl.randb(acts[9].shape, dropout_rate)
            acts[9] = ele.mult(acts[9], mask6)  # drop6
            acts10 = model.weights[6] * acts[9] + model.bias[6]  # fc7
            acts[10] = ele.relu(
                acts10)  #(model.weights[6] * acts[9] + model.bias[6]) # fc7
            mask7 = owl.randb(acts[10].shape, dropout_rate)
            acts[10] = ele.mult(acts[10], mask7)  # drop7
            acts[11] = model.weights[7] * acts[10] + model.bias[7]  # fc8
            acts[12] = softmax_forward(
                acts[11].reshape([1000, 1, 1, num_samples]),
                soft_op.instance).reshape([1000, num_samples])  # prob

            # error
            sens[11] = acts[12] - target

            # BP
            sens[10] = model.weights[7].trans() * sens[11]  # fc8
            sens[10] = ele.mult(sens[10], mask7)  # drop7
            sens[10] = ele.relu_back(sens[10], acts[10], acts10)  # relu7
            sens[9] = model.weights[6].trans() * sens[10]
            sens[9] = ele.mult(sens[9], mask6)  # drop6
            sens[9] = ele.relu_back(sens[9], acts[9], acts9)  # relu6
            sens[8] = (model.weights[5].trans() * sens[9]).reshape(
                acts[8].shape)  # fc6
            sens[7] = pooling_backward(sens[8], acts[8], acts[7],
                                       model.pooling_infos[2])  # pool5
            sens[7] = ele.relu_back(sens[7], acts[7], acts7)  # relu5
            sens[6] = conv_backward_data(sens[7], model.weights[4],
                                         model.conv_infos[4])  # conv5
            sens[6] = ele.relu_back(sens[6], acts[6], acts6)  # relu4
            sens[5] = conv_backward_data(sens[6], model.weights[3],
                                         model.conv_infos[3])  # conv4
            sens[5] = ele.relu_back(sens[5], acts[5], acts5)  # relu3
            sens[4] = conv_backward_data(sens[5], model.weights[2],
                                         model.conv_infos[2])  # conv3
            sens[3] = pooling_backward(sens[4], acts[4], acts[3],
                                       model.pooling_infos[1])  # pool2
            sens[3] = ele.relu_back(sens[3], acts[3], acts3)  # relu2
            sens[2] = conv_backward_data(sens[3], model.weights[1],
                                         model.conv_infos[1])  # conv2
            sens[1] = pooling_backward(sens[2], acts[2], acts[1],
                                       model.pooling_infos[0])  # pool1
            sens[1] = ele.relu_back(sens[1], acts[1], acts1)  # relu1

            model.weightsdelta[
                7] = mom * model.weightsdelta[7] - eps_w / num_samples * (
                    sens[11] * acts[10].trans() + wd * model.weights[7])
            model.biasdelta[7] = mom * model.biasdelta[
                7] - eps_b / num_samples * sens[11].sum(1)

            model.weightsdelta[
                6] = mom * model.weightsdelta[6] - eps_w / num_samples * (
                    sens[10] * acts[9].trans() + wd * model.weights[6])
            model.biasdelta[6] = mom * model.biasdelta[
                6] - eps_b / num_samples * sens[10].sum(1)

            model.weightsdelta[
                5] = mom * model.weightsdelta[5] - eps_w / num_samples * (
                    sens[9] * re_acts8.trans() + wd * model.weights[5])
            model.biasdelta[5] = mom * model.biasdelta[
                5] - eps_b / num_samples * sens[9].sum(1)

            model.weightsdelta[
                4] = mom * model.weightsdelta[4] - eps_w / num_samples * (
                    conv_backward_filter(sens[7], acts[6], model.conv_infos[4])
                    + wd * model.weights[4])
            model.biasdelta[4] = mom * model.biasdelta[
                4] - eps_b / num_samples * conv_backward_bias(sens[7])

            model.weightsdelta[
                3] = mom * model.weightsdelta[3] - eps_w / num_samples * (
                    conv_backward_filter(sens[6], acts[5], model.conv_infos[3])
                    + wd * model.weights[3])
            model.biasdelta[3] = mom * model.biasdelta[
                3] - eps_b / num_samples * conv_backward_bias(sens[6])

            model.weightsdelta[
                2] = mom * model.weightsdelta[2] - eps_w / num_samples * (
                    conv_backward_filter(sens[5], acts[4], model.conv_infos[2])
                    + wd * model.weights[2])
            model.biasdelta[2] = mom * model.biasdelta[
                2] - eps_b / num_samples * conv_backward_bias(sens[5])

            model.weightsdelta[
                1] = mom * model.weightsdelta[1] - eps_w / num_samples * (
                    conv_backward_filter(sens[3], acts[2], model.conv_infos[1])
                    + wd * model.weights[1])
            model.biasdelta[1] = mom * model.biasdelta[
                1] - eps_b / num_samples * conv_backward_bias(sens[3])

            model.weightsdelta[
                0] = mom * model.weightsdelta[0] - eps_w / num_samples * (
                    conv_backward_filter(sens[1], acts[0], model.conv_infos[0])
                    + wd * model.weights[0])
            model.biasdelta[0] = mom * model.biasdelta[
                0] - eps_b / num_samples * conv_backward_bias(sens[1])

            for k in range(8):
                model.weights[k] += model.weightsdelta[k]
                model.bias[k] += model.biasdelta[k]

            count = count + 1
            if count % 10 == 0:
                print_training_accuracy(acts[12], target, num_samples)
                print "time: %s" % (time.time() - last)
                last = time.time()
Esempio n. 6
0
def train_one_mb(model, data, label, weightsgrad, biasgrad, dropout_rate):
    num_samples = data.shape[-1]
    num_layers = 20
    acts = [None] * num_layers
    sens = [None] * num_layers
    # FF
    acts[0] = data
    acts1 = conv_forward(acts[0], model.weights[0], model.bias[0], model.conv_infos[0])
    acts[1] = ele.relu(acts1)#(conv_forward(acts[0], model.weights[0], model.bias[0], model.conv_infos[0])) # conv1
    acts[2] = pooling_forward(acts[1], model.pooling_infos[0]) # pool1
    acts3 = conv_forward(acts[2], model.weights[1], model.bias[1], model.conv_infos[1]) # conv2
    acts[3] = ele.relu(acts3)#(conv_forward(acts[2], model.weights[1], model.bias[1], model.conv_infos[1])) # conv2
    acts[4] = pooling_forward(acts[3], model.pooling_infos[1]) # pool2
    acts5 = conv_forward(acts[4], model.weights[2], model.bias[2], model.conv_infos[2]) # conv3
    acts[5] = ele.relu(acts5)#(conv_forward(acts[4], model.weights[2], model.bias[2], model.conv_infos[2])) # conv3
    acts6 = conv_forward(acts[5], model.weights[3], model.bias[3], model.conv_infos[3]) # conv4
    acts[6] = ele.relu(acts6)#(conv_forward(acts[5], model.weights[3], model.bias[3], model.conv_infos[3])) # conv4
    acts7 = conv_forward(acts[6], model.weights[4], model.bias[4], model.conv_infos[4]) # conv5
    acts[7] = ele.relu(acts7)#(conv_forward(acts[6], model.weights[4], model.bias[4], model.conv_infos[4])) # conv5
    acts[8] = pooling_forward(acts[7], model.pooling_infos[2]) # pool5
    re_acts8 = acts[8].reshape([np.prod(acts[8].shape[0:3]), num_samples])
    acts9 = model.weights[5] * re_acts8 + model.bias[5] # fc6
    acts[9] = ele.relu(acts9)#(model.weights[5] * re_acts8 + model.bias[5]) # fc6
    mask6 = owl.randb(acts[9].shape, dropout_rate)
    acts[9] = ele.mult(acts[9], mask6) # drop6
    acts10 = model.weights[6] * acts[9] + model.bias[6] # fc7
    acts[10] = ele.relu(acts10)#(model.weights[6] * acts[9] + model.bias[6]) # fc7
    mask7 = owl.randb(acts[10].shape, dropout_rate)
    acts[10] = ele.mult(acts[10], mask7) # drop7
    acts[11] = model.weights[7] * acts[10] + model.bias[7] # fc8
    acts[12] = softmax_forward(acts[11].reshape([1000, 1, 1, num_samples]), soft_op.instance).reshape([1000, num_samples]) # prob
    # error
    sens[11] = acts[12] - label
    # BP
    sens[10] = model.weights[7].trans() * sens[11] # fc8
    sens[10] = ele.mult(sens[10], mask7) # drop7
    sens[10] = ele.relu_back(sens[10], acts[10]) # relu7
    sens[9] = model.weights[6].trans() * sens[10]
    sens[9] = ele.mult(sens[9], mask6) # drop6
    sens[9] = ele.relu_back(sens[9], acts[9]) # relu6
    sens[8] = (model.weights[5].trans() * sens[9]).reshape(acts[8].shape) # fc6
    sens[7] = pooling_backward(sens[8], acts[8], acts[7], model.pooling_infos[2]) # pool5
    sens[7] = ele.relu_back(sens[7], acts[7]) # relu5
    sens[6] = conv_backward_data(sens[7], model.weights[4], model.conv_infos[4]) # conv5
    sens[6] = ele.relu_back(sens[6], acts[6]) # relu4
    sens[5] = conv_backward_data(sens[6], model.weights[3], model.conv_infos[3]) # conv4
    sens[5] = ele.relu_back(sens[5], acts[5]) # relu3
    sens[4] = conv_backward_data(sens[5], model.weights[2], model.conv_infos[2]) # conv3
    sens[3] = pooling_backward(sens[4], acts[4], acts[3], model.pooling_infos[1]) # pool2
    sens[3] = ele.relu_back(sens[3], acts[3]) # relu2
    sens[2] = conv_backward_data(sens[3], model.weights[1], model.conv_infos[1]) # conv2
    sens[1] = pooling_backward(sens[2], acts[2], acts[1], model.pooling_infos[0]) # pool1
    sens[1] = ele.relu_back(sens[1], acts[1]) # relu1
    weightsgrad[7] = sens[11] * acts[10].trans()
    weightsgrad[6] = sens[10] * acts[9].trans()
    weightsgrad[5] = sens[9] * re_acts8.trans()
    weightsgrad[4] = conv_backward_filter(sens[7], acts[6], model.conv_infos[4])
    weightsgrad[3] = conv_backward_filter(sens[6], acts[5], model.conv_infos[3])
    weightsgrad[2] = conv_backward_filter(sens[5], acts[4], model.conv_infos[2])
    weightsgrad[1] = conv_backward_filter(sens[3], acts[2], model.conv_infos[1])
    weightsgrad[0] = conv_backward_filter(sens[1], acts[0], model.conv_infos[0])
    biasgrad[7] = sens[11].sum(1)
    biasgrad[6] = sens[10].sum(1)
    biasgrad[5] = sens[9].sum(1)
    biasgrad[4] = conv_backward_bias(sens[7])
    biasgrad[3] = conv_backward_bias(sens[6])
    biasgrad[2] = conv_backward_bias(sens[5])
    biasgrad[1] = conv_backward_bias(sens[3])
    biasgrad[0] = conv_backward_bias(sens[1])
    return acts[12]
Esempio n. 7
0
def train_one_mb(model, data, label, weightsgrad, biasgrad, dropout_rate):
    num_samples = data.shape[-1]
    num_layers = 20
    acts = [None] * num_layers
    sens = [None] * num_layers
    # FF
    acts[0] = data
    acts1 = conv_forward(acts[0], model.weights[0], model.bias[0],
                         model.conv_infos[0])
    acts[1] = ele.relu(
        acts1
    )  #(conv_forward(acts[0], model.weights[0], model.bias[0], model.conv_infos[0])) # conv1
    acts[2] = pooling_forward(acts[1], model.pooling_infos[0])  # pool1
    acts3 = conv_forward(acts[2], model.weights[1], model.bias[1],
                         model.conv_infos[1])  # conv2
    acts[3] = ele.relu(
        acts3
    )  #(conv_forward(acts[2], model.weights[1], model.bias[1], model.conv_infos[1])) # conv2
    acts[4] = pooling_forward(acts[3], model.pooling_infos[1])  # pool2
    acts5 = conv_forward(acts[4], model.weights[2], model.bias[2],
                         model.conv_infos[2])  # conv3
    acts[5] = ele.relu(
        acts5
    )  #(conv_forward(acts[4], model.weights[2], model.bias[2], model.conv_infos[2])) # conv3
    acts6 = conv_forward(acts[5], model.weights[3], model.bias[3],
                         model.conv_infos[3])  # conv4
    acts[6] = ele.relu(
        acts6
    )  #(conv_forward(acts[5], model.weights[3], model.bias[3], model.conv_infos[3])) # conv4
    acts7 = conv_forward(acts[6], model.weights[4], model.bias[4],
                         model.conv_infos[4])  # conv5
    acts[7] = ele.relu(
        acts7
    )  #(conv_forward(acts[6], model.weights[4], model.bias[4], model.conv_infos[4])) # conv5
    acts[8] = pooling_forward(acts[7], model.pooling_infos[2])  # pool5
    re_acts8 = acts[8].reshape([np.prod(acts[8].shape[0:3]), num_samples])
    acts9 = model.weights[5] * re_acts8 + model.bias[5]  # fc6
    acts[9] = ele.relu(
        acts9)  #(model.weights[5] * re_acts8 + model.bias[5]) # fc6
    mask6 = owl.randb(acts[9].shape, dropout_rate)
    acts[9] = ele.mult(acts[9], mask6)  # drop6
    acts10 = model.weights[6] * acts[9] + model.bias[6]  # fc7
    acts[10] = ele.relu(
        acts10)  #(model.weights[6] * acts[9] + model.bias[6]) # fc7
    mask7 = owl.randb(acts[10].shape, dropout_rate)
    acts[10] = ele.mult(acts[10], mask7)  # drop7
    acts[11] = model.weights[7] * acts[10] + model.bias[7]  # fc8
    acts[12] = softmax_forward(acts[11].reshape([1000, 1, 1, num_samples]),
                               soft_op.instance).reshape([1000,
                                                          num_samples])  # prob

    # error
    sens[11] = acts[12] - label

    # BP
    sens[10] = model.weights[7].trans() * sens[11]  # fc8
    sens[10] = ele.mult(sens[10], mask7)  # drop7
    sens[10] = ele.relu_back(sens[10], acts[10], acts10)  # relu7
    sens[9] = model.weights[6].trans() * sens[10]
    sens[9] = ele.mult(sens[9], mask6)  # drop6
    sens[9] = ele.relu_back(sens[9], acts[9], acts9)  # relu6
    sens[8] = (model.weights[5].trans() * sens[9]).reshape(
        acts[8].shape)  # fc6
    sens[7] = pooling_backward(sens[8], acts[8], acts[7],
                               model.pooling_infos[2])  # pool5
    sens[7] = ele.relu_back(sens[7], acts[7], acts7)  # relu5
    sens[6] = conv_backward_data(sens[7], model.weights[4],
                                 model.conv_infos[4])  # conv5
    sens[6] = ele.relu_back(sens[6], acts[6], acts6)  # relu4
    sens[5] = conv_backward_data(sens[6], model.weights[3],
                                 model.conv_infos[3])  # conv4
    sens[5] = ele.relu_back(sens[5], acts[5], acts5)  # relu3
    sens[4] = conv_backward_data(sens[5], model.weights[2],
                                 model.conv_infos[2])  # conv3
    sens[3] = pooling_backward(sens[4], acts[4], acts[3],
                               model.pooling_infos[1])  # pool2
    sens[3] = ele.relu_back(sens[3], acts[3], acts3)  # relu2
    sens[2] = conv_backward_data(sens[3], model.weights[1],
                                 model.conv_infos[1])  # conv2
    sens[1] = pooling_backward(sens[2], acts[2], acts[1],
                               model.pooling_infos[0])  # pool1
    sens[1] = ele.relu_back(sens[1], acts[1], acts1)  # relu1

    weightsgrad[7] = sens[11] * acts[10].trans()
    weightsgrad[6] = sens[10] * acts[9].trans()
    weightsgrad[5] = sens[9] * re_acts8.trans()
    weightsgrad[4] = conv_backward_filter(sens[7], acts[6],
                                          model.conv_infos[4])
    weightsgrad[3] = conv_backward_filter(sens[6], acts[5],
                                          model.conv_infos[3])
    weightsgrad[2] = conv_backward_filter(sens[5], acts[4],
                                          model.conv_infos[2])
    weightsgrad[1] = conv_backward_filter(sens[3], acts[2],
                                          model.conv_infos[1])
    weightsgrad[0] = conv_backward_filter(sens[1], acts[0],
                                          model.conv_infos[0])
    biasgrad[7] = sens[11].sum(1)
    biasgrad[6] = sens[10].sum(1)
    biasgrad[5] = sens[9].sum(1)
    biasgrad[4] = conv_backward_bias(sens[7])
    biasgrad[3] = conv_backward_bias(sens[6])
    biasgrad[2] = conv_backward_bias(sens[5])
    biasgrad[1] = conv_backward_bias(sens[3])
    biasgrad[0] = conv_backward_bias(sens[1])
    return acts[12]
Esempio n. 8
0
def train_network(model, num_epochs = 100, minibatch_size=256,
        dropout_rate = 0.5, eps_w = 0.01, eps_b = 0.01, mom = 0.9, wd = 0.0005):
    gpu = owl.create_gpu_device(1)
    owl.set_device(gpu)
    num_layers = 20
    count = 0
    last = time.time()

    dp = ImageNetDataProvider(mean_file='/home/minjie/data/imagenet/imagenet_mean.binaryproto',
            train_db='/home/minjie/data/imagenet/ilsvrc12_train_lmdb',
            val_db='/home/minjie/data/imagenet/ilsvrc12_val_lmdb',
            test_db='/home/minjie/data/imagenet/ilsvrc12_test_lmdb')

    acts = [None] * num_layers
    sens = [None] * num_layers

    for i in xrange(num_epochs):
        print "---------------------Epoch #", i
        sys.stdout.flush()
        for (samples, labels) in dp.get_train_mb(minibatch_size):
            num_samples = samples.shape[0]

            acts = [None] * num_layers
            sens = [None] * num_layers

            '''
            thisimg = samples[0, :]
            print thisimg
            imgdata = np.transpose(thisimg.reshape([3, 227*227])).reshape([227, 227, 3])
            print imgdata
            img = Image.fromarray(imgdata.astype(np.uint8))
            img.save('testimg.jpg', format='JPEG')
            exit(0)
            '''

            # FF
            acts[0] = owl.from_nparray(samples).reshape([227, 227, 3, num_samples])
            #print np.array(acts[0].tolist())[0:227*227*3]

            target = owl.from_nparray(labels)

            #np.set_printoptions(linewidth=200)
            #print acts[0].shape, model.weights[0].shape, model.bias[0].shape
            #im = np.array(acts[0].tolist()).reshape([num_samples, 227, 227, 3])
            #print im[0,:,:,0]
            #print im[0,:,:,1]
            #print im[0,:,:,2]
            #print target.max_index(0).tolist()[0:20]
            #sys.exit()

            acts1 = conv_forward(acts[0], model.weights[0], model.bias[0], model.conv_infos[0])
            acts[1] = ele.relu(acts1)#(conv_forward(acts[0], model.weights[0], model.bias[0], model.conv_infos[0])) # conv1
            acts[2] = pooling_forward(acts[1], model.pooling_infos[0]) # pool1
            acts3 = conv_forward(acts[2], model.weights[1], model.bias[1], model.conv_infos[1]) # conv2
            acts[3] = ele.relu(acts3)#(conv_forward(acts[2], model.weights[1], model.bias[1], model.conv_infos[1])) # conv2
            acts[4] = pooling_forward(acts[3], model.pooling_infos[1]) # pool2
            acts5 = conv_forward(acts[4], model.weights[2], model.bias[2], model.conv_infos[2]) # conv3
            acts[5] = ele.relu(acts5)#(conv_forward(acts[4], model.weights[2], model.bias[2], model.conv_infos[2])) # conv3
            acts6 = conv_forward(acts[5], model.weights[3], model.bias[3], model.conv_infos[3]) # conv4
            acts[6] = ele.relu(acts6)#(conv_forward(acts[5], model.weights[3], model.bias[3], model.conv_infos[3])) # conv4
            acts7 = conv_forward(acts[6], model.weights[4], model.bias[4], model.conv_infos[4]) # conv5
            acts[7] = ele.relu(acts7)#(conv_forward(acts[6], model.weights[4], model.bias[4], model.conv_infos[4])) # conv5
            acts[8] = pooling_forward(acts[7], model.pooling_infos[2]) # pool5
            re_acts8 = acts[8].reshape([np.prod(acts[8].shape[0:3]), num_samples])
            acts9 = model.weights[5] * re_acts8 + model.bias[5] # fc6
            acts[9] = ele.relu(acts9)#(model.weights[5] * re_acts8 + model.bias[5]) # fc6
            mask6 = owl.randb(acts[9].shape, dropout_rate)
            acts[9] = ele.mult(acts[9], mask6) # drop6
            acts10 = model.weights[6] * acts[9] + model.bias[6] # fc7
            acts[10] = ele.relu(acts10)#(model.weights[6] * acts[9] + model.bias[6]) # fc7
            mask7 = owl.randb(acts[10].shape, dropout_rate)
            acts[10] = ele.mult(acts[10], mask7) # drop7
            acts[11] = model.weights[7] * acts[10] + model.bias[7] # fc8
            acts[12] = softmax_forward(acts[11].reshape([1000, 1, 1, num_samples]), soft_op.instance).reshape([1000, num_samples]) # prob

            # error
            sens[11] = acts[12] - target

            # BP
            sens[10] = model.weights[7].trans() * sens[11] # fc8
            sens[10] = ele.mult(sens[10], mask7) # drop7
            sens[10] = ele.relu_back(sens[10], acts[10], acts10) # relu7
            sens[9] = model.weights[6].trans() * sens[10]
            sens[9] = ele.mult(sens[9], mask6) # drop6
            sens[9] = ele.relu_back(sens[9], acts[9], acts9) # relu6
            sens[8] = (model.weights[5].trans() * sens[9]).reshape(acts[8].shape) # fc6
            sens[7] = pooling_backward(sens[8], acts[8], acts[7], model.pooling_infos[2]) # pool5
            sens[7] = ele.relu_back(sens[7], acts[7], acts7) # relu5
            sens[6] = conv_backward_data(sens[7], model.weights[4], model.conv_infos[4]) # conv5
            sens[6] = ele.relu_back(sens[6], acts[6], acts6) # relu4
            sens[5] = conv_backward_data(sens[6], model.weights[3], model.conv_infos[3]) # conv4
            sens[5] = ele.relu_back(sens[5], acts[5], acts5) # relu3
            sens[4] = conv_backward_data(sens[5], model.weights[2], model.conv_infos[2]) # conv3
            sens[3] = pooling_backward(sens[4], acts[4], acts[3], model.pooling_infos[1]) # pool2
            sens[3] = ele.relu_back(sens[3], acts[3], acts3) # relu2
            sens[2] = conv_backward_data(sens[3], model.weights[1], model.conv_infos[1]) # conv2
            sens[1] = pooling_backward(sens[2], acts[2], acts[1], model.pooling_infos[0]) # pool1
            sens[1] = ele.relu_back(sens[1], acts[1], acts1) # relu1

	    model.weightsdelta[7] = mom * model.weightsdelta[7] - eps_w / num_samples  * (sens[11] * acts[10].trans() + wd * model.weights[7])
            model.biasdelta[7] = mom * model.biasdelta[7] - eps_b / num_samples  * (sens[11].sum(1) + wd * model.bias[7])
            
	    model.weightsdelta[6] = mom * model.weightsdelta[6] - eps_w / num_samples  * (sens[10] * acts[9].trans() + wd * model.weights[6])
            model.biasdelta[6] = mom * model.biasdelta[6] - eps_b / num_samples  * (sens[10].sum(1) + wd * model.bias[6])
    	    
	    model.weightsdelta[5] = mom * model.weightsdelta[5] - eps_w / num_samples  * (sens[9] * re_acts8.trans() + wd * model.weights[5])
            model.biasdelta[5] = mom * model.biasdelta[5] - eps_b / num_samples  * (sens[9].sum(1) + wd * model.bias[5])
            	
            model.weightsdelta[4] = mom * model.weightsdelta[4] - eps_w / num_samples  * (conv_backward_filter(sens[7], acts[6], model.conv_infos[4]) + wd * model.weights[4])
	    model.biasdelta[4] = mom * model.biasdelta[4] - eps_b / num_samples  * (conv_backward_bias(sens[7]) + wd * model.bias[4])

	    model.weightsdelta[3] = mom * model.weightsdelta[3] - eps_w / num_samples  * (conv_backward_filter(sens[6], acts[5], model.conv_infos[3]) + wd * model.weights[3])
	    model.biasdelta[3] = mom * model.biasdelta[3] - eps_b / num_samples  * (conv_backward_bias(sens[6]) + wd * model.bias[3])

 	    model.weightsdelta[2] = mom * model.weightsdelta[2] - eps_w / num_samples  * (conv_backward_filter(sens[5], acts[4], model.conv_infos[2]) + wd * model.weights[2])
	    model.biasdelta[2] = mom * model.biasdelta[2] - eps_b / num_samples  * (conv_backward_bias(sens[5]) + wd * model.bias[2])

  	    model.weightsdelta[1] = mom * model.weightsdelta[1] - eps_w / num_samples  * (conv_backward_filter(sens[3], acts[2], model.conv_infos[1]) + wd * model.weights[1])
	    model.biasdelta[1] = mom * model.biasdelta[1] - eps_b / num_samples  * (conv_backward_bias(sens[3]) + wd * model.bias[1])

            model.weightsdelta[0] = mom * model.weightsdelta[0] - eps_w / num_samples  * (conv_backward_filter(sens[1], acts[0], model.conv_infos[0]) + wd * model.weights[0])
	    model.biasdelta[0] = mom * model.biasdelta[0] - eps_b / num_samples  * (conv_backward_bias(sens[1]) + wd * model.bias[0])

            for k in range(8):
                model.weights[k] += model.weightsdelta[k]
                model.bias[k] += model.biasdelta[k]

            count = count + 1
            #if count % 2 == 0:
                #acts[18].start_eval()
            if count % 10 == 0:
                print_training_accuracy(acts[12], target, num_samples)
                print "time: %s" % (time.time() - last)
                last = time.time()
Esempio n. 9
0
 def ff(self, x):
     self.dropmask = owl.randb(x, self.params.dropout_param.dropout_ratio)
     return ele.mult(x, self.dropmask)
Esempio n. 10
0
    def train_one_mb(self, data, label, dropout_rate):
        num_samples = data.shape[-1]
        num_layers = 12
        acts = [None] * num_layers
        sens = [None] * num_layers
        weightsgrad = [None] * self.num_weights
        biasgrad = [None] * self.num_weights

        # FF
        acts[0] = data
        acts[1] = ele.relu(self.convs[0].ff(acts[0], self.weights[0],
                                            self.bias[0]))  # conv1
        acts[2] = self.poolings[0].ff(acts[1])  # pool1
        acts[3] = ele.relu(self.convs[1].ff(acts[2], self.weights[1],
                                            self.bias[1]))  # conv2
        acts[4] = self.poolings[1].ff(acts[3])  # pool2
        acts[5] = ele.relu(self.convs[2].ff(acts[4], self.weights[2],
                                            self.bias[2]))  # conv3
        acts[6] = ele.relu(self.convs[3].ff(acts[5], self.weights[3],
                                            self.bias[3]))  # conv4
        acts[7] = ele.relu(self.convs[4].ff(acts[6], self.weights[4],
                                            self.bias[4]))  # conv5
        acts[8] = self.poolings[2].ff(acts[7])  # pool5
        re_acts8 = acts[8].reshape([np.prod(acts[8].shape[0:3]), num_samples])
        acts[9] = ele.relu(self.weights[5] * re_acts8 + self.bias[5])  # fc6
        mask6 = owl.randb(acts[9].shape, dropout_rate)
        acts[9] = ele.mult(acts[9], mask6)  # drop6
        acts[10] = ele.relu(self.weights[6] * acts[9] + self.bias[6])  # fc7
        mask7 = owl.randb(acts[10].shape, dropout_rate)
        acts[10] = ele.mult(acts[10], mask7)  # drop7
        acts[11] = self.weights[7] * acts[10] + self.bias[7]  # fc8

        out = co.softmax(acts[11], co.soft_op.instance)  # prob

        sens[11] = out - label
        sens[10] = self.weights[7].trans() * sens[11]  # fc8
        sens[10] = ele.mult(sens[10], mask7)  # drop7
        sens[10] = ele.relu_back(sens[10], acts[10])  # relu7
        sens[9] = self.weights[6].trans() * sens[10]
        sens[9] = ele.mult(sens[9], mask6)  # drop6
        sens[9] = ele.relu_back(sens[9], acts[9])  # relu6
        sens[8] = (self.weights[5].trans() * sens[9]).reshape(
            acts[8].shape)  # fc6
        sens[7] = ele.relu_back(self.poolings[2].bp(sens[8], acts[8], acts[7]),
                                acts[7])  # pool5, relu5
        sens[6] = ele.relu_back(self.convs[4].bp(sens[7], self.weights[4]),
                                acts[6])  # conv5, relu4
        sens[5] = ele.relu_back(self.convs[3].bp(sens[6], self.weights[3]),
                                acts[5])  # conv4, relu3
        sens[4] = self.convs[2].bp(sens[5], self.weights[2])  # conv3
        sens[3] = ele.relu_back(self.poolings[1].bp(sens[4], acts[4], acts[3]),
                                acts[3])  # pool2, relu2
        sens[2] = self.convs[1].bp(sens[3], self.weights[1])  # conv2
        sens[1] = self.poolings[0].bp(sens[2], acts[2], acts[1])  # pool1
        sens[1] = ele.relu_back(sens[1], acts[1])  # relu1

        weightsgrad[7] = sens[11] * acts[10].trans()
        weightsgrad[6] = sens[10] * acts[9].trans()
        weightsgrad[5] = sens[9] * re_acts8.trans()
        weightsgrad[4] = self.convs[4].weight_grad(sens[7], acts[6])
        weightsgrad[3] = self.convs[3].weight_grad(sens[6], acts[5])
        weightsgrad[2] = self.convs[2].weight_grad(sens[5], acts[4])
        weightsgrad[1] = self.convs[1].weight_grad(sens[3], acts[2])
        weightsgrad[0] = self.convs[0].weight_grad(sens[1], acts[0])
        biasgrad[7] = sens[11].sum(1)
        biasgrad[6] = sens[10].sum(1)
        biasgrad[5] = sens[9].sum(1)
        biasgrad[4] = self.convs[4].bias_grad(sens[7])
        biasgrad[3] = self.convs[3].bias_grad(sens[6])
        biasgrad[2] = self.convs[2].bias_grad(sens[5])
        biasgrad[1] = self.convs[1].bias_grad(sens[3])
        biasgrad[0] = self.convs[0].bias_grad(sens[1])
        return (out, weightsgrad, biasgrad)
Esempio n. 11
0
    def train_one_mb(self, data, label, dropout_rate):
        num_samples = data.shape[-1]
        num_layers = 12
        acts = [None] * num_layers
        sens = [None] * num_layers
        weightsgrad = [None] * self.num_weights
        biasgrad = [None] * self.num_weights

        # FF
        acts[0] = data
        acts[1] = ele.relu(self.convs[0].ff(acts[0], self.weights[0], self.bias[0])) # conv1
        acts[2] = self.poolings[0].ff(acts[1]) # pool1
        acts[3] = ele.relu(self.convs[1].ff(acts[2], self.weights[1], self.bias[1])) # conv2
        acts[4] = self.poolings[1].ff(acts[3]) # pool2
        acts[5] = ele.relu(self.convs[2].ff(acts[4], self.weights[2], self.bias[2])) # conv3
        acts[6] = ele.relu(self.convs[3].ff(acts[5], self.weights[3], self.bias[3])) # conv4
        acts[7] = ele.relu(self.convs[4].ff(acts[6], self.weights[4], self.bias[4])) # conv5
        acts[8] = self.poolings[2].ff(acts[7]) # pool5
        re_acts8 = acts[8].reshape([np.prod(acts[8].shape[0:3]), num_samples])
        acts[9] = ele.relu(self.weights[5] * re_acts8 + self.bias[5]) # fc6
        mask6 = owl.randb(acts[9].shape, dropout_rate)
        acts[9] = ele.mult(acts[9], mask6) # drop6
        acts[10] = ele.relu(self.weights[6] * acts[9] + self.bias[6]) # fc7
        mask7 = owl.randb(acts[10].shape, dropout_rate)
        acts[10] = ele.mult(acts[10], mask7) # drop7
        acts[11] = self.weights[7] * acts[10] + self.bias[7] # fc8

        out = co.softmax(acts[11], co.soft_op.instance) # prob

        sens[11] = out - label
        sens[10] = self.weights[7].trans() * sens[11] # fc8
        sens[10] = ele.mult(sens[10], mask7) # drop7
        sens[10] = ele.relu_back(sens[10], acts[10]) # relu7
        sens[9] = self.weights[6].trans() * sens[10]
        sens[9] = ele.mult(sens[9], mask6) # drop6
        sens[9] = ele.relu_back(sens[9], acts[9]) # relu6
        sens[8] = (self.weights[5].trans() * sens[9]).reshape(acts[8].shape) # fc6
        sens[7] = ele.relu_back(self.poolings[2].bp(sens[8], acts[8], acts[7]), acts[7]) # pool5, relu5
        sens[6] = ele.relu_back(self.convs[4].bp(sens[7], acts[6], self.weights[4]), acts[6]) # conv5, relu4
        sens[5] = ele.relu_back(self.convs[3].bp(sens[6], acts[5], self.weights[3]), acts[5]) # conv4, relu3
        sens[4] = self.convs[2].bp(sens[5], acts[4], self.weights[2]) # conv3
        sens[3] = ele.relu_back(self.poolings[1].bp(sens[4], acts[4], acts[3]), acts[3]) # pool2, relu2
        sens[2] = self.convs[1].bp(sens[3], acts[2], self.weights[1]) # conv2
        sens[1] = self.poolings[0].bp(sens[2], acts[2], acts[1]) # pool1
        sens[1] = ele.relu_back(sens[1], acts[1]) # relu1

        weightsgrad[7] = sens[11] * acts[10].trans()
        weightsgrad[6] = sens[10] * acts[9].trans()
        weightsgrad[5] = sens[9] * re_acts8.trans()
        weightsgrad[4] = self.convs[4].weight_grad(sens[7], acts[6], self.weights[4])
        weightsgrad[3] = self.convs[3].weight_grad(sens[6], acts[5], self.weights[3])
        weightsgrad[2] = self.convs[2].weight_grad(sens[5], acts[4], self.weights[2])
        weightsgrad[1] = self.convs[1].weight_grad(sens[3], acts[2], self.weights[1])
        weightsgrad[0] = self.convs[0].weight_grad(sens[1], acts[0], self.weights[0])
        biasgrad[7] = sens[11].sum(1)
        biasgrad[6] = sens[10].sum(1)
        biasgrad[5] = sens[9].sum(1)
        biasgrad[4] = self.convs[4].bias_grad(sens[7])
        biasgrad[3] = self.convs[3].bias_grad(sens[6])
        biasgrad[2] = self.convs[2].bias_grad(sens[5])
        biasgrad[1] = self.convs[1].bias_grad(sens[3])
        biasgrad[0] = self.convs[0].bias_grad(sens[1])
        return (out, weightsgrad, biasgrad)
Esempio n. 12
0
def train_one_mb(model, data, label, weightsgrad, biasgrad):
    #Be careful, python list is like pointer
    acts = [None] * model.num_layers
    sens = [None] * model.num_layers
    beforeacts = [None] * model.num_layers
    beforedropout = [None] * model.num_layers
    dropoutmask = [None] * model.num_layers
    before2fullyact = []
    conv2fullylayer = model.num_layers

    acts[0] = data

    num_samples = data.shape[-1]
    num_class = label.shape[0]

    #find the reshape layer
    for i in range(0, model.num_layers - 1):
        #if from conv 2 fully
        if (i < model.num_layers - 2) and (
                model.ff_infos[i]['ff_type'] == 'conv'
                or model.ff_infos[i]['ff_type'] == 'pooling') and (
                    model.ff_infos[i + 1]['ff_type'] == 'fully'):
            conv2fullylayer = i + 1
            break

    for i in range(0, model.num_layers - 1):
        if model.ff_infos[i]['ff_type'] == 'conv':
            #print '%d conv ff' % (i)
            beforeacts[i + 1] = conv_forward(acts[i], model.weights[i],
                                             model.bias[i],
                                             model.ff_infos[i]['conv_info'])
        elif model.ff_infos[i]['ff_type'] == 'pooling':
            #print '%d pooling ff' % (i)
            beforeacts[i + 1] = pooling_forward(
                acts[i], model.ff_infos[i]['pooling_info'])
        else:
            #print '%d fully ff' % (i)
            beforeacts[i + 1] = model.weights[i] * acts[i] + model.bias[i]

        #activation function
        if model.ff_infos[i]['neuron_type'] == 'RELU':
            #print '%d relu ff' % (i)
            acts[i + 1] = ele.relu(beforeacts[i + 1])
        elif model.ff_infos[i]['neuron_type'] == 'SOFTMAX':
            #print '%d softmax ff' % (i)
            acts[i + 1] = softmax_forward(
                beforeacts[i + 1].reshape([num_class, 1, 1, num_samples]),
                soft_op.instance).reshape([num_class, num_samples])  # prob
        else:
            #print '%d linear ff' % (i)
            acts[i + 1] = beforeacts[i + 1]

        #dropout
        beforedropout[i + 1] = acts[i + 1]
        if model.ff_infos[i]['dropout_rate'] > 0:
            #print '%d dropout ff' % (i)
            dropoutmask[i + 1] = owl.randb(acts[i + 1].shape,
                                           model.ff_infos[i]['dropout_rate'])
            acts[i + 1] = ele.mult(beforedropout[i + 1], dropoutmask[i + 1])

        if i + 1 == conv2fullylayer:
            before2fullyact = acts[i + 1]
            acts[i + 1] = before2fullyact.reshape(
                [np.prod(before2fullyact.shape[0:3]), num_samples])

    # error
    sens[model.num_layers - 1] = acts[model.num_layers - 1] - label

    #bp
    for i in range(model.num_layers - 1, 0, -1):
        if model.ff_infos[i - 1]['ff_type'] == 'conv':
            sens[i - 1] = conv_backward_data(
                sens[i], model.weights[i - 1],
                model.ff_infos[i - 1]['conv_info'])
        elif model.ff_infos[i - 1]['ff_type'] == 'pooling':
            if i == conv2fullylayer:
                sens[i - 1] = pooling_backward(
                    sens[i].reshape(before2fullyact.shape), before2fullyact,
                    acts[i - 1], model.ff_infos[i - 1]['pooling_info'])
            else:
                sens[i - 1] = pooling_backward(
                    sens[i], acts[i], acts[i - 1],
                    model.ff_infos[i - 1]['pooling_info'])
        else:
            sens[i - 1] = model.weights[i - 1].trans() * sens[i]

        if i - 2 >= 0:
            #dropout
            if model.ff_infos[i - 2]['dropout_rate'] > 0:
                sens[i - 1] = ele.mult(sens[i - 1], dropoutmask[i - 1])

            #backact
            if model.ff_infos[i - 2]['neuron_type'] == 'RELU':
                sens[i - 1] = ele.relu_back(sens[i - 1], beforedropout[i - 1],
                                            beforeacts[i - 1])
            else:
                sens[i - 1] = sens[i - 1]

    #gradient
    for i in range(0, model.num_layers - 1):
        if model.ff_infos[i]['ff_type'] == 'conv':
            weightsgrad[i] = conv_backward_filter(
                sens[i + 1], acts[i], model.ff_infos[i]['conv_info'])
            biasgrad[i] = conv_backward_bias(sens[i + 1])
        elif model.ff_infos[i]['ff_type'] == 'fully':
            weightsgrad[i] = sens[i + 1] * acts[i].trans()
            biasgrad[i] = sens[i + 1].sum(1)
        else:
            continue
    return acts[model.num_layers - 1]