def get_hidden_values(self, input):

        # convolve input feature maps with filters
        self.conv_out = conv.conv2d(
            input=input, filters=self.W, border_mode="full", filter_shape=self.kshp, image_shape=self.imshp
        )

        # k-max pooling.
        k = T.cast(T.max((self.k_Top, T.ceil(self.factor * self.s))), "int32")
        pool_shape = self.conv_out.shape
        pool = self.kmaxPool(self.conv_out, pool_shape, k)

        output = T.tanh(pool + self.b.dimshuffle("x", 0, "x", "x"))
        self.shape = output.shape

        hidden_input = output.flatten(2)
        self.fully_connected = AE(
            (self.rng), input=hidden_input, n_visible=self.kshp[0] * 25 * self.k_Top, n_hidden=60
        )  # nkerns[0] replaced with 8
        self.params.extend(self.fully_connected.params)

        return self.fully_connected.get_hidden_values(hidden_input)
def SCAE_preTrain(batch_size=1, nkerns= [3, 4], dataset=None, n_epochs=70, k_Top=5, learning_rate=1e-1, binary=True):
    """
        Stacked Convolutional AutoEncoders.
    """

    with open('SCAE_MR_1e-1K34', 'r') as f:
        rval = cPickle.load(f)
    return rval

    if dataset is None:
        dataset = Preprocess().load_data(binary)
    train_set_x = dataset[0][0]
    train_set_z = dataset[0][2]
    n_train_batch = train_set_x.get_value(borrow=True).shape[0]
    n_train_batch /= batch_size

    print '... Building Stacked Conv. AutoEncoders'

    rng = numpy.random.RandomState(96813)
    index = T.lscalar('index')
    
    x = T.dmatrix('Input Sentence')
    z = T.iscalar('Sentence length')
    layer0_input = x[:, :z*50].reshape((batch_size, 1, 50, -1))
    
    layer0 = SCAE(rng, input=layer0_input, image_shape=None, filter_shape=(nkerns[0], 1, 1, 8), \
                                        factor=.5, s=z, k_Top=k_Top, do_fold=False)

    layer1_input = layer0.get_hidden_values(layer0_input)
    layer1 = SCAE(rng, input=layer1_input, filter_shape=(nkerns[1], nkerns[0], 1, 5), \
                        image_shape=None, factor = .0, s=z, k_Top=k_Top, do_fold=False)

    layer1_output = layer1.get_hidden_values(layer1_input)
    
    hidden_input = layer1_output.flatten(2)
    layer2 = AE(rng, input=hidden_input, n_visible=layer1.kshp[0]*50*k_Top, n_hidden=100)
    
    Y = layer2.get_hidden_values(hidden_input)
    
    ################
    #   DECODING   #
    ################

    decode_hidden_layer = layer2.get_reconstructed_input(Y)
    decode_input = decode_hidden_layer.reshape(layer1.shape)
    
    decode_layer1 = layer1.get_reconstructed_input(decode_input)
    Z = layer0.get_reconstructed_input(decode_layer1)

    params = layer2.params + layer1.params + layer0.params
    
    def get_cost_updates(X, Z, params, learning_rate):
        ''' Update the Stacked Convolutional Auto-Encoders. '''
        
        L = T.sum((X-Z) ** 2, axis=(1,2,3))
        cost = T.mean(L)
        
        gparams = T.grad(cost, params)
        
        rho = 1e-7
        G = [(theano.shared(value=numpy.zeros_like(param.get_value()), name="AdaGrad_" + param.name, borrow=True)) for param in params]
        G_update = [T.add(g_adag, T.sqr(grad_i)) for g_adag, grad_i in zip(G, gparams)]
        
        updates = []
        for param_i, g_update, grad_i, g in zip(params, G_update, gparams, G):
            updates.append((param_i, param_i - learning_rate * grad_i / T.sqrt(g_update) ))
            updates.append((g, g_update))
        
        return (cost, updates)
    
    cost, updates = get_cost_updates(layer0_input, Z, params, learning_rate)
    
    train_model = theano.function([index], cost, updates=updates, \
                        givens={x: train_set_x[index * batch_size: (index + 1) * batch_size],
                                z: train_set_z[index]})

    print '... Pretraining model'

    plot_SCAE = []
    epoch = 0
    while epoch < n_epochs:
        epoch += 1
        for minibatch in xrange(n_train_batch):
            cost_ij = train_model(minibatch)
        print '\tepoch %i,\tcost  %f' % (epoch, cost_ij)
        plot_SCAE.append(cost_ij)
    plot('SCAE_Movie Results.', numpy.asarray(plot_SCAE), 74e-2)

    # Serialise the learned parameters
    with open('SCAE_MR_1e-1K%i%i'%(nkerns[0], nkerns[1]), 'wb') as f:
        cPickle.dump(params, f)
    return params
def Layer_Wise_preTrain(batch_size = 1, nkerns = [3, 4], dataset=None, n_epochs=6, k_Top=5):
    """
        Layer-wise Convolutional auto-encoders.
    """

    if dataset is None:
        dataset = Preprocess_Input().load_data()
    train_set_x = dataset[0][0]
    train_set_z = dataset[0][2]
    n_train_batch = train_set_x.get_value(borrow=True).shape[0]
    n_train_batch /= batch_size

    print '... Building AutoEncoders'
    rng = numpy.random.RandomState(96813)
    index = T.lscalar('index')
    learning_rate = T.dscalar('rate')
    x = T.matrix('x')
    z = T.iscalar('z')
    #index.tag.test_value = 0
    #learning_rate.tag.test_value = .3
    em = 50
    layer0_input = x[:, :z*50].reshape((batch_size, 1, 50, -1))

    #   Auto-Encoder for Conv. LAYER 1
    layer0 = CAE(rng, input=layer0_input, image_shape=(batch_size, 1, em, None), \
        filter_shape=(nkerns[0], 1, 1, 7), factor=.5, s=z, k_Top=k_Top, do_fold=True)
    
    #zz = layer0.get_cost_updates(learning_rate)
    #print 'hidden:', theano.function([index, learning_rate], [zz], on_unused_input='ignore', \
    #                                        givens={x: train_set_x[index * batch_size: (index + 1) * batch_size], \
    #                                        z: train_set_z[index]})(0, .3)
    cost, updates = layer0.get_cost_updates(learning_rate)
    #print 'DECODE RESULT:\n', theano.function([index], [layer0.output.shape, layer0_input.shape, z.type, layer0.zz.shape], \
    #                                        givens={x: train_set_x[index * batch_size: (index + 1) * batch_size], \
    #                                        z: train_set_z[index]})(0)
    
    train0 = theano.function([index, learning_rate], cost, updates=updates, \
                        givens={x: train_set_x[index * batch_size: (index + 1) * batch_size],
                                z: train_set_z[index]})
    em /= 2
    #   Auto-Encoder for Conv. LAYER 2.
    layer1_input = layer0.Output()
    layer1 = CAE(rng, input=layer1_input, image_shape=(batch_size, nkerns[0], em, None), \
                        filter_shape=(nkerns[1], nkerns[0], 1, 3), factor=.0, s=z, k_Top=k_Top, do_fold=True)

    cost1, updates1 = layer1.get_cost_updates(learning_rate)

    train1 = theano.function([index, learning_rate], cost1, updates=updates1, \
                        givens={x: train_set_x[index * batch_size: (index + 1) * batch_size],
                                z: train_set_z[index]})
    em /= 2
    #   Auto-Encoder for Hidden Layer.
    hidden_input = layer1.Output().flatten(2)
    hidden_layer = AE(rng, input=hidden_input, n_visible=nkerns[1]*em*k_Top, n_hidden=100)
    cost_h, updates_h = hidden_layer.get_cost_updates(learning_rate)
    train_h = theano.function([index, learning_rate], cost_h, updates=updates_h, \
                        givens={x: train_set_x[index * batch_size: (index + 1) * batch_size],
                                z: train_set_z[index]})
    print '... Pretraining model'

    ls_1 = []
    rate = 1e-2
    epoch = 0
    while epoch < n_epochs:
        epoch = epoch + 1
        for minibatch in xrange(n_train_batch):
            cost_ij = train0(minibatch, rate)
        ls_1.append(cost_ij)
        rate *= .95
        print '\tepoch %i : cost: %f' % (epoch, cost_ij)
        # print layer0.W.get_value(borrow=True)

    ls_2 = []
    rate = 1e-2
    epoch = 0
    while epoch < n_epochs:
        epoch = epoch + 1
        for minibatch in xrange(n_train_batch):
            cost_ij = train1(minibatch, rate)
        ls_2.append(cost_ij)
        rate *= .95
        print '\tepoch %i : cost: %f' % (epoch, cost_ij)
        # print layer1.W.get_value(borrow=True)

    ls_3 = []
    rate=4e-2
    epoch = 0
    while epoch < n_epochs:
        epoch = epoch + 1
        for minibatch in xrange(n_train_batch):
            cost_ij = train_h(minibatch, rate)
        ls_3.append(cost_ij)
        rate *= .95
        print '\tepoch %i : cost: %f' % (epoch, cost_ij)
        
    
    #  PLOT AutoEncoder Cost Function
    plt.subplot(3, 1, 1)
    plt.plot(numpy.arange(len(ls_1)) + 1, numpy.asarray(ls_1), 'r.-')
    plt.title('AutoEncoder Cost function Results')
    plt.xlabel('Epochs')
    plt.ylabel('Convolutional Layer 1')
    
    plt.subplot(3, 1, 2)
    plt.plot(numpy.arange(len(ls_2)) + 1, numpy.asarray(ls_2), 'r.-')
    #plt.title('AutoEncoder Cost function Results')
    plt.xlabel('Epochs')
    plt.ylabel('Convolutional Layer 2')
    
    plt.subplot(3, 1, 3)
    plt.plot(numpy.arange(len(ls_3)) + 1, numpy.asarray(ls_3), 'r.-')
    plt.xlabel('Epochs')
    plt.ylabel('Hidden Layer values')
    
    plt.show()
    
    return [layer0.params, layer1.params, hidden_layer.params]
Esempio n. 4
0
true_class = []
n_classes = np.size(data_set, axis=0)
class_name = ["coast", "insidecity", "opencountry", "street", "tallbuilding"]
for i in range(n_classes):
    my_class = i
    for j in range(np.size(data_set[i], axis=0)):
        true_class.append(my_class)

true_class = np.array(true_class)

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# use AutoEncoder for dimension reduction
print("training auto-encoder")
red_dim = 64
model = AE(dim, red_dim)
learning_rate = 0.004
epochs = 100
batch_size = 32
AE_training_errors = model.train(train, epochs, learning_rate, batch_size)
red_train = model.get_encoding(train)

train_data, val_data, train_class, val_class = train_test_split(red_train,
                                                                true_class,
                                                                test_size=0.20,
                                                                shuffle=True)

print("training classifier")
learning_rate = 0.003
batch_size = 32
epochs = 200
class SCAE(object):
    def __init__(self, rng, input, filter_shape, image_shape, factor, s, k_Top=5, do_fold=True):

        #   Input will be image_shape, filter_shape, input, rng
        self.kshp = filter_shape
        self.imshp = None
        self.i_kshp = (self.kshp[1], self.kshp[0], self.kshp[2], self.kshp[3])
        self.i_imshp = None
        self.do_fold = do_fold
        self.k_Top = k_Top
        self.factor = factor
        self.s = s
        self.rng = rng

        fan_in = numpy.prod(filter_shape[1:])
        fan_out = filter_shape[0] * numpy.prod(filter_shape[2:])

        # initialize weights with random weights
        W_bound = numpy.sqrt(6.0 / (fan_in + fan_out))
        self.W = theano.shared(
            numpy.asarray(
                rng.uniform(low=-W_bound, high=W_bound, size=filter_shape), dtype=theano.config.floatX  #   2, 1, 1, 3
            ),
            name="conv_W",
            borrow=True,
        )

        # the bias is a 1D tensor -- one bias per output feature map
        b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
        self.b = theano.shared(value=b_values, name="conv_b", borrow=True)

        self.c = theano.shared(value=0.0, name="deconv_c")

        self.W_tilde = self.W[:, :, ::-1, ::-1].dimshuffle(1, 0, 2, 3)

        if input == None:
            self.x = T.dmatrix(name="input")
        else:
            self.x = input

        self.params = [self.W, self.b, self.c]

    def Fold(self, conv_out, ds=(2, 1)):
        """Fold into two. (Sum up vertical neighbours)"""
        imgs = images2neibs(
            conv_out, T.as_tensor_variable(ds), mode="ignore_borders"
        )  # Correct 'mode' if there's a typo!
        orig = conv_out.shape
        shp = (orig[0], orig[1], T.cast(orig[2] / 2, "int32"), orig[3])
        res = T.reshape(T.sum(imgs, axis=-1), shp)
        return res

    def kmaxPool(self, conv_out, pool_shape, k):
        """
        Perform k-max Pooling.
        """
        n0, n1, d, size = pool_shape
        imgs = images2neibs(conv_out, T.as_tensor_variable((1, size)))

        indices = T.argsort(T.mul(imgs, -1))
        self.k_max_indices = T.sort(indices[:, :k])

        S = T.arange(d * n1 * n0).reshape((d * n1 * n0, 1))
        return imgs[S, self.k_max_indices].reshape((n0, n1, d, k))

    def unpooling(self, Y_4D, Z, X_4D):
        """ This method reverses pooling operation.
            """
        Y = images2neibs(Y_4D, T.as_tensor_variable((1, Y_4D.shape[3])))
        X = images2neibs(X_4D, T.as_tensor_variable((1, X_4D.shape[3])))
        X_z = T.zeros_like(X)
        X_ = T.set_subtensor(X_z[T.arange(X.shape[0]).reshape((X.shape[0], 1)), Z], Y)

        return X_.reshape(X_4D.shape)

    def Output(self):

        #  Convolve input with trained parameters.
        conv_out = conv.conv2d(
            input=self.x, filters=self.W, border_mode="full", filter_shape=self.kshp, image_shape=self.imshp
        )
        # Fold conv result into two.
        if self.do_fold:
            fold = self.Fold(conv_out)

        # k-max pooling.
        k = T.cast(T.max((self.k_Top, T.ceil(self.factor * self.s))), "int32")
        if self.do_fold:
            pool_shape = fold.shape
            pooled_out = self.kmaxPool(fold, pool_shape, k)
        else:
            pool_shape = conv_out.shape
            pooled_out = self.kmaxPool(conv_out, pool_shape, k)

        return T.tanh(pooled_out + self.b.dimshuffle("x", 0, "x", "x"))

    def get_hidden_values(self, input):

        # convolve input feature maps with filters
        self.conv_out = conv.conv2d(
            input=input, filters=self.W, border_mode="full", filter_shape=self.kshp, image_shape=self.imshp
        )

        # k-max pooling.
        k = T.cast(T.max((self.k_Top, T.ceil(self.factor * self.s))), "int32")
        pool_shape = self.conv_out.shape
        pool = self.kmaxPool(self.conv_out, pool_shape, k)

        output = T.tanh(pool + self.b.dimshuffle("x", 0, "x", "x"))
        self.shape = output.shape

        hidden_input = output.flatten(2)
        self.fully_connected = AE(
            (self.rng), input=hidden_input, n_visible=self.kshp[0] * 25 * self.k_Top, n_hidden=60
        )  # nkerns[0] replaced with 8
        self.params.extend(self.fully_connected.params)

        return self.fully_connected.get_hidden_values(hidden_input)

    def get_reconstructed_input(self, hidden, start):

        reconstruct_AE = self.fully_connected.get_reconstructed_input(hidden)
        hidden_NN = reconstruct_AE.reshape(self.shape)

        unpool = self.unpooling(hidden_NN, self.k_max_indices, start)
        deconv = conv.conv2d(input=unpool, filters=self.W_tilde, filter_shape=self.i_kshp, image_shape=None)

        return T.tanh(deconv + self.c.dimshuffle("x", "x", "x", "x"))
        # return val*(val>0)

    def get_cost_updates(self, learning_rate):

        y = self.get_hidden_values(self.x)
        z = self.get_reconstructed_input(y, self.conv_out)

        L = T.sum((self.x - z) ** 2, axis=(1, 2, 3))
        cost = T.mean(L)

        gparams = T.grad(cost, self.params)

        rho = 1e-7
        G = [
            (theano.shared(value=numpy.zeros_like(param.get_value()), name="AdaGrad_" + param.name, borrow=True))
            for param in self.params
        ]
        G_update = [T.add(g_adag, T.sqr(grad_i)) for g_adag, grad_i in zip(G, gparams)]

        updates = []
        for param_i, g_update, grad_i, g in zip(self.params, G_update, gparams, G):
            updates.append((param_i, param_i - learning_rate * grad_i / T.sqrt(g_update)))
            updates.append((g, g_update))

        return (cost, updates)