コード例 #1
0
ファイル: main.py プロジェクト: pyvault/mnist_tut
    def __init__(self):
        # Define some model hyperparameters to work with MNIST images!
        input_size = 28*28  # dimensions of image
        hidden_size = 1000  # number of hidden units - generally bigger than input size for DAE

        # Now, define the symbolic input to the model (Theano)
        # We use a matrix rather than a vector so that minibatch processing can be done in parallel.
        x = T.matrix("X")
        self.inputs = [x]

        # Build the model's parameters - a weight matrix and two bias vectors
        W = get_weights_uniform(shape=(input_size, hidden_size), name="W")
        b0 = get_bias(shape=input_size, name="b0")
        b1 = get_bias(shape=hidden_size, name="b1")
        self.params = [W, b0, b1]

        # Perform the computation for a denoising autoencoder!
        # first, add noise (corrupt) the input
        corrupted_input = salt_and_pepper_custom(input=x)
        # next, run the hidden layer given the inputs (the encoding function)
        hiddens = tanh(T.dot(corrupted_input, W) + b1)
        # finally, create the reconstruction from the hidden layer (we tie the weights with W.T)
        reconstruction = sigmoid(T.dot(hiddens, W.T) + b0)
        # the training cost is reconstruction error - with MNIST this is binary cross-entropy
        self.train_cost = binary_crossentropy(output=reconstruction, target=x)

        # Compile everything into a Theano function for prediction!
        # When using real-world data in predictions, we wouldn't corrupt the input first.
        # Therefore, create another version of the hiddens and reconstruction without adding the noise
        hiddens_predict      = tanh(T.dot(x, W) + b1)
        self.recon_predict   = sigmoid(T.dot(hiddens_predict, W.T) + b0)
コード例 #2
0
    def __init__(self, inputs_hook=None, hiddens_hook=None, params_hook=None,
                 input_size=28*28, hidden_size=1000, noise_level=0.4,
                 hidden_activation='tanh', visible_activation='sigmoid', cost_function='binary_crossentropy'):
        # initialize the Model superclass
        super(DenoisingAutoencoder, self).__init__(
            **{arg: val for (arg, val) in locals().iteritems() if arg is not 'self'}
        )

        # Define model hyperparameters
        # deal with the inputs_hook and hiddens_hook for the size parameters!
        # if the hook exists, grab the size from the first element of the tuple.
        if self.inputs_hook is not None:
            assert len(self.inputs_hook) == 2, "Was expecting inputs_hook to be a tuple."
            self.input_size = inputs_hook[0]

        if self.hiddens_hook is not None:
            assert len(self.hiddens_hook) == 2, "was expecting hiddens_hook to be a tuple."
            hidden_size = hiddens_hook[0]


        # use the helper methods to grab appropriate activation functions from names!
        hidden_activation  = get_activation_function(hidden_activation)
        visible_activation = get_activation_function(visible_activation)

        # do the same for the cost function
        cost_function = get_cost_function(cost_function)

        # Now, define the symbolic input to the model (Theano)
        # We use a matrix rather than a vector so that minibatch processing can be done in parallel.
        # Make sure to deal with 'inputs_hook' if it exists!
        if self.inputs_hook is not None:
            # grab the new input variable from the inputs_hook tuple
            x = self.inputs_hook[1]
        else:
            x = T.matrix("X")
        self.inputs = [x]

        # Build the model's parameters - a weight matrix and two bias vectors
        # Make sure to deal with 'params_hook' if it exists!
        if self.params_hook:
            # check to see if it contains the three necessary variables
            assert len(self.params_hook) == 3, "Not correct number of params to DAE, needs 3!"
            W, b0, b1 = self.params_hook
        else:
            W  = get_weights_uniform(shape=(self.input_size, hidden_size), name="W")
            b0 = get_bias(shape=self.input_size, name="b0")
            b1 = get_bias(shape=hidden_size, name="b1")
        self.params = [W, b0, b1]

        # Perform the computation for a denoising autoencoder!
        # first, add noise (corrupt) the input
        corrupted_input = salt_and_pepper(input=x, noise_level=noise_level)
        # next, run the hidden layer given the inputs (the encoding function)
        # We don't need to worry about hiddens_hook during training, because we can't
        # run a cost without having the input!
        # hiddens_hook is more for the run function and linking methods below.
        hiddens = hidden_activation(T.dot(corrupted_input, W) + b1)
        # finally, create the reconstruction from the hidden layer (we tie the weights with W.T)
        reconstruction = visible_activation(T.dot(hiddens, W.T) + b0)
        # the training cost is reconstruction error
        self.train_cost = cost_function(output=reconstruction, target=x)

        # Compile everything into a Theano function for prediction!
        # When using real-world data in predictions, we wouldn't corrupt the input first.
        # Therefore, create another version of the hiddens and reconstruction without adding the noise.
        # Here is where we would handle hiddens_hook because this is a generative model!
        # For the run function, it would take in the hiddens instead of the input variable x.
        if self.hiddens_hook is not None:
            self.hiddens = self.hiddens_hook[1]
        else:
            self.hiddens = hidden_activation(T.dot(x, W) + b1)
        # make the reconstruction (generated) from the hiddens
        self.recon_predict = visible_activation(T.dot(self.hiddens, W.T) + b0)
        # now compile the run function accordingly - if it used x or hiddens as the input.
        if self.hiddens_hook is not None:
            self.f_run = function(inputs=[self.hiddens], outputs=self.recon_predict)
        else:
            self.f_run = function(inputs=[x], outputs=self.recon_predict)
コード例 #3
0
ファイル: convolutional.py プロジェクト: chagge/OpenDeep
    def __init__(self, inputs_hook, params_hook=None, input_shape=None, filter_shape=None, stride=None,
                 weights_init=None, weights_interval=None, weights_mean=None, weights_std=None, bias_init=None,
                 border_mode=None, activation=None, convolution=None, config=None, defaults=defaults):
        # combine everything by passing to Model's init
        super(Conv1D, self).__init__(**{arg: val for (arg, val) in locals().iteritems() if arg is not 'self'})
        # configs can now be accessed through self dictionary

        ##################
        # specifications #
        ##################
        # grab info from the inputs_hook, or from parameters
        # expect input to be in the form (B, C, I) (batch, channel, input data)
        #  inputs_hook is a tuple of (Shape, Input)
        if self.inputs_hook is not None:
            # make sure inputs_hook is a tuple
            assert len(self.inputs_hook) == 2, "expecting inputs_hook to be tuple of (shape, input)"
            self.input_shape = inputs_hook[0] or self.input_shape
            self.input = inputs_hook[1]
        else:
            # make the input a symbolic matrix
            self.input = T.ftensor3('X')

        # activation function!
        # if a string name was given, look up the correct function from our utils.
        if isinstance(self.activation, basestring):
            activation_func = get_activation_function(self.activation)
        # otherwise, if a 'callable' was passed (i.e. custom function), use that directly.
        else:
            assert callable(self.activation), "Activation function either needs to be a string name or callable!"
            activation_func = self.activation

        # filter shape should be in the form (num_filters, num_channels, filter_length)
        num_filters = self.filter_shape[0]
        filter_length = self.filter_shape[2]

        ################################################
        # Params - make sure to deal with params_hook! #
        ################################################
        if self.params_hook:
            # make sure the params_hook has W and b
            assert len(self.params_hook) == 2, \
                "Expected 2 params (W and b) for Conv1D, found {0!s}!".format(len(self.params_hook))
            W, b = self.params_hook
        else:
            # if we are initializing weights from a gaussian
            if self.weights_init.lower() == 'gaussian':
                W = get_weights_gaussian(
                    shape=self.filter_shape, mean=self.weights_mean, std=self.weights_std, name="W"
                )
            # if we are initializing weights from a uniform distribution
            elif self.weights_init.lower() == 'uniform':
                W = get_weights_uniform(shape=self.filter_shape, interval=self.weights_interval, name="W")
            # otherwise not implemented
            else:
                log.error("Did not recognize weights_init %s! Pleas try gaussian or uniform" %
                          str(self.weights_init))
                raise NotImplementedError(
                    "Did not recognize weights_init %s! Pleas try gaussian or uniform" %
                    str(self.weights_init))

            b = get_bias(shape=(num_filters,), name="b", init_values=self.bias_init)

        # Finally have the two parameters!
        self.params = [W, b]

        ########################
        # Computational Graph! #
        ########################
        if self.border_mode in ['valid', 'full']:
            conved = convolution(self.input,
                                 W,
                                 subsample=(self.stride,),
                                 image_shape=self.input_shape,
                                 filter_shape=self.filter_shape,
                                 border_mode=self.border_mode)
        elif self.border_mode == 'same':
            conved = convolution(self.input,
                                 W,
                                 subsample=(self.stride,),
                                 image_shape=self.input_shape,
                                 filter_shape=self.filter_shape,
                                 border_mode='full')
            shift = (filter_length - 1) // 2
            conved = conved[:, :, shift:self.input_shape[2] + shift]

        else:
            log.error("Invalid border mode: '%s'" % self.border_mode)
            raise RuntimeError("Invalid border mode: '%s'" % self.border_mode)

        self.output = activation_func(conved + b.dimshuffle('x', 0, 'x'))
コード例 #4
0
ファイル: convolutional.py プロジェクト: chagge/OpenDeep
    def __init__(self, inputs_hook, input_shape=None, filter_shape=None, convstride=None, padsize=None, group=None,
                 poolsize=None, poolstride=None, bias_init=None, local_response_normalization=None,
                 convolution=None, activation=None, params_hook=None, config=None, defaults=defaults):
        # combine everything by passing to Model's init
        super(ConvPoolLayer, self).__init__(**{arg: val for (arg, val) in locals().iteritems() if arg is not 'self'})
        # configs can now be accessed through self!

        # deal with the inputs coming from inputs_hook - necessary for now to give an input hook
        # inputs_hook is a tuple of (Shape, Input)
        if self.inputs_hook:
            assert len(self.inputs_hook) == 2, "expecting inputs_hook to be tuple of (shape, input)"
            self.input_shape = self.inputs_hook[0] or self.input_shape
            self.input = inputs_hook[1]
        else:
            self.input = T.ftensor4("X")

        #######################
        # layer configuration #
        #######################
        # activation function!
        # if a string name was given, look up the correct function from our utils.
        if isinstance(self.activation, basestring):
            self.activation_func = get_activation_function(self.activation)
        # otherwise, if a 'callable' was passed (i.e. custom function), use that directly.
        else:
            assert callable(self.activation), "Activation function either needs to be a string name or callable!"
            self.activation_func = self.activation

        # expect image_shape to be bc01!
        self.channel = self.input_shape[1]

        # shortening a word
        self.lrn = self.local_response_normalization

        # if lib_conv is cudnn, it works only on square images and the grad works only when channel % 16 == 0

        assert self.group in [1, 2], "group argument needs to be 1 or 2 (1 for default conv2d)"

        self.filter_shape = numpy.asarray(self.filter_shape)
        self.input_shape = numpy.asarray(self.input_shape)

        if self.lrn:
            self.lrn_func = cross_channel_normalization_bc01

        ################################################
        # Params - make sure to deal with params_hook! #
        ################################################
        if self.group == 1:
            if self.params_hook:
                # make sure the params_hook has W and b
                assert len(self.params_hook) == 2, \
                    "Expected 2 params (W and b) for ConvPoolLayer, found {0!s}!".format(len(self.params_hook))
                self.W, self.b = self.params_hook
            else:
                # if we are initializing weights from a gaussian
                if self.weights_init.lower() == 'gaussian':
                    self.W = get_weights_gaussian(
                        shape=self.filter_shape, mean=self.weights_mean, std=self.weights_std, name="W"
                    )
                # if we are initializing weights from a uniform distribution
                elif self.weights_init.lower() == 'uniform':
                    self.W = get_weights_uniform(shape=self.filter_shape, interval=self.weights_interval, name="W")
                # otherwise not implemented
                else:
                    log.error("Did not recognize weights_init %s! Pleas try gaussian or uniform" %
                              str(self.weights_init))
                    raise NotImplementedError(
                        "Did not recognize weights_init %s! Pleas try gaussian or uniform" %
                        str(self.weights_init))

                self.b = get_bias(shape=self.filter_shape[0], init_values=self.bias_init, name="b")
            self.params = [self.W, self.b]

        else:
            self.filter_shape[0] = self.filter_shape[0] / 2
            self.filter_shape[1] = self.filter_shape[1] / 2

            self.input_shape[0] = self.input_shape[0] / 2
            self.input_shape[1] = self.input_shape[1] / 2
            if self.params_hook:
                assert len(self.params_hook) == 4, "expected params_hook to have 4 params"
                self.W0, self.W1, self.b0, self.b1 = self.params_hook
            else:
                self.W0 = get_weights_gaussian(shape=self.filter_shape, name="W0")
                self.W1 = get_weights_gaussian(shape=self.filter_shape, name="W1")
                self.b0 = get_bias(shape=self.filter_shape[0], init_values=self.bias_init, name="b0")
                self.b1 = get_bias(shape=self.filter_shape[0], init_values=self.bias_init, name="b1")
            self.params = [self.W0, self.b0, self.W1, self.b1]

        #############################################
        # build appropriate graph for conv. version #
        #############################################
        self.output = self.build_computation_graph()

        # Local Response Normalization (for AlexNet)
        if self.lrn:
            self.output = self.lrn_func(self.output)

        log.debug("convpool layer initialized with shape_in: %s", str(self.input_shape))