def main(): # First, let's create a simple feedforward MLP with one hidden layer as a Prototype. mlp = Prototype() mlp.add( Dense(input_size=28 * 28, output_size=1000, activation='rectifier', noise='dropout')) mlp.add(SoftmaxLayer(output_size=10)) # Now, we get to choose what values we want to monitor, and what datasets we would like to monitor on! # Each Model (in our case, the Prototype), has a get_monitors method that will return a useful # dictionary of {string_name: monitor_theano_expression} for various computations of the model we might # care about. By default, this method returns an empty dictionary - it was the model creator's job to # include potential monitor values. mlp_monitors = mlp.get_monitors() mlp_channel = MonitorsChannel(name="error") for name, expression in mlp_monitors.items(): mlp_channel.add( Monitor(name=name, expression=expression, train=True, valid=True, test=True)) # create some monitors for statistics about the hidden and output weights! # let's look at the mean, variance, and standard deviation of the weights matrices. weights_channel = MonitorsChannel(name="weights") hiddens_1 = mlp[0].get_params()[0] hiddens1_mean = T.mean(hiddens_1) weights_channel.add( Monitor(name="hiddens_mean", expression=hiddens1_mean, train=True)) hiddens_2 = mlp[1].get_params()[0] hiddens2_mean = T.mean(hiddens_2) weights_channel.add( Monitor(name="out_mean", expression=hiddens2_mean, train=True)) # create our plot object to do live plotting! plot = Plot(bokeh_doc_name="Monitor Tutorial", monitor_channels=[mlp_channel, weights_channel], open_browser=True) # use SGD optimizer optimizer = SGD(model=mlp, dataset=MNIST(concat_train_valid=False), epochs=500, save_freq=100, batch_size=600, learning_rate=.01, lr_decay=False, momentum=.9, nesterov_momentum=True) # train, with the plot! optimizer.train(plot=plot)
def main(): # First, let's create a simple feedforward MLP with one hidden layer as a Prototype. mlp = Prototype() mlp.add(Dense(input_size=28*28, output_size=1000, activation='rectifier', noise='dropout')) mlp.add(SoftmaxLayer(output_size=10)) # Now, we get to choose what values we want to monitor, and what datasets we would like to monitor on! # Each Model (in our case, the Prototype), has a get_monitors method that will return a useful # dictionary of {string_name: monitor_theano_expression} for various computations of the model we might # care about. By default, this method returns an empty dictionary - it was the model creator's job to # include potential monitor values. mlp_monitors = mlp.get_monitors() mlp_channel = MonitorsChannel(name="error") for name, expression in mlp_monitors.items(): mlp_channel.add(Monitor(name=name, expression=expression, train=True, valid=True, test=True)) # create some monitors for statistics about the hidden and output weights! # let's look at the mean, variance, and standard deviation of the weights matrices. weights_channel = MonitorsChannel(name="weights") hiddens_1 = mlp[0].get_params()[0] hiddens1_mean = T.mean(hiddens_1) weights_channel.add(Monitor(name="hiddens_mean", expression=hiddens1_mean, train=True)) hiddens_2 = mlp[1].get_params()[0] hiddens2_mean = T.mean(hiddens_2) weights_channel.add(Monitor(name="out_mean", expression=hiddens2_mean, train=True)) # create our plot object to do live plotting! plot = Plot(bokeh_doc_name="Monitor Tutorial", monitor_channels=[mlp_channel, weights_channel], open_browser=True) # use SGD optimizer optimizer = SGD(model=mlp, dataset=MNIST(concat_train_valid=False), epochs=500, save_freq=100, batch_size=600, learning_rate=.01, lr_decay=False, momentum=.9, nesterov_momentum=True) # train, with the plot! optimizer.train(plot=plot)
def run_mlp(): # test the new way to automatically fill in inputs for models mlp = Prototype() x = ((None, 784), matrix("x")) mlp.add(Dense(inputs=x, outputs=1000, activation='rectifier')) mlp.add(Dense, outputs=1500, activation='tanh') mlp.add(Softmax, outputs=10, out_as_probs=False) # define our loss to optimize for the model (and the target variable) # targets from MNIST are int64 numbers 0-9 y = lvector('y') loss = Neg_LL(inputs=mlp.models[-1].p_y_given_x, targets=y, one_hot=False) mnist = MNIST() optimizer = AdaDelta(model=mlp, loss=loss, dataset=mnist, epochs=10) optimizer.train() test_data, test_labels = mnist.test_inputs, mnist.test_targets test_data = test_data[:25] test_labels = test_labels[:25] # use the run function! yhat = mlp.run(test_data) print('-------') print('Prediction: %s' % str(yhat)) print('Actual: %s' % str(test_labels.astype('int32')))
def sequential_add_layers(): # This method is to demonstrate adding layers one-by-one to a Prototype container. # As you can see, inputs_hook are created automatically by Prototype so we don't need to specify! mlp = Prototype() mlp.add(Dense(input_size=28*28, output_size=1000, activation='rectifier', noise='dropout', noise_level=0.5)) mlp.add(Dense(output_size=512, activation='rectifier', noise='dropout', noise_level=0.5)) mlp.add(SoftmaxLayer(output_size=10)) return mlp
# some debugging output to see what is going on under the hood config_root_logger() ######### # Model # ######### # build a Prototype container to easily add layers and make a cohesive model! lenet = Prototype() # need to define a variable for the inputs to this model, as well as the shape # (batch, channel, row, col) images = T.tensor4('xs') images_shape = (None, 1, 28, 28) # conv/pool/droput layer 1 lenet.add( Conv2D(inputs=(images_shape, images), n_filters=20, filter_size=(5, 5), border_mode='full', activation='relu')) lenet.add(Pool2D, size=(2, 2)) lenet.add(Noise, noise='dropout', noise_level=0.5) # conv/pool/droput layer 2 lenet.add(Conv2D, n_filters=50, filter_size=(5, 5), border_mode='full', activation='relu') lenet.add(Pool2D, size=(2, 2)) lenet.add(Noise, noise='dropout', noise_level=0.5) # reshape convolution output to be 2D to feed into fully-connected layers # Prototype container keeps its layers in the `models` attribute list: grab the latest model output dense_input = lenet.models[-1].get_outputs().flatten(2)
if __name__ == '__main__': # some debugging output to see what is going on under the hood config_root_logger() ######### # Model # ######### # build a Prototype container to easily add layers and make a cohesive model! lenet = Prototype() # need to define a variable for the inputs to this model, as well as the shape # (batch, channel, row, col) images = T.tensor4('xs') images_shape = (None, 1, 28, 28) # conv/pool/droput layer 1 lenet.add( Conv2D(inputs=(images_shape, images), n_filters=20, filter_size=(5, 5), border_mode='full', activation='relu') ) lenet.add(Pool2D, size=(2, 2)) lenet.add(Noise, noise='dropout', noise_level=0.5) # conv/pool/droput layer 2 lenet.add(Conv2D, n_filters=50, filter_size=(5, 5), border_mode='full', activation='relu') lenet.add(Pool2D, size=(2, 2)) lenet.add(Noise, noise='dropout', noise_level=0.5) # reshape convolution output to be 2D to feed into fully-connected layers # Prototype container keeps its layers in the `models` attribute list: grab the latest model output dense_input = lenet.models[-1].get_outputs().flatten(2) dense_in_shape = lenet.models[-1].output_size[:1] + (np.prod(lenet.models[-1].output_size[1:]), ) # now make the dense (fully-connected) layers! lenet.add(Dense(inputs=(dense_in_shape, dense_input), outputs=500, activation='tanh')) lenet.add(Noise, noise='dropout', noise_level=0.5) # softmax classification layer!
def testLeNet(self): try: # quick and dirty way to create a model from arbitrary layers lenet = Prototype(outdir=None) # our input is going to be 4D tensor of images with shape (batch_size, 1, 28, 28) x = ((None, 1, 28, 28), ftensor4('x')) # our first convolutional layer lenet.add( Conv2D(inputs=x, n_filters=20, filter_size=(5, 5), outdir=None)) # our first pooling layer, automatically hooking inputs to the previous convolutional outputs lenet.add(Pool2D, size=(2, 2)) # our second convolutional layer lenet.add(Conv2D, n_filters=50, filter_size=(5, 5), outdir=None) # our second pooling layer lenet.add(Pool2D, size=(2, 2)) # now we need to flatten the 4D convolution outputs into 2D matrix (just flatten the trailing dimensions) lenet.add(Flatten, ndim=2) # one dense hidden layer lenet.add(Dense, outputs=500, activation='tanh', outdir=None) # hook a softmax classification layer, outputting the probabilities. lenet.add(Softmax, outputs=10, out_as_probs=True, outdir=None) # Grab the MNIST dataset data = MNIST(path="../../../datasets/{!s}".format(mnist_name), concat_train_valid=False, flatten=False) # define our loss to optimize for the model (and the target variable) # targets from MNIST are int64 numbers 0-9 y = lvector('y') loss = Neg_LL(inputs=lenet.get_outputs(), targets=y, one_hot=False) # monitor error_monitor = Monitor( name='error', expression=mean(neq(lenet.models[-1].y_pred, y)), valid=True, test=True, out_service=FileService('outputs/lenet_error.txt')) # optimize our model to minimize loss given the dataset using SGD optimizer = SGD(model=lenet, dataset=data, loss=loss, epochs=10, batch_size=128, learning_rate=.1, momentum=False) print("Training LeNet...") optimizer.train(monitor_channels=error_monitor) def test_subset(filename, expected, conf=0.001): with open(filename, 'r') as f: errs = [float(err) for err in f] for i, (err, exp) in enumerate(zip(errs, expected)): if i == 0: c = conf * 10 else: c = conf self.assertTrue( exp - c < round(err, 4) < exp + c, "Errors: {!s} and Expected: {!s} -- Error at {!s} and {!s}" .format(errs, expected, err, exp)) test_subset('outputs/lenet_error_train.txt', [ .0753, .0239, .0159, .0113, .0088, .0064, .0050, .0037, .0026, .0019 ]) test_subset('outputs/lenet_error_valid.txt', [ .0283, .0209, .0170, .0151, .0139, .0129, .0121, .0118, .0112, .0113 ]) test_subset('outputs/lenet_error_test.txt', [ .0319, .0213, .0167, .0134, .0122, .0119, .0116, .0107, .0104, .0105 ]) shutil.rmtree('outputs/') finally: if 'lenet' in locals(): del lenet if 'data' in locals(): del data if 'y' in locals(): del y if 'x' in locals(): del x if 'loss' in locals(): del loss if 'optimizer' in locals(): del optimizer
print "Getting data..." # data = MNIST() df = pd.read_csv("train.csv") features = np.array(df.ix[:,1:]) targets = np.array(df["label"]) features_train, features_test, targets_train, targets_test = train_test_split(features,targets, test_size=0.4, random_state=4) data = Dataset(features_train, train_targets=targets_train, test_inputs=features_test, test_targets=targets_test) print "Creating model..." in_shape = (None, 28*28) in_var = matrix('xs') mlp = Prototype() mlp.add(Dense(inputs=(in_shape, in_var), outputs=512, activation='relu')) mlp.add(Noise, noise='dropout', noise_level=0.5) mlp.add(Dense, outputs=512, activation='relu') mlp.add(Noise, noise='dropout', noise_level=0.5) mlp.add(Softmax, outputs=10, out_as_probs=False) print "Training..." target_var = lvector('ys') loss = Neg_LL(inputs=mlp.models[-1].p_y_given_x, targets=target_var, one_hot=False) optimizer = AdaDelta(model=mlp, loss=loss, dataset=data, epochs=10) optimizer.train() print "Predicting..." predictions = mlp.run(data.test_inputs)
if __name__ == '__main__': # some debugging output to see what is going on under the hood config_root_logger() ######### # Model # ######### # build a Prototype container to easily add layers and make a cohesive model! mlp = Prototype() # need to define a variable for the inputs to this model, as well as the shape # we are doing minibatch training (where we don't know the minibatch size), and the image is a (784,) array. x = T.matrix('xs') x_shape = (None, 28*28) # add our first dense (fully-connected) layer! mlp.add(Dense(inputs=(x_shape, x), outputs=500, activation='tanh')) # noise is used to regularize the layer from overfitting to data (helps generalization) # when adding subsequent layers, we can simply provide the class type and any other kwargs # (omitting the `inputs` kwarg) and it will route the previous layer's outputs as the current # layer's inputs. mlp.add(Noise, noise='dropout', noise_level=0.5) # add our classification layer mlp.add(Softmax, outputs=10, out_as_probs=False) ################ # Optimization # ################ # Now that our model is complete, let's define the loss function to optimize # first need a target variable labels = T.lvector('ys') # negative log-likelihood for classification cost
def build_lenet(): # quick and dirty way to create a model from arbitrary layers lenet = Prototype() # our input is going to be 4D tensor of images with shape (batch_size, 1, 28, 28) x = ((None, 1, 28, 28), tensor4('x')) # our first convolutional layer lenet.add( Conv2D(inputs=x, n_filters=20, filter_size=(5, 5)) ) # our first pooling layer, automatically hooking inputs to the previous convolutional outputs lenet.add( Pool2D, size=(2, 2) ) # our second convolutional layer lenet.add( Conv2D, n_filters=50, filter_size=(5, 5) ) # our second pooling layer lenet.add( Pool2D, size=(2, 2) ) # now we need to flatten the 4D convolution outputs into 2D matrix (just flatten the trailing dimensions) lenet.add( Flatten, ndim=2 ) # one dense hidden layer lenet.add( Dense, outputs=500, activation='tanh' ) # hook a softmax classification layer, outputting the probabilities. lenet.add( Softmax, outputs=10, out_as_probs=True ) return lenet
if __name__ == '__main__': # some debugging output to see what is going on under the hood config_root_logger() ######### # Model # ######### # build a Prototype container to easily add layers and make a cohesive model! mlp = Prototype() # need to define a variable for the inputs to this model, as well as the shape # we are doing minibatch training (where we don't know the minibatch size), and the image is a (784,) array. x = T.matrix('xs') x_shape = (None, 28 * 28) # add our first dense (fully-connected) layer! mlp.add(Dense(inputs=(x_shape, x), outputs=500, activation='tanh')) # noise is used to regularize the layer from overfitting to data (helps generalization) # when adding subsequent layers, we can simply provide the class type and any other kwargs # (omitting the `inputs` kwarg) and it will route the previous layer's outputs as the current # layer's inputs. mlp.add(Noise, noise='dropout', noise_level=0.5) # add our classification layer mlp.add(Softmax, outputs=10, out_as_probs=False) ################ # Optimization # ################ # Now that our model is complete, let's define the loss function to optimize # first need a target variable labels = T.lvector('ys') # negative log-likelihood for classification cost
def build_lenet(): # quick and dirty way to create a model from arbitrary layers lenet = Prototype() # our input is going to be 4D tensor of images with shape (batch_size, 1, 28, 28) x = ((None, 1, 28, 28), tensor4('x')) # our first convolutional layer lenet.add(Conv2D(inputs=x, n_filters=20, filter_size=(5, 5))) # our first pooling layer, automatically hooking inputs to the previous convolutional outputs lenet.add(Pool2D, size=(2, 2)) # our second convolutional layer lenet.add(Conv2D, n_filters=50, filter_size=(5, 5)) # our second pooling layer lenet.add(Pool2D, size=(2, 2)) # now we need to flatten the 4D convolution outputs into 2D matrix (just flatten the trailing dimensions) lenet.add(Flatten, ndim=2) # one dense hidden layer lenet.add(Dense, outputs=500, activation='tanh') # hook a softmax classification layer, outputting the probabilities. lenet.add(Softmax, outputs=10, out_as_probs=True) return lenet
def build_lenet(): # quick and dirty way to create a model from arbitrary layers lenet = Prototype() # our input is going to be 4D tensor of images with shape (batch_size, 1, 28, 28) x = ((None, 1, 28, 28), tensor4('x')) # our first convolutional layer lenet.add( Conv2D(inputs=x, n_filters=20, filter_size=(5, 5)) ) # our first pooling layer, automatically hooking inputs to the previous convolutional outputs lenet.add( Pool2D, size=(2, 2) ) # our second convolutional layer lenet.add( Conv2D, n_filters=50, filter_size=(5, 5) ) # our second pooling layer lenet.add( Pool2D, size=(2, 2) ) # now we need to flatten the 4D convolution outputs into 2D matrix (just flatten the trailing dimensions) dense_input = lenet.models[-1].get_outputs().flatten(2) # redefine the size appropriately for flattening (since we are doing a Theano modification) dense_input_shape = (None, np.prod(lenet.models[-1].output_size[1:])) # pass this flattened matrix as the input to a Dense layer! lenet.add( Dense( inputs=[(dense_input_shape, dense_input)], outputs=500, activation='tanh' ) ) # automatically hook a softmax classification layer, outputting the probabilities. lenet.add( Softmax, outputs=10, out_as_probs=True ) return lenet
from opendeep.models.utils import Noise, Pool2D from opendeep.optimization.loss import Neg_LL from opendeep.optimization import AdaDelta from opendeep.monitor import Monitor from opendeep.data.stream import ModifyStream from opendeep.data import ImageDataset import filter_test #config_root_logger() lenet = Prototype() images = T.tensor4('xs') images_shape = (None, 1, 480, 640) lenet.add(Conv2D(inputs=(images_shape, images), n_filters=6, filter_size=(5, 5), border_mode="full", activation="relu")) lenet.add(Pool2D, size=(2, 2)) lenet.add(Noise, noise="dropout", noise_level=0.5) lenet.add(Conv2D, n_filters=20, filter_size=(5, 5), border_mode="full", activation="relu") lenet.add(Pool2D, size=(2, 2)) lenet.add(Noise, noise="dropout", noise_level=0.5) dense_input = lenet.models[-1].get_outputs().flatten(2) dense_in_shape = lenet.models[-1].output_size[:1] + (np.prod(lenet.models[-1].output_size[1:]), ) lenet.add(Dense(inputs=(dense_in_shape, dense_input), outputs=500, activation="relu")) lenet.add(Noise, noise="dropout", noise_level=0.5) lenet.add(Dense, outputs=2)
def testLeNet(self): try: # quick and dirty way to create a model from arbitrary layers lenet = Prototype(outdir=None) # our input is going to be 4D tensor of images with shape (batch_size, 1, 28, 28) x = ((None, 1, 28, 28), ftensor4('x')) # our first convolutional layer lenet.add( Conv2D(inputs=x, n_filters=20, filter_size=(5, 5), outdir=None) ) # our first pooling layer, automatically hooking inputs to the previous convolutional outputs lenet.add( Pool2D, size=(2, 2) ) # our second convolutional layer lenet.add( Conv2D, n_filters=50, filter_size=(5, 5), outdir=None ) # our second pooling layer lenet.add( Pool2D, size=(2, 2) ) # now we need to flatten the 4D convolution outputs into 2D matrix (just flatten the trailing dimensions) lenet.add( Flatten, ndim=2 ) # one dense hidden layer lenet.add( Dense, outputs=500, activation='tanh', outdir=None ) # hook a softmax classification layer, outputting the probabilities. lenet.add( Softmax, outputs=10, out_as_probs=True, outdir=None ) # Grab the MNIST dataset data = MNIST(path="../../../datasets/{!s}".format(mnist_name), concat_train_valid=False, flatten=False) # define our loss to optimize for the model (and the target variable) # targets from MNIST are int64 numbers 0-9 y = lvector('y') loss = Neg_LL(inputs=lenet.get_outputs(), targets=y, one_hot=False) # monitor error_monitor = Monitor(name='error', expression=mean(neq(lenet.models[-1].y_pred, y)), valid=True, test=True, out_service=FileService('outputs/lenet_error.txt')) # optimize our model to minimize loss given the dataset using SGD optimizer = SGD(model=lenet, dataset=data, loss=loss, epochs=10, batch_size=128, learning_rate=.1, momentum=False) print("Training LeNet...") optimizer.train(monitor_channels=error_monitor) def test_subset(filename, expected, conf=0.001): with open(filename, 'r') as f: errs = [float(err) for err in f] for i, (err, exp) in enumerate(zip(errs, expected)): if i == 0: c = conf*10 else: c = conf self.assertTrue(exp-c < round(err, 4) < exp+c, "Errors: {!s} and Expected: {!s} -- Error at {!s} and {!s}".format( errs, expected, err, exp) ) test_subset('outputs/lenet_error_train.txt', [.0753, .0239, .0159, .0113, .0088, .0064, .0050, .0037, .0026, .0019] ) test_subset('outputs/lenet_error_valid.txt', [.0283, .0209, .0170, .0151, .0139, .0129, .0121, .0118, .0112, .0113] ) test_subset('outputs/lenet_error_test.txt', [.0319, .0213, .0167, .0134, .0122, .0119, .0116, .0107, .0104, .0105] ) shutil.rmtree('outputs/') finally: if 'lenet' in locals(): del lenet if 'data' in locals(): del data if 'y' in locals(): del y if 'x' in locals(): del x if 'loss' in locals(): del loss if 'optimizer' in locals(): del optimizer