示例#1
0
def getModel (n_input, n_output):
    model = Sequential()
    model.add(LSTM(32, input_shape=(int(n_input / conf['n_timesteps']), conf['n_timesteps'])))
    model.add(Dense(1))

    return model
    
示例#2
0
def getModel (n_input, n_output):
    model = Sequential()
    model.add(Dense(round(n_input * 2), activation='relu' , input_dim=n_input))
    model.add(Dense(n_input, activation="relu"))
    model.add(Dense(units=n_output, activation='softmax'))  

    return model
示例#3
0
def getModel (n_input, n_output):
    
    model = Sequential()
    #model.add(LSTM(round(n_input), input_shape=(conf['n_timesteps'], int(n_input / conf['n_timesteps']))))
    model.add(Dense(round(n_input), activation='relu' , input_dim=n_input))
    model.add(Dense(round(n_input/2), activation='relu'))
    model.add(Dense(units=n_output, activation='tanh'))

    return model
示例#4
0
def _get_simple_sequential_model(compile_metrics):
  model = Sequential()
  model.add(
      layers.Dense(
          3, activation='relu', input_dim=4, kernel_initializer='ones'))
  model.add(layers.Dense(1, activation='sigmoid', kernel_initializer='ones'))
  model.compile(
      loss='mae',
      metrics=compile_metrics,
      optimizer=RMSPropOptimizer(learning_rate=0.001))
  return model
示例#5
0
# ==========================================
if task is 'train':
    # preprocessing data =======================

    # in CNN, all input must be 4D, (n_sample, n_pixel_x, n_pixel_y, channel)
    (x_train, y_train), (x_test, y_test) = load_data()
    x_train = x_train.reshape(x_train.shape[0], 28, 28,
                              1).astype('float32') / 255
    x_test = x_test.reshape(x_test.shape[0], 28, 28, 1).astype('float32') / 255

    # one-hot encoding
    y_train = to_categorical(y_train)
    y_test = to_categorical(y_test)

    # build model =================================
    model = Sequential()

    # construct CNN part
    for n_filter in tot_filter:
        # convolution
        model.add(
            Conv2D(filters=n_filter,
                   kernel_size=filter_size,
                   padding=padding,
                   input_shape=x_train.shape[1:],
                   activation=cnn_activation))

        # maxpooling
        model.add(MaxPooling2D(pool_size=pool_size))

    # connect to DNN
示例#6
0
def evaluate_model(trainX, trainy, testX, testy):
    global best_accuracy
    verbose, epochs, batch_size = 0, 10, 32
    # n_timesteps, n_features, n_outputs = trainX.shape[1], trainX.shape[2], trainy.shape[1]
    model = Sequential()
    #     model.add(Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(n_timesteps,n_features)))  (x_train.shape[1],1)
    model.add(
        Conv1D(filters=64,
               kernel_size=3,
               activation='relu',
               input_shape=(trainX.shape[1], 1)))
    model.add(Conv1D(filters=64, kernel_size=3, activation='relu'))
    model.add(Dropout(0.5))
    model.add(MaxPooling1D(pool_size=2))
    model.add(Flatten())
    model.add(Dense(100, activation='relu'))
    model.add(Dense(trainy.shape[1], activation='softmax'))
    model.compile(loss='categorical_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy'])
    # fit network
    model.fit(trainX, trainy, epochs=epochs, batch_size=batch_size, verbose=1)
    # evaluate model
    _, accuracy = model.evaluate(testX,
                                 testy,
                                 batch_size=batch_size,
                                 verbose=0)
    if accuracy > best_accuracy:
        best_accuracy = accuracy
        model.save(BestModleFilePath)
    return accuracy
示例#7
0
def ffnthree(xtrain,
             ytrain,
             xtest,
             ytest,
             input_shape,
             num_classes,
             batch_size,
             epochs,
             callbacks,
             ismodelsaved=False,
             tl=False):
    if ismodelsaved == False:
        # model definition
        ffn3 = Sequential()
        ffn3.add(
            Dense(100,
                  input_dim=input_shape,
                  kernel_initializer="lecun_uniform",
                  activation="relu"))
        ffn3.add(BatchNormalization())
        ffn3.add(Dense(50, activation="relu", kernel_initializer="uniform"))
        ffn3.add(Dropout(0.5))
        ffn3.add(Dense(10, activation="relu", kernel_initializer="uniform"))
        ffn3.add(Dense(num_classes, activation='softmax'))
        #
        ffn3.compile(loss=binary_crossentropy,
                     optimizer=tf.keras.optimizers.RMSprop(0.001, rho=0.9),
                     metrics=['accuracy'])
        #
        historyffn3 = ffn3.fit(xtrain,
                               ytrain,
                               batch_size=batch_size,
                               epochs=epochs,
                               verbose=0,
                               validation_data=(xtest, ytest),
                               callbacks=callbacks)
        score = ffn3.evaluate(xtest, ytest, verbose=0)
        p('Test loss:', score[0])
        p('Test accuracy:', score[1])
        #
        # display learning curves
        if True:
            plt.figure()
            plt.plot(historyffn3.history['loss'], label='train loss')
            plt.plot(historyffn3.history['val_loss'], label='test loss')
            plt.title('Learning Curves')
            plt.xlabel('epochs')
            plt.ylabel('loss')
            plt.legend()
            plt.show()
    else:
        if input_shape == 92:
            ffn3 = tf.keras.models.load_model(flpath +
                                              'saved_model_4x23/ffn3_4x23')
        else:
            if tl:
                ffn3 = tf.keras.models.load_model(
                    flpath + 'saved_model_guideseq_8x23/ffn3_8x23')
            else:
                ffn3 = tf.keras.models.load_model(
                    flpath + 'saved_model_crispr_8x23/ffn3crispr_8x23')
    p("FFN3: Done")
    return ffn3
示例#8
0
import numpy as np
import pandas as pd

from keras.preprocessing.image import load_img
from keras.preprocessing.image import img_to_array
"""num classes will be equal to the number of categories we want to classify"""
num_classes = 12
"""
size of images is the default size that it was trained on ImageNet dataset
"""
image_size = 224
target_size = (image_size, image_size)
"""
we set up a sequential model that we can add layers to
"""
my_new_model = Sequential()
"""
first we add all of pre-trained  model
we've written include_top=False, this is how specify that we want to exlude
the layer that makes prediction into the thousands of categories used in the ImageNet competition
we set the weights to be 'ImageNet' to specify that we use the pre-traind model on ImageNet
pooling equals average says that if we had extra channels in our tensor at the end of this step
we want to collapse them to 1d tensor by taking an average across channels
now we have a pre-trained model that creates the layer before the last layer
that we saw in the slides
"""
my_new_model.add(
    MobileNet(weights='imagenet', include_top=False, pooling='avg'))
"""
we add a dense layer to make predictions,
we specify the number of nodes in this layer which in this case is
    def __init__(
            self, data, periods=288, batch_size=64, sequence_length=20,
            warmup_steps=50, epochs=20, display=False):
        """Instantiate the class.

        Args:
            data: Dict of values keyed by timestamp
            periods: Number of timestamp data points per vector
            batch_size: Size of batch
            sequence_length: Length of vectors for for each target
            warmup_steps:

        Returns:
            None

        """
        # Initialize key variables
        self.periods = periods
        self.target_names = ['value']
        self.warmup_steps = warmup_steps
        self.epochs = epochs
        self.batch_size = batch_size
        self.display = display

        ###################################
        # TensorFlow wizardry
        config = tf.ConfigProto()

        # Don't pre-allocate memory; allocate as-needed
        config.gpu_options.allow_growth = True

        # Only allow a total of half the GPU memory to be allocated
        config.gpu_options.per_process_gpu_memory_fraction = 0.95

        # Crash with DeadlineExceeded instead of hanging forever when your
        # queues get full/empty
        config.operation_timeout_in_ms = 60000

        # Create a session with the above options specified.
        backend.tensorflow_backend.set_session(tf.Session(config=config))
        ###################################

        # Get data
        (x_data, y_data) = convert_data(data, periods, self.target_names)

        print('\n> Numpy Data Type: {}'.format(type(x_data)))
        print("> Numpy Data Shape: {}".format(x_data.shape))
        print("> Numpy Data Row[0]: {}".format(x_data[0]))
        print('> Numpy Targets Type: {}'.format(type(y_data)))
        print("> Numpy Targets Shape: {}".format(y_data.shape))

        '''
        This is the number of observations (aka. data-points or samples) in
        the data-set:
        '''

        num_data = len(x_data)

        '''
        This is the fraction of the data-set that will be used for the
        training-set:
        '''

        train_split = 0.9

        '''
        This is the number of observations in the training-set:
        '''

        self.num_train = int(train_split * num_data)

        '''
        This is the number of observations in the test-set:
        '''

        num_test = num_data - self.num_train

        print('> Number of Samples: {}'.format(num_data))
        print("> Number of Training Samples: {}".format(self.num_train))
        print("> Number of Test Samples: {}".format(num_test))

        # Create test and training data
        x_train = x_data[0:self.num_train]
        x_test = x_data[self.num_train:]
        self.y_train = y_data[0:self.num_train]
        self.y_test = y_data[self.num_train:]
        self.num_x_signals = x_data.shape[1]
        self.num_y_signals = y_data.shape[1]

        print("> Training Minimum Value:", np.min(x_train))
        print("> Training Maximum Value:", np.max(x_train))

        '''
        steps_per_epoch is the number of batch iterations before a training
        epoch is considered finished.
        '''

        self.steps_per_epoch = int(self.num_train / batch_size) + 1
        print("> Epochs:", epochs)
        print("> Batch Size:", batch_size)
        print("> Steps:", self.steps_per_epoch)

        '''
        Calculate the estimated memory footprint.
        '''

        print("> Data size: {:.2f} Bytes".format(x_data.nbytes))

        '''
        if memory_footprint > 7:
            print('\n\n{}\n\n'.format(
                '> Estimated GPU memory usage too large. Use new parameters '
                'to reduce the footprint.'))
            sys.exit(0)
        '''

        '''
        The neural network works best on values roughly between -1 and 1, so we
        need to scale the data before it is being input to the neural network.
        We can use scikit-learn for this.

        We first create a scaler-object for the input-signals.

        Then we detect the range of values from the training-data and scale
        the training-data.
        '''

        x_scaler = MinMaxScaler()
        self.x_train_scaled = x_scaler.fit_transform(x_train)

        print('> Scaled Training Minimum Value: {}'.format(
            np.min(self.x_train_scaled)))
        print('> Scaled Training Maximum Value: {}'.format(
            np.max(self.x_train_scaled)))

        self.x_test_scaled = x_scaler.transform(x_test)

        '''
        The target-data comes from the same data-set as the input-signals,
        because it is the weather-data for one of the cities that is merely
        time-shifted. But the target-data could be from a different source with
        different value-ranges, so we create a separate scaler-object for the
        target-data.
        '''

        self.y_scaler = MinMaxScaler()
        self.y_train_scaled = self.y_scaler.fit_transform(self.y_train)
        y_test_scaled = self.y_scaler.transform(self.y_test)

        # Data Generator

        '''
        The data-set has now been prepared as 2-dimensional numpy arrays. The
        training-data has almost 300k observations, consisting of 20
        input-signals and 3 output-signals.

        These are the array-shapes of the input and output data:
        '''

        print('> Scaled Training Data Shape: {}'.format(
            self.x_train_scaled.shape))
        print('> Scaled Training Targets Shape: {}'.format(
            self.y_train_scaled.shape))

        # We then create the batch-generator.

        generator = self.batch_generator(batch_size, sequence_length)

        # Validation Set

        '''
        The neural network trains quickly so we can easily run many training
        epochs. But then there is a risk of overfitting the model to the
        training-set so it does not generalize well to unseen data. We will
        therefore monitor the model's performance on the test-set after each
        epoch and only save the model's weights if the performance is improved
        on the test-set.

        The batch-generator randomly selects a batch of short sequences from
        the training-data and uses that during training. But for the
        validation-data we will instead run through the entire sequence from
        the test-set and measure the prediction accuracy on that entire
        sequence.
        '''

        validation_data = (np.expand_dims(self.x_test_scaled, axis=0),
                           np.expand_dims(y_test_scaled, axis=0))

        # Create the Recurrent Neural Network

        self.model = Sequential()

        '''
        We can now add a Gated Recurrent Unit (GRU) to the network. This will
        have 512 outputs for each time-step in the sequence.

        Note that because this is the first layer in the model, Keras needs to
        know the shape of its input, which is a batch of sequences of arbitrary
        length (indicated by None), where each observation has a number of
        input-signals (num_x_signals).
        '''

        self.model.add(GRU(
            units=512,
            return_sequences=True,
            input_shape=(None, self.num_x_signals,)))

        '''
        The GRU outputs a batch of sequences of 512 values. We want to predict
        3 output-signals, so we add a fully-connected (or dense) layer which
        maps 512 values down to only 3 values.

        The output-signals in the data-set have been limited to be between 0
        and 1 using a scaler-object. So we also limit the output of the neural
        network using the Sigmoid activation function, which squashes the
        output to be between 0 and 1.'''

        self.model.add(Dense(self.num_y_signals, activation='sigmoid'))

        '''
        A problem with using the Sigmoid activation function, is that we can
        now only output values in the same range as the training-data.

        For example, if the training-data only has temperatures between -20
        and +30 degrees, then the scaler-object will map -20 to 0 and +30 to 1.
        So if we limit the output of the neural network to be between 0 and 1
        using the Sigmoid function, this can only be mapped back to temperature
        values between -20 and +30.

        We can use a linear activation function on the output instead. This
        allows for the output to take on arbitrary values. It might work with
        the standard initialization for a simple network architecture, but for
        more complicated network architectures e.g. with more layers, it might
        be necessary to initialize the weights with smaller values to avoid
        NaN values during training. You may need to experiment with this to
        get it working.
        '''

        if False:
            # Maybe use lower init-ranges.
            # init = RandomUniform(minval=-0.05, maxval=0.05)
            init = RandomUniform(minval=-0.05, maxval=0.05)

            self.model.add(Dense(
                self.num_y_signals,
                activation='linear',
                kernel_initializer=init))

        # Compile Model

        '''
        This is the optimizer and the beginning learning-rate that we will use.
        We then compile the Keras model so it is ready for training.
        '''
        optimizer = RMSprop(lr=1e-3)
        self.model.compile(loss=self.loss_mse_warmup, optimizer=optimizer)

        '''
        This is a very small model with only two layers. The output shape of
        (None, None, 3) means that the model will output a batch with an
        arbitrary number of sequences, each of which has an arbitrary number of
        observations, and each observation has 3 signals. This corresponds to
        the 3 target signals we want to predict.
        '''
        print('> Model Summary:\n')
        print(self.model.summary())

        # Callback Functions

        '''
        During training we want to save checkpoints and log the progress to
        TensorBoard so we create the appropriate callbacks for Keras.

        This is the callback for writing checkpoints during training.
        '''

        path_checkpoint = '/tmp/23_checkpoint.keras'
        callback_checkpoint = ModelCheckpoint(filepath=path_checkpoint,
                                              monitor='val_loss',
                                              verbose=1,
                                              save_weights_only=True,
                                              save_best_only=True)

        '''
        This is the callback for stopping the optimization when performance
        worsens on the validation-set.
        '''

        callback_early_stopping = EarlyStopping(monitor='val_loss',
                                                patience=5, verbose=1)

        '''
        This is the callback for writing the TensorBoard log during training.
        '''

        callback_tensorboard = TensorBoard(log_dir='/tmp/23_logs/',
                                           histogram_freq=0,
                                           write_graph=False)

        '''
        This callback reduces the learning-rate for the optimizer if the
        validation-loss has not improved since the last epoch
        (as indicated by patience=0). The learning-rate will be reduced by
        multiplying it with the given factor. We set a start learning-rate of
        1e-3 above, so multiplying it by 0.1 gives a learning-rate of 1e-4.
        We don't want the learning-rate to go any lower than this.
        '''

        callback_reduce_lr = ReduceLROnPlateau(monitor='val_loss',
                                               factor=0.1,
                                               min_lr=1e-4,
                                               patience=0,
                                               verbose=1)

        callbacks = [callback_early_stopping,
                     callback_checkpoint,
                     callback_tensorboard,
                     callback_reduce_lr]

        # Train the Recurrent Neural Network

        '''We can now train the neural network.

        Note that a single "epoch" does not correspond to a single processing
        of the training-set, because of how the batch-generator randomly
        selects sub-sequences from the training-set. Instead we have selected
        steps_per_epoch so that one "epoch" is processed in a few minutes.

        With these settings, each "epoch" took about 2.5 minutes to process on
        a GTX 1070. After 14 "epochs" the optimization was stopped because the
        validation-loss had not decreased for 5 "epochs". This optimization
        took about 35 minutes to finish.

        Also note that the loss sometimes becomes NaN (not-a-number). This is
        often resolved by restarting and running the Notebook again. But it may
        also be caused by your neural network architecture, learning-rate,
        batch-size, sequence-length, etc. in which case you may have to modify
        those settings.
        '''

        print('\n> Starting data training\n')

        try:
            self.model.fit_generator(
                generator=generator,
                epochs=self.epochs,
                steps_per_epoch=self.steps_per_epoch,
                validation_data=validation_data,
                callbacks=callbacks)
        except Exception as error:
            print('\n>{}\n'.format(error))
            traceback.print_exc()
            sys.exit(0)

        # Load Checkpoint

        '''
        Because we use early-stopping when training the model, it is possible
        that the model's performance has worsened on the test-set for several
        epochs before training was stopped. We therefore reload the last saved
        checkpoint, which should have the best performance on the test-set.
        '''

        print('> Loading model weights')

        try:
            self.model.load_weights(path_checkpoint)
        except Exception as error:
            print('\n> Error trying to load checkpoint.\n\n{}'.format(error))
            traceback.print_exc()
            sys.exit(0)

        # Performance on Test-Set

        '''
        We can now evaluate the model's performance on the test-set. This
        function expects a batch of data, but we will just use one long
        time-series for the test-set, so we just expand the
        array-dimensionality to create a batch with that one sequence.
        '''

        result = self.model.evaluate(
            x=np.expand_dims(self.x_test_scaled, axis=0),
            y=np.expand_dims(y_test_scaled, axis=0))

        print('> Loss (test-set): {}'.format(result))

        # If you have several metrics you can use this instead.
        if False:
            for res, metric in zip(result, self.model.metrics_names):
                print('{0}: {1:.3e}'.format(metric, res))
示例#10
0
def lenet(network_input: NetworkInput) -> Model:
    model = Sequential()

    model.add(layers.Lambda(lambda x: norm(x, network_input.mean, network_input.std), input_shape=network_input.input_shape, output_shape=network_input.input_shape))
    model.add(layers.Conv2D(32, kernel_size=(3, 3), activation='relu'))
    
    model.add(layers.Conv2D(64, (3, 3), activation='relu'))
    model.add(layers.MaxPooling2D(pool_size=(2, 2)))
    model.add(layers.Dropout(0.2))
    
    model.add(layers.Flatten())
    model.add(layers.Dense(128, activation='relu'))
    model.add(layers.Dropout(0.2))
    
    model.add(layers.Dense(network_input.number_of_classes, activation='softmax'))

    return model
示例#11
0
class modeloSNN():
    """Clase modelo.json SNN"""

    Selectedmodel = Sequential()
    preprocesador1 = make_column_transformer(
        (StandardScaler(), ['Age', 'Fare']),
        (OneHotEncoder(), ['Pclass', 'Sex', 'Embarked']))

    def suma(num1=0, num2=0):
        resultado = num1 + num2
        return resultado

    def cargarRNN(nombreArchivoModelo, nombreArchivoPesos):
        K.reset_uids()
        # Cargar la Arquitectura desde el archivo JSON
        with open(nombreArchivoModelo + '.json', 'r') as f:
            model = model_from_json(f.read())
        # Cargar Pesos (weights) en el nuevo modelo.json
        model.load_weights(nombreArchivoPesos + '.h5')
        print("Red Neuronal Cargada desde Archivo")
        return model

    def predecirSobrevivencia(self,
                              Pclass=1,
                              Sex='female',
                              Age=60,
                              Fare=0,
                              Embarked='C'):
        #Modelo optimizado
        print('MODELO OPTIMIZADO')
        nombreArchivoModelo = r'apiSNN/Logica/arquitectura_optimizada'
        nombreArchivoPesos = r'apiSNN/Logica/pesos_optimizados'
        #return (str(pathlib.Path().absolute())+'\Modelos')
        self.Selectedmodel = self.cargarRNN(nombreArchivoModelo,
                                            nombreArchivoPesos)
        print(self.Selectedmodel)
        print(self.Selectedmodel.summary())
        self.preprocesamiento(self)
        resultado = self.predict(self, Pclass, Sex, Age, Fare, Embarked)
        resultado = resultado[0, 0]
        print('Predicción:', resultado)
        #print('Predicción:',self.predict(self,Age=32 ,Fare=9))
        #print('Predicción:',self.predict(self,Pclass=1, Sex='female', Age=60 ,Fare=0, Embarked='C'))
        #print('Predicción:',self.predict(self,Pclass=3, Sex='female', Age=78 ,Fare=4563, Embarked='Q'))
        mensaje = ''
        if resultado == 1:
            mensaje = 'Sobrevive'
        else:
            mensaje = 'No sobrevive'
        return mensaje

    def predict(self, Pclass=1, Sex='female', Age=60, Fare=0, Embarked='C'):
        cnames = ['Pclass', 'Sex', 'Age', 'Fare', 'Embarked']
        data = [[Pclass, Sex, Age, Fare, Embarked]]
        my_X = pd.DataFrame(data=data, columns=cnames)
        my_X = self.preprocesador1.transform(my_X)
        Survived = self.Selectedmodel.predict_classes(my_X)
        dbReg = models.Persona(pclass=Pclass,
                               sex=Sex,
                               age=Age,
                               fare=Fare,
                               embarked=Embarked,
                               survived=Survived)
        dbReg.save()
        return Survived

    def preprocesamiento(self):
        df = pd.read_csv('apiSNN/Datasets/titanic/train.csv')
        df.head()
        df = df.dropna(
            subset=['Pclass', 'Sex', 'Age', 'Embarked', 'Fare', 'SibSp'])
        df.head()
        Xsubset = df[['Pclass', 'Sex', 'Age', 'Fare', 'Embarked']]
        y = df.Survived.values
        self.preprocesador1.fit_transform(Xsubset)
示例#12
0
def fit_lstm(train, batch_size, nb_epoch, neurons):
    X, y = train[:, 0:-1], train[:, -1]
    X = X.reshape(X.shape[0], 1, X.shape[1])
    model = Sequential()
    model.add(
        LSTM(neurons,
             batch_input_shape=(batch_size, X.shape[1], X.shape[2]),
             stateful=True))
    model.add(Dense(1))
    model.compile(loss='mean_squared_error', optimizer='adam')
    for i in range(nb_epoch):
        model.fit(X,
                  y,
                  epochs=1,
                  batch_size=batch_size,
                  verbose=0,
                  shuffle=False)
        model.reset_states()
    return model
示例#13
0
    (data.shape[0] / 14 * 4))  # timesteps * number of rows for 13 participants
values = reframed.values
train = values[:train_shape, :]
test = values[train_shape:, :]

# split into input and outputs
n_obs = n_secs * n_features
train_X, train_y = train[:, :n_obs], train[:, -n_features]
test_X, test_y = test[:, :n_obs], test[:, -n_features]

train_X = train_X.reshape((train_X.shape[0], n_secs, n_features))
test_X = test_X.reshape((test_X.shape[0], n_secs, n_features))
print(train_X.shape, train_y.shape, test_X.shape, test_y.shape)

# design network
model = Sequential()
model.add(LSTM(50, input_shape=(train_X.shape[1], train_X.shape[2])))
model.add(Dense(1))
model.compile(loss='mae', optimizer='adam')
# fit network
history = model.fit(train_X,
                    train_y,
                    epochs=50,
                    batch_size=32,
                    validation_data=(test_X, test_y),
                    verbose=2,
                    shuffle=False)

# plot history
plt.plot(history.history['loss'], label='train')
plt.plot(history.history['val_loss'], label='test')
示例#14
0
test_images = test_images.reshape(test_images.shape[0], 28, 28, 1)

# Normalización: los datos pasan a tener un valor entre [0, 1]
training_images = training_images / 255
test_images = test_images / 255
"""## CONJUNTO DE VALIDACIÓN"""

# Se separa el conjunto de validación y de entrenamiento
validation_images = training_images[-10000:]
validation_labels = training_labels[-10000:]

training_images = training_images[:50000]
training_labels = training_labels[:50000]
"""## ENTRENAMIENTO"""

model = Sequential()

# Se añaden las capas
model.add(
    Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=(28, 28, 1)))
model.add(Conv2D(32, kernel_size=(1, 1), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))
model.add(Conv2D(64, kernel_size=(1, 1), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(128, activation='relu'))
model.add(Dense(10, activation='softmax'))
示例#15
0
    def build(input_shape_width, input_shape_height, classes, 
              weight_path = '', input_shape_depth = 3):
        '''
        weight_path: a .hdf5 file. If exists, we can load model.
        '''
        
        # initialize the model
        model = Sequential()
        
        input_shape = (input_shape_height, input_shape_width, 
                       input_shape_depth)
        # if we are using "channels first", update the input shape
        if K.image_data_format() == 'channels_first':
             input_shape = (input_shape_depth, input_shape_height, 
                            input_shape_width)
        
        # first Convolution + relu + pooling layer
        model.add(Conv2D(filters = 20, kernel_size = (5, 5), 
                         padding = 'same', input_shape = input_shape))
        model.add(Activation('relu'))
        model.add(MaxPooling2D(pool_size = (2, 2), strides=(2, 2)))
        
        # second convolutional layer
        model.add(Conv2D(filters = 50, kernel_size = (5, 5), 
                         padding = 'same'))
        model.add(Activation('relu'))
        model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
        
        # Flattening
        model.add(Flatten())

        # Full connection
        model.add(Dense(units = 500))
        model.add(Activation('relu'))

        # output layer
        model.add(Dense(units = classes))
        model.add(Activation('softmax'))

        if weight_path:
            model.load_weights(weight_path)

        # return the constructed network architecture
        return model
def model_v3(top_layer_units):
    model = Sequential()

    model.add(
        Conv2D(filters=16,
               kernel_size=(7, 7),
               strides=(1, 1),
               input_shape=(NUM_MFCC, NUM_FRAMES, 1),
               activation=tf.nn.relu))
    model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same'))
    model.add(BatchNormalization())

    model.add(Conv2D(filters=32, kernel_size=(5, 5), strides=(1, 1)))
    model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same'))
    model.add(BatchNormalization())

    model.add(Conv2D(filters=32, kernel_size=(3, 3), strides=(1, 1)))
    model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same'))
    model.add(BatchNormalization())

    model.add(Conv2D(filters=32, kernel_size=(3, 3), strides=(1, 1)))
    model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same'))
    # model.add(BatchNormalization())

    model.add(LSTM(50, return_sequences=True))
    model.add(Flatten())
    model.add(Dropout(0.3))

    model.add(
        Dense(units=top_layer_units,
              activation=tf.nn.softmax,
              name='top_layer'))

    return model
def model_v2(top_layer_units):
    model = Sequential()

    model.add(
        TimeDistributed(Conv1D(filters=16,
                               kernel_size=4,
                               padding='same',
                               activation=tf.nn.relu,
                               data_format='channels_last'),
                        input_shape=(NUM_MFCC, NUM_FRAMES, 1)))

    model.add(
        TimeDistributed(
            Conv1D(filters=8,
                   kernel_size=2,
                   padding='same',
                   activation=tf.nn.relu)))
    model.add(TimeDistributed(MaxPooling1D(pool_size=2)))
    model.add(TimeDistributed(Flatten()))
    model.add(LSTM(50, return_sequences=True))
    model.add(Dropout(0.3))
    model.add(Flatten())
    model.add(Dense(units=512, activation=tf.nn.tanh))
    model.add(Dense(units=256, activation=tf.nn.tanh))
    model.add(
        Dense(units=top_layer_units,
              activation=tf.nn.softmax,
              name='top_layer'))
    return model
示例#18
0
def google_net(size=256, kernel=3):
    model = Sequential()
    model.add(Conv2D(32, (kernel, kernel), 
                     activation='relu', 
                     input_shape=(size, size, 3),
                     strides=2,
                     kernel_regularizer=regularizers.l2(0.01),
                     name='cv1'))
    model.add(BatchNormalization())
    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Conv2D(64, (kernel, kernel), 
                     activation='relu',
                     strides=2,
                     kernel_regularizer=regularizers.l2(0.01),
                     name='cv2'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Conv2D(128, (kernel, kernel), activation='relu', strides=2, name='cv3.3'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Flatten())
    model.add(Dense(256, kernel_regularizer=regularizers.l2(0.01),  name='features'))
    model.add(Activation('relu'))
    model.add(Dense(3, activation='softmax', name='denseout'))

    print(model.summary())

    model.compile(
        loss='categorical_crossentropy',
        optimizer=RMSprop(lr=1e-4, decay=0.1e-6),
        metrics=['accuracy'])

    return model
def get_model_1(x,y,Vocab_size,maxlen):
    model = Sequential()
    model.add(Embedding(Vocab_size, 50, input_length=maxlen))
    model.add(LSTM(128, return_sequences=True))
    model.add(LSTM(128))
    model.add(Dense(128, activation='relu'))
    model.add(Dense(Vocab_size))
    model.add(Activation("softmax"))
    print(model.summary())
    model.compile(loss="sparse_categorical_crossentropy", optimizer='adam',  metrics=['accuracy'])
    return model
#padding
from tensorflow.python.keras.preprocessing.sequence import pad_sequences

x_train = pad_sequences(x_train, value = word_index['the'], padding = 'post', maxlen = 256)
x_test = pad_sequences(x_test, value = word_index['the'], padding = 'post', maxlen = 256)
#Showing lengths after padding
show_lengths()


#Creating and training model with 20 epchos and spilited to 75% for training and 25% for testing
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Embedding, Dense, GlobalAveragePooling1D

model = Sequential([
    Embedding(10000, 16),
    GlobalAveragePooling1D(),
    Dense(16, activation = 'relu'),
    Dense(1, activation = 'sigmoid')
])

model.compile(
    optimizer = 'adam',
    loss = 'binary_crossentropy',
    metrics = ['acc']
)

model.summary()

from tensorflow.python.keras.callbacks import LambdaCallback

simple_logging = LambdaCallback(on_epoch_end = lambda e, l: print(e, end='.'))
def keras_model_fn(hyperparameters):
    """keras_model_fn receives hyperparameters from the training job and returns a compiled keras model.
    The model will be transformed into a TensorFlow Estimator before training and it will be saved in a
    TensorFlow Serving SavedModel at the end of training.
    Args:
        hyperparameters: The hyperparameters passed to the SageMaker TrainingJob that runs your TensorFlow
                         training script.
    Returns: A compiled Keras model
    """
    model = Sequential()

    # TensorFlow Serving default prediction input tensor name is PREDICT_INPUTS.
    # We must conform to this naming scheme.
    model.add(InputLayer(input_shape=(HEIGHT, WIDTH, DEPTH), name=PREDICT_INPUTS))
    model.add(Conv2D(32, (3, 3), padding='same'))
    model.add(Activation('relu'))
    model.add(Conv2D(32, (3, 3)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Conv2D(64, (3, 3), padding='same'))
    model.add(Activation('relu'))
    model.add(Conv2D(64, (3, 3)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Flatten())
    model.add(Dense(512))
    model.add(Activation('relu'))
    model.add(Dropout(0.5))
    model.add(Dense(NUM_CLASSES))
    model.add(Activation('softmax'))

    _model = tf.keras.Model(inputs=model.input, outputs=model.output)

    opt = RMSprop(lr=hyperparameters['learning_rate'], decay=hyperparameters['decay'])

    _model.compile(loss='categorical_crossentropy',
                   optimizer=opt,
                   metrics=['accuracy'])

    return _model
示例#22
0
entrenamiento_generador = entrenamiento_datagen.flow_from_directory(
    data_entrenamiento,
    target_size=(altura, longitud),
    batch_size=batch_size,
    class_mode='categorical')

validacion_generador = test_datagen.flow_from_directory(
    data_validacion,
    target_size=(altura, longitud),
    batch_size=batch_size,
    class_mode='categorical')

##Creacion red nueronal

cnn = Sequential()
cnn.add(
    Convolution2D(filtrosConv1,
                  tamano_filtro1,
                  padding="same",
                  input_shape=(longitud, altura, 3),
                  activation='relu'))
cnn.add(MaxPooling2D(pool_size=tamano_pool))

cnn.add(Convolution2D(filtrosConv2, tamano_filtro2, padding="same"))
cnn.add(MaxPooling2D(pool_size=tamano_pool))

cnn.add(Flatten())
cnn.add(Dense(256, activation='relu'))
cnn.add(Dropout(0.5))
cnn.add(Dense(clases, activation='softmax'))
示例#23
0
def keras_model_regression(input_dim):
    nn_deep_model = Sequential()
    nn_deep_model.add(Dense(288, input_dim=input_dim, activation='relu'))
    nn_deep_model.add(Dense(144, activation='relu'))
    nn_deep_model.add(Dropout(0.5))
    nn_deep_model.add(Dense(12, activation='relu'))
    nn_deep_model.add(Dense(1, activation='sigmoid'))
    

    model_optimizer  = optimizers.Adam(lr=0.001)
    nn_deep_model.compile(loss='mean_squared_error', optimizer=model_optimizer, metrics=['mae'])


    return nn_deep_model
示例#24
0
def keras_model_classification(num_classes, input_dim):
    # nn_deep_model = OverwrittenSequentialClassifier()
    nn_deep_model = Sequential()

    nn_deep_model.add(Dense(288, input_dim=input_dim, activation='relu'))
    nn_deep_model.add(Dense(144, activation='relu'))
    nn_deep_model.add(Dropout(0.5))
    nn_deep_model.add(Dense(12, activation='relu'))
    nn_deep_model.add(Dense(num_classes, activation='softmax'))

    model_optimizer = optimizers.Adam(lr=0.001)
    nn_deep_model.compile(loss='mean_squared_error', optimizer=model_optimizer, metrics=['accuracy'])

    return nn_deep_model
示例#25
0
def create_CNNmodel():
    # Model
    model = Sequential()
    # Add convolution 2D
    model.add(
        Conv2D(32,
               kernel_size=(3, 3),
               activation='relu',
               kernel_initializer='he_normal',
               input_shape=(ROWS, COLS, 1)))
    model.add(MaxPooling2D((2, 2)))
    model.add(Flatten())
    model.add(Dense(128, activation='relu'))
    model.add(Dense(CLASSES, activation='softmax'))

    model.compile(loss=keras.losses.categorical_crossentropy,
                  optimizer='adam',
                  metrics=['accuracy'])

    model.summary()
    return model
示例#26
0
def main_fun(args, ctx):
    import numpy
    import os
    import tensorflow as tf
    from tensorflow.python import keras
    from tensorflow.python.keras import backend as K
    from tensorflow.python.keras.datasets import mnist
    from tensorflow.python.keras.models import Sequential, load_model, save_model
    from tensorflow.python.keras.layers import Dense, Dropout
    from tensorflow.python.keras.optimizers import RMSprop
    from tensorflow.python.keras.callbacks import LambdaCallback, TensorBoard
    from tensorflow.python.saved_model import builder as saved_model_builder
    from tensorflow.python.saved_model import tag_constants
    from tensorflow.python.saved_model.signature_def_utils_impl import predict_signature_def
    from tensorflowonspark import TFNode

    cluster, server = TFNode.start_cluster_server(ctx)

    if ctx.job_name == "ps":
        server.join()
    elif ctx.job_name == "worker":

        def generate_rdd_data(tf_feed, batch_size):
            print("generate_rdd_data invoked")
            while True:
                batch = tf_feed.next_batch(batch_size)
                imgs = []
                lbls = []
                for item in batch:
                    imgs.append(item[0])
                    lbls.append(item[1])
                images = numpy.array(imgs).astype('float32') / 255
                labels = numpy.array(lbls).astype('float32')
                yield (images, labels)

        with tf.device(
                tf.train.replica_device_setter(
                    worker_device="/job:worker/task:%d" % ctx.task_index,
                    cluster=cluster)):

            IMAGE_PIXELS = 28
            batch_size = 100
            num_classes = 10

            # the data, shuffled and split between train and test sets
            if args.input_mode == 'tf':
                (x_train, y_train), (x_test, y_test) = mnist.load_data()
                x_train = x_train.reshape(60000, 784)
                x_test = x_test.reshape(10000, 784)
                x_train = x_train.astype('float32') / 255
                x_test = x_test.astype('float32') / 255

                # convert class vectors to binary class matrices
                y_train = keras.utils.to_categorical(y_train, num_classes)
                y_test = keras.utils.to_categorical(y_test, num_classes)
            else:  # args.mode == 'spark'
                x_train = tf.placeholder(tf.float32,
                                         [None, IMAGE_PIXELS * IMAGE_PIXELS],
                                         name="x_train")
                y_train = tf.placeholder(tf.float32, [None, 10],
                                         name="y_train")
                (_, _), (x_test, y_test) = mnist.load_data()
                x_test = x_test.reshape(10000, 784)
                y_test = keras.utils.to_categorical(y_test, num_classes)

            model = Sequential()
            model.add(Dense(512, activation='relu', input_shape=(784, )))
            model.add(Dropout(0.2))
            model.add(Dense(512, activation='relu'))
            model.add(Dropout(0.2))
            model.add(Dense(10, activation='softmax'))

            model.summary()

            model.compile(loss='categorical_crossentropy',
                          optimizer=RMSprop(),
                          metrics=['accuracy'])

        saver = tf.train.Saver()

        with tf.Session(server.target) as sess:
            K.set_session(sess)

            def save_checkpoint(epoch, logs=None):
                if epoch == 1:
                    tf.train.write_graph(sess.graph.as_graph_def(),
                                         args.model_dir, 'graph.pbtxt')
                saver.save(sess,
                           os.path.join(args.model_dir, 'model.ckpt'),
                           global_step=epoch * args.steps_per_epoch)

            ckpt_callback = LambdaCallback(on_epoch_end=save_checkpoint)
            tb_callback = TensorBoard(log_dir=args.model_dir,
                                      histogram_freq=1,
                                      write_graph=True,
                                      write_images=True)

            # add callbacks to save model checkpoint and tensorboard events (on worker:0 only)
            callbacks = [ckpt_callback, tb_callback
                         ] if ctx.task_index == 0 else None

            if args.input_mode == 'tf':
                # train & validate on in-memory data
                model.fit(x_train,
                          y_train,
                          batch_size=batch_size,
                          epochs=args.epochs,
                          verbose=1,
                          validation_data=(x_test, y_test),
                          callbacks=callbacks)
            else:  # args.input_mode == 'spark':
                # train on data read from a generator which is producing data from a Spark RDD
                tf_feed = TFNode.DataFeed(ctx.mgr)
                model.fit_generator(generator=generate_rdd_data(
                    tf_feed, batch_size),
                                    steps_per_epoch=args.steps_per_epoch,
                                    epochs=args.epochs,
                                    verbose=1,
                                    validation_data=(x_test, y_test),
                                    callbacks=callbacks)

            if args.export_dir and ctx.job_name == 'worker' and ctx.task_index == 0:
                # save a local Keras model, so we can reload it with an inferencing learning_phase
                save_model(model, "tmp_model")

                # reload the model
                K.set_learning_phase(False)
                new_model = load_model("tmp_model")

                # export a saved_model for inferencing
                builder = saved_model_builder.SavedModelBuilder(
                    args.export_dir)
                signature = predict_signature_def(
                    inputs={'images': new_model.input},
                    outputs={'scores': new_model.output})
                builder.add_meta_graph_and_variables(
                    sess=sess,
                    tags=[tag_constants.SERVING],
                    signature_def_map={'predict': signature},
                    clear_devices=True)
                builder.save()

            if args.input_mode == 'spark':
                tf_feed.terminate()
示例#27
0
def sentiment_analysis(num_words, max_tokens):
    model = Sequential()

    #Embedding Layer. This layer will output the word vectors for each one of the words in the sentence
    embedding_size = 8
    model.add(Embedding(input_dim=num_words, 
                        output_dim=embedding_size,
                       input_length=max_tokens,
                       name='embedding_layer'))

    model.add(LSTM(units=16, return_sequences=True))
    model.add(LSTM(units=8, return_sequences=True))
    model.add(LSTM(units=4, return_sequences=False))
    model.add(Dense(1, activation='sigmoid'))

    optimizer = Adam(lr=0.001)
    model.compile(loss='binary_crossentropy',
                  optimizer=optimizer,
                  metrics=['accuracy'])
    
    return model
示例#28
0
                                                    train_target,
                                                    test_size=0.1,
                                                    random_state=12)
# 查看训练样本,确认无误
print(reverse_tokens(X_train[35]))
print('class: ',y_train[35])


# 用LSTM对样本进行分类
# 现在我们用keras搭建LSTM模型,模型的第一层是Embedding层,
# 只有当我们把tokens索引转换为词向量矩阵之后,才可以用神经网络对文本进行处理。
#  keras提供了Embedding接口,避免了繁琐的稀疏矩阵操作。
# 在Embedding层我们输入的矩阵为:(batchsize, maxtokens)
#  输出矩阵为: $$(batchsize, maxtokens, embeddingdim)$$

model = Sequential()

# 模型第一层为embedding,trainable=false 不训练embedding层
model.add(Embedding(num_words,
                    embedding_dim,
                    weights=[embedding_matrix],
                    input_length=max_tokens,
                    trainable=False))
# 加lstm层,有32个训练单元
model.add(Bidirectional(LSTM(units=32, return_sequences=True)))
model.add(LSTM(units=16, return_sequences=False))

# 注:构建模型
# 我在这个教程中尝试了几种神经网络结构,因为训练样本比较少,所以我们可以尽情尝试,训练过程等待时间并不长:
# (1)GRU:如果使用GRU的话,测试样本可以达到87%的准确率,但我测试自己的文本内容时发现,GRU最后一层激活函数的输出都在0.5左右,说明模型的判断不是很明确,信心比较低,而且经过测试发现模型对于否定句的判断有时会失误,我们期望对于负面样本输出接近0,正面样本接近1而不是都徘徊于0.5之间。
# (2)BiLSTM:测试了LSTM和BiLSTM,发现BiLSTM的表现最好,LSTM的表现略好于GRU,这可能是因为BiLSTM对于比较长的句子结构有更好的记忆,有兴趣的朋友可以深入研究一下。
# loading in the data
(X_train, y_train), (X_test, y_test) = cifar10.load_data()

# normalize the inputs from 0-255 to between 0 and 1 by dividing by 255
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train = X_train / 255.0
X_test = X_test / 255.0

# one hot encode outputs
y_train = np_utils.to_categorical(y_train)
y_test = np_utils.to_categorical(y_test)
class_num = y_test.shape[1]

# Create the model
model = Sequential()

# Design the model
#Convolutional layer
model.add(
    Conv2D(32, (3, 3),
           input_shape=(32, 32, 3),
           activation='relu',
           padding='same'))
model.add(Dropout(0.2))
model.add(BatchNormalization())

model.add(Conv2D(64, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
# number of unique categories in data
num_classes = 6

# reshaping the data to feed in the CNN
# the four parameters are no. of images, width,height,channels

x = x.reshape(-1, 128, 128, 1)

train_x = x[:]
train_y = y[:]

train_x, train_y = shuffle(train_x, train_y)

# model architechture

model = Sequential()

# Input layer 1

model.add(
    Conv2D(128,
           kernel_size=(7, 7),
           activation='relu',
           input_shape=(128, 128, 1)))
model.add(MaxPooling2D(pool_size=(3, 3)))

# layer 2
model.add(Conv2D(256, (7, 7), activation='relu'))

model.add(MaxPooling2D(pool_size=(3, 3)))
epochs = 100

x = x/255.0

#the best parameter setting from model4
dense_layers = [1]
layer_sizes = [32]
conv_layers = [3]

for dense_layer in dense_layers:
    for layer_size in layer_sizes:
        for conv_layer in conv_layers:
            name = "{}-conv-{}-nodes-{}-dense-{}".format(conv_layer, layer_size, dense_layer, int(time.time()))
            print(name)

            model = Sequential()
            model.add(Conv2D(layer_size, (3,3), input_shape = x.shape[1:]))
            model.add(Activation("relu"))
            model.add(MaxPooling2D(pool_size=(2,2)))
            model.add(Dropout(0.2))

            for l in range(conv_layer-1):
                model.add(Conv2D(layer_size, (3,3)))
                model.add(Activation("relu"))
                model.add(Conv2D(layer_size, (3,3)))
                model.add(Activation("relu"))
                model.add(MaxPooling2D(pool_size=(2,2)))
                model.add(Dropout(0.2))

            model.add(Flatten())
            for l in range(dense_layer):

# define dataset
seed(1)
n_samples = 1000
n_numbers = 2
largest = 10
alphabet = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', ' ']
n_chars = len(alphabet)
n_in_seq_length = n_numbers * ceil(log10(largest + 1)) + n_numbers - 1
n_out_seq_length = ceil(log10(n_numbers * (largest + 1)))
# define LSTM configuration
n_batch = 10
n_epoch = 30
# create LSTM
model = Sequential()
model.add(LSTM(100, input_shape=(n_in_seq_length, n_chars)))
model.add(RepeatVector(n_out_seq_length))
model.add(LSTM(50, return_sequences=True))
model.add(TimeDistributed(Dense(n_chars, activation='softmax')))
model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])
print(model.summary())
# train LSTM
for i in range(n_epoch):
    X, y = generate_data(n_samples, n_numbers, largest, alphabet)
    print(i)
    model.fit(X, y, epochs=1, batch_size=n_batch)

# evaluate on some new patterns
示例#33
0
class RNNGRU(object):
    """Process data for ingestion."""

    def __init__(
            self, data, sequence_length=20, warmup_steps=50, dropout=0,
            layers=1, patience=10, units=512, display=False):
        """Instantiate the class.

        Args:
            data: Tuple of (x_data, y_data, target_names)
            batch_size: Size of batch
            sequence_length: Length of vectors for for each target
            warmup_steps:

        Returns:
            None

        """
        # Initialize key variables
        self._warmup_steps = warmup_steps
        self._data = data
        self.display = display
        path_checkpoint = '/tmp/checkpoint.keras'
        _layers = int(abs(layers))

        # Delete any stale checkpoint file
        if os.path.exists(path_checkpoint) is True:
            os.remove(path_checkpoint)

        ###################################
        # TensorFlow wizardry
        config = tf.ConfigProto()

        # Don't pre-allocate memory; allocate as-needed
        config.gpu_options.allow_growth = True

        # Only allow a total of half the GPU memory to be allocated
        config.gpu_options.per_process_gpu_memory_fraction = 0.95

        # Crash with DeadlineExceeded instead of hanging forever when your
        # queues get full/empty
        config.operation_timeout_in_ms = 60000

        # Create a session with the above options specified.
        backend.tensorflow_backend.set_session(tf.Session(config=config))
        ###################################

        # Get data
        self._y_current = self._data.close()

        # Create training arrays
        x_train = self._data.vectors_train()
        self._y_train = self._data.classes_train()

        # Create test arrays for VALIDATION and EVALUATION
        xv_test = self._data.vectors_test()
        self._yv_test = self._data.classes_test()

        (self.training_rows, self._training_vector_count) = x_train.shape
        (self.test_rows, _) = xv_test.shape
        (_, self._training_class_count) = self._y_train.shape

        # Print stuff
        print('\n> Numpy Data Type: {}'.format(type(x_train)))
        print("> Numpy Data Shape: {}".format(x_train.shape))
        print("> Numpy Data Row[0]: {}".format(x_train[0]))
        print("> Numpy Data Row[Last]: {}".format(x_train[-1]))
        print('> Numpy Targets Type: {}'.format(type(self._y_train)))
        print("> Numpy Targets Shape: {}".format(self._y_train.shape))

        print('> Number of Samples: {}'.format(self._y_current.shape[0]))
        print('> Number of Training Samples: {}'.format(x_train.shape[0]))
        print('> Number of Training Classes: {}'.format(
            self._training_class_count))
        print('> Number of Test Samples: {}'.format(self.test_rows))
        print("> Training Minimum Value:", np.min(x_train))
        print("> Training Maximum Value:", np.max(x_train))
        print('> Number X signals: {}'.format(self._training_vector_count))
        print('> Number Y signals: {}'.format(self._training_class_count))

        # Print epoch related data
        print('> Epochs:', self._data.epochs())
        print('> Batch Size:', self._data.batch_size())
        print('> Steps:', self._data.epoch_steps())

        # Display estimated memory footprint of training data.
        print("> Data size: {:.2f} Bytes".format(x_train.nbytes))

        '''
        The neural network works best on values roughly between -1 and 1, so we
        need to scale the data before it is being input to the neural network.
        We can use scikit-learn for this.

        We first create a scaler-object for the input-signals.

        Then we detect the range of values from the training-data and scale
        the training-data.
        '''

        self._x_scaler = MinMaxScaler()
        self._x_train_scaled = self._x_scaler.fit_transform(x_train)

        print('> Scaled Training Minimum Value: {}'.format(
            np.min(self._x_train_scaled)))
        print('> Scaled Training Maximum Value: {}'.format(
            np.max(self._x_train_scaled)))

        self._xv_test_scaled = self._x_scaler.transform(xv_test)

        '''
        The target-data comes from the same data-set as the input-signals,
        because it is the weather-data for one of the cities that is merely
        time-shifted. But the target-data could be from a different source with
        different value-ranges, so we create a separate scaler-object for the
        target-data.
        '''

        self._y_scaler = MinMaxScaler()
        self._y_train_scaled = self._y_scaler.fit_transform(self._y_train)
        yv_test_scaled = self._y_scaler.transform(self._yv_test)

        # Data Generator

        '''
        The data-set has now been prepared as 2-dimensional numpy arrays. The
        training-data has almost 300k observations, consisting of 20
        input-signals and 3 output-signals.

        These are the array-shapes of the input and output data:
        '''

        print('> Scaled Training Data Shape: {}'.format(
            self._x_train_scaled.shape))
        print('> Scaled Training Targets Shape: {}'.format(
            self._y_train_scaled.shape))

        # We then create the batch-generator.

        generator = self._batch_generator(
            self._data.batch_size(), sequence_length)

        # Validation Set

        '''
        The neural network trains quickly so we can easily run many training
        epochs. But then there is a risk of overfitting the model to the
        training-set so it does not generalize well to unseen data. We will
        therefore monitor the model's performance on the test-set after each
        epoch and only save the model's weights if the performance is improved
        on the test-set.

        The batch-generator randomly selects a batch of short sequences from
        the training-data and uses that during training. But for the
        validation-data we will instead run through the entire sequence from
        the test-set and measure the prediction accuracy on that entire
        sequence.
        '''

        validation_data = (np.expand_dims(self._xv_test_scaled, axis=0),
                           np.expand_dims(yv_test_scaled, axis=0))

        # Create the Recurrent Neural Network

        self._model = Sequential()

        '''
        We can now add a Gated Recurrent Unit (GRU) to the network. This will
        have 512 outputs for each time-step in the sequence.

        Note that because this is the first layer in the model, Keras needs to
        know the shape of its input, which is a batch of sequences of arbitrary
        length (indicated by None), where each observation has a number of
        input-signals (num_x_signals).
        '''

        self._model.add(GRU(
            units=units,
            return_sequences=True,
            recurrent_dropout=dropout,
            input_shape=(None, self._training_vector_count,)))

        for _ in range(0, _layers):
            self._model.add(GRU(
                units=units,
                recurrent_dropout=dropout,
                return_sequences=True))

        '''
        The GRU outputs a batch of sequences of 512 values. We want to predict
        3 output-signals, so we add a fully-connected (or dense) layer which
        maps 512 values down to only 3 values.

        The output-signals in the data-set have been limited to be between 0
        and 1 using a scaler-object. So we also limit the output of the neural
        network using the Sigmoid activation function, which squashes the
        output to be between 0 and 1.'''

        self._model.add(
            Dense(self._training_class_count, activation='sigmoid'))

        '''
        A problem with using the Sigmoid activation function, is that we can
        now only output values in the same range as the training-data.

        For example, if the training-data only has temperatures between -20
        and +30 degrees, then the scaler-object will map -20 to 0 and +30 to 1.
        So if we limit the output of the neural network to be between 0 and 1
        using the Sigmoid function, this can only be mapped back to temperature
        values between -20 and +30.

        We can use a linear activation function on the output instead. This
        allows for the output to take on arbitrary values. It might work with
        the standard initialization for a simple network architecture, but for
        more complicated network architectures e.g. with more layers, it might
        be necessary to initialize the weights with smaller values to avoid
        NaN values during training. You may need to experiment with this to
        get it working.
        '''

        if False:
            # Maybe use lower init-ranges.
            # init = RandomUniform(minval=-0.05, maxval=0.05)
            init = RandomUniform(minval=-0.05, maxval=0.05)

            self._model.add(Dense(
                self._training_class_count,
                activation='linear',
                kernel_initializer=init))

        # Compile Model

        '''
        This is the optimizer and the beginning learning-rate that we will use.
        We then compile the Keras model so it is ready for training.
        '''
        optimizer = RMSprop(lr=1e-3)
        self._model.compile(
            loss=self._loss_mse_warmup,
            optimizer=optimizer,
            metrics=['accuracy'])

        '''
        This is a very small model with only two layers. The output shape of
        (None, None, 3) means that the model will output a batch with an
        arbitrary number of sequences, each of which has an arbitrary number of
        observations, and each observation has 3 signals. This corresponds to
        the 3 target signals we want to predict.
        '''
        print('> Model Summary:\n')
        print(self._model.summary())

        # Callback Functions

        '''
        During training we want to save checkpoints and log the progress to
        TensorBoard so we create the appropriate callbacks for Keras.

        This is the callback for writing checkpoints during training.
        '''

        callback_checkpoint = ModelCheckpoint(filepath=path_checkpoint,
                                              monitor='val_loss',
                                              verbose=1,
                                              save_weights_only=True,
                                              save_best_only=True)

        '''
        This is the callback for stopping the optimization when performance
        worsens on the validation-set.
        '''

        callback_early_stopping = EarlyStopping(monitor='val_loss',
                                                patience=patience, verbose=1)

        '''
        This is the callback for writing the TensorBoard log during training.
        '''

        callback_tensorboard = TensorBoard(log_dir='/tmp/23_logs/',
                                           histogram_freq=0,
                                           write_graph=False)

        '''
        This callback reduces the learning-rate for the optimizer if the
        validation-loss has not improved since the last epoch
        (as indicated by patience=0). The learning-rate will be reduced by
        multiplying it with the given factor. We set a start learning-rate of
        1e-3 above, so multiplying it by 0.1 gives a learning-rate of 1e-4.
        We don't want the learning-rate to go any lower than this.
        '''

        callback_reduce_lr = ReduceLROnPlateau(monitor='val_loss',
                                               factor=0.1,
                                               min_lr=1e-4,
                                               patience=0,
                                               verbose=1)

        callbacks = [callback_early_stopping,
                     callback_checkpoint,
                     callback_tensorboard,
                     callback_reduce_lr]

        # Train the Recurrent Neural Network

        '''We can now train the neural network.

        Note that a single "epoch" does not correspond to a single processing
        of the training-set, because of how the batch-generator randomly
        selects sub-sequences from the training-set. Instead we have selected
        steps_per_epoch so that one "epoch" is processed in a few minutes.

        With these settings, each "epoch" took about 2.5 minutes to process on
        a GTX 1070. After 14 "epochs" the optimization was stopped because the
        validation-loss had not decreased for 5 "epochs". This optimization
        took about 35 minutes to finish.

        Also note that the loss sometimes becomes NaN (not-a-number). This is
        often resolved by restarting and running the Notebook again. But it may
        also be caused by your neural network architecture, learning-rate,
        batch-size, sequence-length, etc. in which case you may have to modify
        those settings.
        '''

        print('\n> Starting data training\n')

        self._history = self._model.fit_generator(
            generator=generator,
            epochs=self._data.epochs(),
            steps_per_epoch=self._data.epoch_steps(),
            validation_data=validation_data,
            callbacks=callbacks)

        # Load Checkpoint

        '''
        Because we use early-stopping when training the model, it is possible
        that the model's performance has worsened on the test-set for several
        epochs before training was stopped. We therefore reload the last saved
        checkpoint, which should have the best performance on the test-set.
        '''

        print('> Loading model weights')
        if os.path.exists(path_checkpoint):
            self._model.load_weights(path_checkpoint)

        # Performance on Test-Set

        '''
        We can now evaluate the model's performance on the test-set. This
        function expects a batch of data, but we will just use one long
        time-series for the test-set, so we just expand the
        array-dimensionality to create a batch with that one sequence.
        '''

        result = self._model.evaluate(
            x=np.expand_dims(self._xv_test_scaled, axis=0),
            y=np.expand_dims(yv_test_scaled, axis=0))

        print('> Loss (test-set): {}'.format(result))

        # If you have several metrics you can use this instead.
        if False:
            for res, metric in zip(result, self._model.metrics_names):
                print('{0}: {1:.3e}'.format(metric, res))

    def _batch_generator(self, batch_size, sequence_length):
        """Create generator function to create random batches of training-data.

        Args:
            batch_size: Size of batch
            sequence_length: Length of sequence

        Returns:
            (x_batch, y_batch)

        """
        # Infinite loop.
        while True:
            # Allocate a new array for the batch of input-signals.
            x_shape = (
                batch_size, sequence_length, self._training_vector_count)
            x_batch = np.zeros(shape=x_shape, dtype=np.float16)

            # Allocate a new array for the batch of output-signals.
            y_shape = (batch_size, sequence_length, self._training_class_count)
            y_batch = np.zeros(shape=y_shape, dtype=np.float16)

            # Fill the batch with random sequences of data.
            for i in range(batch_size):
                # Get a random start-index.
                # This points somewhere into the training-data.
                idx = np.random.randint(
                    self.training_rows - sequence_length)

                # Copy the sequences of data starting at this index.
                x_batch[i] = self._x_train_scaled[idx:idx+sequence_length]
                y_batch[i] = self._y_train_scaled[idx:idx+sequence_length]

            yield (x_batch, y_batch)

    def _loss_mse_warmup(self, y_true, y_pred):
        """Calculate the Mean Squared Errror.

        Calculate the Mean Squared Error between y_true and y_pred,
        but ignore the beginning "warmup" part of the sequences.

        We will use Mean Squared Error (MSE) as the loss-function that will be
        minimized. This measures how closely the model's output matches the
        true output signals.

        However, at the beginning of a sequence, the model has only seen
        input-signals for a few time-steps, so its generated output may be very
        inaccurate. Using the loss-value for the early time-steps may cause the
        model to distort its later output. We therefore give the model a
        "warmup-period" of 50 time-steps where we don't use its accuracy in the
        loss-function, in hope of improving the accuracy for later time-steps

        Args:
            y_true: Desired output.
            y_pred: Model's output.

        Returns:
            loss_mean: Mean Squared Error

        """
        warmup_steps = self._warmup_steps

        # The shape of both input tensors are:
        # [batch_size, sequence_length, num_y_signals].

        # Ignore the "warmup" parts of the sequences
        # by taking slices of the tensors.
        y_true_slice = y_true[:, warmup_steps:, :]
        y_pred_slice = y_pred[:, warmup_steps:, :]

        # These sliced tensors both have this shape:
        # [batch_size, sequence_length - warmup_steps, num_y_signals]

        # Calculate the MSE loss for each value in these tensors.
        # This outputs a 3-rank tensor of the same shape.
        loss = tf.losses.mean_squared_error(labels=y_true_slice,
                                            predictions=y_pred_slice)

        # Keras may reduce this across the first axis (the batch)
        # but the semantics are unclear, so to be sure we use
        # the loss across the entire tensor, we reduce it to a
        # single scalar with the mean function.
        loss_mean = tf.reduce_mean(loss)

        return loss_mean

    def plot_train(self, start_idx, length=100):
        """Plot the predicted and true output-signals.

        Args:
            start_idx: Start-index for the time-series.
            length: Sequence-length to process and plot.

        Returns:
            None

        """
        # Plot
        self._plot_comparison(start_idx, length=length, train=True)

    def plot_test(self, start_idx, length=100):
        """Plot the predicted and true output-signals.

        Args:
            start_idx: Start-index for the time-series.
            length: Sequence-length to process and plot.

        Returns:
            None

        """
        # Plot
        self._plot_comparison(start_idx, length=length, train=False)

    def _plot_comparison(self, start_idx, length=100, train=True):
        """Plot the predicted and true output-signals.

        Args:
            start_idx: Start-index for the time-series.
            length: Sequence-length to process and plot.
            train: Boolean whether to use training- or test-set.

        Returns:
            None

        """
        # Initialize key variables
        datetimes = {}
        num_train = self.training_rows

        # End-index for the sequences.
        end_idx = start_idx + length

        # Variables for date formatting
        days = mdates.DayLocator()   # Every day
        months = mdates.MonthLocator()  # Every month
        months_format = mdates.DateFormatter('%b %Y')
        days_format = mdates.DateFormatter('%d')

        # Assign other variables dependent on the type of data we are plotting
        if train is True:
            # Use training-data.
            x_values = self._x_train_scaled[start_idx:end_idx]
            y_true = self._y_train[start_idx:end_idx]
            shim = 'Train'

            # Datetimes to use for training
            datetimes[shim] = self._data.datetime()[
                :num_train][start_idx:end_idx]

        else:
            # Scale the data
            x_test_scaled = self._x_scaler.transform(
                self._data.vectors_test_all())

            # Use test-data.
            x_values = x_test_scaled[start_idx:end_idx]
            y_true = self._yv_test[start_idx:end_idx]
            shim = 'Test'

            # Datetimes to use for testing
            datetimes[shim] = self._data.datetime()[
                num_train:][start_idx:end_idx]

        # Input-signals for the model.
        x_values = np.expand_dims(x_values, axis=0)

        # Use the model to predict the output-signals.
        y_pred = self._model.predict(x_values)

        # The output of the model is between 0 and 1.
        # Do an inverse map to get it back to the scale
        # of the original data-set.
        y_pred_rescaled = self._y_scaler.inverse_transform(y_pred[0])

        # For each output-signal.
        for signal in range(len(self._data.labels())):
            # Assign other variables dependent on the type of data plot
            if train is True:
                # Only get current values that are a part of the training data
                current = self._y_current[:num_train][start_idx:end_idx]

                # The number of datetimes for the 'actual' plot must match
                # that of current values
                datetimes['actual'] = self._data.datetime()[
                    :num_train][start_idx:end_idx]

            else:
                # Only get current values that are a part of the test data.
                current = self._y_current[
                    num_train:][start_idx:]

                # The number of datetimes for the 'actual' plot must match
                # that of current values
                datetimes['actual'] = self._data.datetime()[
                    num_train:][start_idx:]

            # Create a filename
            filename = (
                '/tmp/batch_{}_epochs_{}_training_{}_{}_{}_{}.png').format(
                    self._data.batch_size(),
                    self._data.epochs(),
                    num_train,
                    signal,
                    int(time.time()),
                    shim)

            # Get the output-signal predicted by the model.
            signal_pred = y_pred_rescaled[:, signal]

            # Get the true output-signal from the data-set.
            signal_true = y_true[:, signal]

            # Create a new chart
            (fig, axis) = plt.subplots(figsize=(15, 5))

            # Plot and compare the two signals.
            axis.plot(
                datetimes[shim][:len(signal_true)],
                signal_true,
                label='Current +{}'.format(self._data.labels()[signal]))
            axis.plot(
                datetimes[shim][:len(signal_pred)],
                signal_pred,
                label='Prediction')
            axis.plot(datetimes['actual'], current, label='Current')

            # Set plot labels and titles
            axis.set_title('{1}ing Forecast ({0} Future Intervals)'.format(
                self._data.labels()[signal], shim))
            axis.set_ylabel('Values')
            axis.legend(
                bbox_to_anchor=(1.04, 0.5),
                loc='center left', borderaxespad=0)

            # Add gridlines and ticks
            ax = plt.gca()
            ax.grid(True)

            # Add major gridlines
            ax.xaxis.grid(which='major', color='black', alpha=0.2)
            ax.yaxis.grid(which='major', color='black', alpha=0.2)

            # Add minor ticks (They must be turned on first)
            ax.minorticks_on()
            ax.xaxis.grid(which='minor', color='black', alpha=0.1)
            ax.yaxis.grid(which='minor', color='black', alpha=0.1)

            # Format the tick labels
            ax.xaxis.set_major_locator(months)
            ax.xaxis.set_major_formatter(months_format)
            ax.xaxis.set_minor_locator(days)

            # Remove tick marks
            ax.tick_params(axis='both', which='both', length=0)

            # Print day numbers on xaxis for Test data only
            if train is False:
                ax.xaxis.set_minor_formatter(days_format)
                plt.setp(ax.xaxis.get_minorticklabels(), rotation=90)

            # Rotates and right aligns the x labels, and moves the bottom of
            # the axes up to make room for them
            fig.autofmt_xdate()

            # Plot grey box for warmup-period if we are working with training
            # data and the start is within the warmup-period
            if (0 < start_idx < self._warmup_steps):
                if train is True:
                    plt.axvspan(
                        datetimes[shim][start_idx],
                        datetimes[shim][self._warmup_steps],
                        facecolor='black', alpha=0.15)

            # Show and save the image
            if self.display is True:
                fig.savefig(filename, bbox_inches='tight')
                plt.show()
            else:
                fig.savefig(filename, bbox_inches='tight')
            print('> Saving file: {}'.format(filename))

            # Close figure
            plt.close(fig=fig)

    def plot_accuracy(self):
        """Plot the predicted and true output-signals.

        Args:
            None

        Returns:
            None

        """
        # Summarize history for accuracy
        plt.figure(figsize=(15, 5))
        plt.plot(self._history.history['acc'])
        plt.plot(self._history.history['val_acc'])
        plt.title('Model Accuracy')
        plt.ylabel('Accuracy')
        plt.xlabel('Epoch')
        plt.legend(['train', 'test'], loc='upper left')
        plt.show()

        # Summarize history for loss
        plt.figure(figsize=(15, 5))
        plt.plot(self._history.history['loss'])
        plt.plot(self._history.history['val_loss'])
        plt.title('Model loss')
        plt.ylabel('Loss')
        plt.xlabel('Epoch')
        plt.legend(['Train', 'Test'], loc='upper left')
        plt.show()
def keras_model_fn(hyperparameters):
    """keras_model_fn receives hyperparameters from the training job and returns a compiled keras model.
    The model will be transformed into a TensorFlow Estimator before training and it will be saved in a 
    TensorFlow Serving SavedModel at the end of training.

    Args:
        hyperparameters: The hyperparameters passed to the SageMaker TrainingJob that runs your TensorFlow 
                         training script.
    Returns: A compiled Keras model
    """
    model = Sequential()

    model.add(Conv2D(32, (3, 3), padding='same', name='inputs', input_shape=(HEIGHT, WIDTH, DEPTH)))
    model.add(Activation('relu'))
    model.add(Conv2D(32, (3, 3)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Conv2D(64, (3, 3), padding='same'))
    model.add(Activation('relu'))
    model.add(Conv2D(64, (3, 3)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Flatten())
    model.add(Dense(512))
    model.add(Activation('relu'))
    model.add(Dropout(0.5))
    model.add(Dense(NUM_CLASSES))
    model.add(Activation('softmax'))
    
    opt = RMSPropOptimizer(learning_rate=hyperparameters['learning_rate'], decay=hyperparameters['decay'])

    model.compile(loss='categorical_crossentropy',
                  optimizer=opt,
                  metrics=['accuracy'])

    return model
示例#35
0
def getModel (n_input, n_output):
    model = Sequential()
    
    # model.add(Dense(round(n_input * 2), activation='relu' , input_dim=n_input))
    # model.add(Dropout(0.5))
    # model.add(CustomDropout(conf['dropout_rate'], conf['perma_drop']))
    # model.add(Dense(round(n_input), activation='relu'))
    # model.add(Dense(round(n_input / 2), activation='relu'))
    # model.add(CustomDropout(conf['dropout_rate'], conf['perma_drop']))
    # model.add(Dense(round(n_input / 3), activation='relu'))
    # model.add(Dense(units=n_output, activation='sigmoid'))  
    
    # model = Sequential()
    # model.add(Dense(round(n_input), activation='relu', input_dim=n_input)) 
    # model.add(Dropout(conf['dropout_rate']))
    # model.add(Dense(units=round(n_input ), activation='relu'))    
    # model.add(Dense(units=round(n_input / 2 ), activation='relu'))         
    # model.add(Dense(units=n_output, activation='sigmoid'))  
    
    asdf = random.seed()
    
    # model.add(Dense(round(n_input), activation='relu' ,kernel_initializer=glorot_uniform(seed=asdf), input_dim=n_input))
    # model.add(Dropout(0.5))
    # model.add(Dense(round(n_input / 2), activation='relu',kernel_initializer=glorot_uniform(seed=asdf)))
    # model.add(Dropout(0.5))
    # model.add(Dense(round(n_input / 3), activation='relu', kernel_initializer=glorot_uniform(seed=asdf)))
    # model.add(Dense(units=n_output, activation='sigmoid', kernel_initializer=glorot_uniform(seed=asdf)))
    
    #model.add(Dense(round(n_input * 2), activation='relu' , input_dim=n_input,kernel_initializer=glorot_uniform(seed=asdf)))
    model.add(LSTM(32, input_shape=(int(n_input / conf['n_timesteps']), conf['n_timesteps'])))
    model.add(Dropout(0.25))
    model.add(Dense(round(n_input), activation='relu',kernel_initializer=glorot_uniform(seed=asdf)))
    model.add(Dense(round(n_input / 2), activation='relu',kernel_initializer=glorot_uniform(seed=asdf)))
    model.add(Dense(units=n_output, activation='sigmoid',kernel_initializer=glorot_uniform(seed=asdf)))  

    return model
示例#36
0
class KerasCNN(object):
    """Support vector machine class."""

    # Convolutional Layer 1.
    filter_size1 = 5          # Convolution filters are 5 x 5 pixels.
    num_filters1 = 16         # There are 16 of these filters.

    # Convolutional Layer 2.
    filter_size2 = 5          # Convolution filters are 5 x 5 pixels.
    num_filters2 = 36         # There are 36 of these filters.

    # Fully-connected layer.
    fc_size = 128             # Number of neurons in fully-connected laye

    # Get data from files
    data = MNIST(data_dir='/tmp/data/MNIST/')

    # The number of pixels in each dimension of an image.
    img_size = data.img_size

    # The images are stored in one-dimensional arrays of this length.
    img_size_flat = data.img_size_flat

    # Tuple with height and width of images used to reshape arrays.
    img_shape = data.img_shape

    # Tuple with height, width and depth used to reshape arrays.
    # This is used for reshaping in Keras.
    img_shape_full = data.img_shape_full

    # Number of classes, one class for each of 10 digits.
    num_classes = data.num_classes

    # Number of colour channels for the images: 1 channel for gray-scale.
    num_channels = data.num_channels

    def __init__(self):
        """Instantiate the class.

        Args:
            train_batch_size: Training batch size

        Returns:
            None

        """
        # Initialize variables
        epochs = 2

        """
        print('{0: <{1}} {2}'.format('Encoded X image:', fill, self.x_image))
        """

        # Start construction of the Keras Sequential model.
        self.model = Sequential()

        # Add an input layer which is similar to a feed_dict in TensorFlow.
        # Note that the input-shape must be a tuple containing the image-size.
        self.model.add(InputLayer(input_shape=(self.img_size_flat,)))

        # The input is a flattened array with 784 elements,
        # but the convolutional layers expect images with shape (28, 28, 1)
        self.model.add(Reshape(self.img_shape_full))

        # First convolutional layer with ReLU-activation and max-pooling.
        self.model.add(
            Conv2D(kernel_size=5, strides=1, filters=16, padding='same',
                   activation='relu', name='layer_conv1'))
        self.model.add(MaxPooling2D(pool_size=2, strides=2))

        # Second convolutional layer with ReLU-activation and max-pooling.
        self.model.add(
            Conv2D(kernel_size=5, strides=1, filters=36, padding='same',
                   activation='relu', name='layer_conv2'))
        self.model.add(MaxPooling2D(pool_size=2, strides=2))

        # Flatten the 4-rank output of the convolutional layers
        # to 2-rank that can be input to a fully-connected / dense layer.
        self.model.add(Flatten())

        # First fully-connected / dense layer with ReLU-activation.
        self.model.add(Dense(128, activation='relu'))

        # Last fully-connected / dense layer with softmax-activation
        # for use in classification.
        self.model.add(Dense(self.num_classes, activation='softmax'))

        # Model Compilation

        '''
        The Neural Network has now been defined and must be finalized by adding
        a loss-function, optimizer and performance metrics. This is called
        model "compilation" in Keras.

        We can either define the optimizer using a string, or if we want more
        control of its parameters then we need to instantiate an object. For
        example, we can set the learning-rate.
        '''

        optimizer = Adam(lr=1e-3)

        '''
        For a classification-problem such as MNIST which has 10 possible
        classes, we need to use the loss-function called
        categorical_crossentropy. The performance metric we are interested in
        is the classification accuracy.
        '''

        self.model.compile(
            optimizer=optimizer,
            loss='categorical_crossentropy',
            metrics=['accuracy'])

        # Training

        '''
        Now that the model has been fully defined with loss-function and
        optimizer, we can train it. This function takes numpy-arrays and
        performs the given number of training epochs using the given
        batch-size. An epoch is one full use of the entire training-set. So for
        10 epochs we would iterate randomly over the entire training-set 10
        times.
        '''

        self.model.fit(x=self.data.x_train,
                       y=self.data.y_train,
                       epochs=epochs, batch_size=128)

        # Evaluation

        '''
        Now that the model has been trained we can test its performance on the
        test-set. This also uses numpy-arrays as input.
        '''

        result = self.model.evaluate(x=self.data.x_test, y=self.data.y_test)

        '''
        Print actual versus predicted values
        '''

        print('\nActual vs Predicted X values')
        start = 0
        stop = 300
        predictions = self.model.predict(self.data.x_test[start:stop])
        for pointer in range(start, stop):
            predicted = np.argmax(predictions[pointer])
            actual = np.argmax(self.data.y_test[pointer])
            print(
                '{}: Actual: {}\tPredicted: {}\tMatch: {}'.format(
                    str(pointer).zfill(3),
                    predicted, actual, predicted == actual))

        '''
        We can print all the performance metrics for the test-set.
        '''
        print('\nPerfomance metrics')
        for name, value in zip(self.model.metrics_names, result):
            print('{} {}'.format(name, value))

        '''
        Print the model summary
        '''

        print('\n\nModel Summary\n\n{}'.format(self.model.summary()))

    def plot_example_errors(self, cls_pred):
        """Plot 9 images in a 3x3 grid.

        Function used to plot 9 images in a 3x3 grid, and writing the true and
        predicted classes below each image.

        Args:
            cls_pred: Array of the predicted class-number for all images in the
                test-set.

        Returns:
            None

        """
        # Boolean array whether the predicted class is incorrect.
        incorrect = (cls_pred != self.data.y_test_cls)

        # Get the images from the test-set that have been
        # incorrectly classified.
        images = self.data.x_test[incorrect]

        # Get the predicted classes for those images.
        cls_pred = cls_pred[incorrect]

        # Get the true classes for those images.
        cls_true = self.data.y_test_cls[incorrect]

        # Plot the first 9 images.
        plot_images(
            images[0:9], self.img_shape, cls_true[0:9], cls_pred=cls_pred[0:9])
示例#37
0
plt.show()
for i in range(10):
    plt.subplot(2, 5, i+1)
    plt.title("M_%d" %i)
    plt.axis("off")
    plt.imshow(x_train_gauss[i].reshape(28, 28), cmap=None)
plt.show()
for i in range(10):
    plt.subplot(2, 5, i+1)
    plt.title("M_%d" %i)
    plt.axis("off")
    plt.imshow(x_train_masked[i].reshape(28, 28), cmap=None)
plt.show()

# Create the neural network
autoencoder = Sequential()

## Create the encoder parts
autoencoder.add(Conv2D(16, (3,3), 1, activation='relu', padding='same', input_shape=(28, 28, 1)))
autoencoder.add(MaxPool2D((2,2), padding='same'))
autoencoder.add(Conv2D(8, (3,3), 1, activation='relu', padding='same'))
autoencoder.add(MaxPool2D((2,2), padding='same'))

## Create the decoder parts
autoencoder.add(Conv2D(8, (3,3), 1, activation='relu', padding='same'))
autoencoder.add(UpSampling2D((2,2)))
autoencoder.add(Conv2D(16, (3,3), 1, activation='relu', padding='same'))
autoencoder.add(UpSampling2D((2,2)))
autoencoder.add(Conv2D(1, (3,3), 1, activation='sigmoid', padding='same'))

# Setting for learning
示例#38
0
# net_base = efn.EfficientNetB0(weights='imagenet', include_top=False)

flag = False
if (flag):

    net_base.trainable = True
    set_trainable = False
    for layer in net_base.layers:
        if layer.name == "block8_10_conv":
            set_trainable = True
        if set_trainable:
            layer.trainable = True
        else:
            layer.trainable = False

model = Sequential()
model.add(net_base)
model.add(GlobalAveragePooling2D())
model.add(Dense(512, activation='relu'))
model.add(Dense(1, activation='sigmoid'))

model.compile(optimizer=optimizers.RMSprop(lr=1e-4), loss='binary_crossentropy', metrics=['accuracy'])

history = model.fit(train_generator, steps_per_epoch=50, epochs=100, validation_data=validation_generator,
                    validation_steps=50)

history_dict = history.history
accuracy = history_dict['accuracy']
val_acc = history_dict['val_accuracy']

epoch = range(1, len(accuracy) + 1)
示例#39
0
def getModel (n_input, n_output):
    model = Sequential()
    model.add(Dense(round(n_input * 2), activation='relu' , input_dim=n_input))
    model.add(Dropout(0.5))
    model.add(CustomDropout(conf['dropout_rate'], conf['perma_drop']))
    model.add(Dense(round(n_input), activation='relu'))
    model.add(Dense(round(n_input / 2), activation='relu'))
    model.add(CustomDropout(conf['dropout_rate'], conf['perma_drop']))
    model.add(Dense(round(n_input / 3), activation='relu'))
    model.add(Dense(units=n_output, activation='tanh'))  

    return model
# 1 for grayscale
num_channels = 1

# The images are stored in one-dimensional arrays of this length.
img_size_flat = img_size * img_size * num_channels

# Tuple with height and width of images used to reshape arrays.
img_shape = (img_size, img_size, num_channels)

img_shape_full = (img_size, img_size, num_channels)

# Number of classes, one class for each of 10 digits.
num_classes = 8

model = Sequential()

model.add(InputLayer(input_shape=(img_size_flat, )))

model.add(Reshape(img_shape_full))

model.add(
    Conv2D(kernel_size=5,
           strides=1,
           filters=32,
           padding="same",
           activation="relu",
           name="conv_layer_1"))

model.add(MaxPooling2D(pool_size=2, strides=2))
示例#41
0
    def __init__(self):
        """Instantiate the class.

        Args:
            train_batch_size: Training batch size

        Returns:
            None

        """
        # Initialize variables
        epochs = 2

        """
        print('{0: <{1}} {2}'.format('Encoded X image:', fill, self.x_image))
        """

        # Start construction of the Keras Sequential model.
        self.model = Sequential()

        # Add an input layer which is similar to a feed_dict in TensorFlow.
        # Note that the input-shape must be a tuple containing the image-size.
        self.model.add(InputLayer(input_shape=(self.img_size_flat,)))

        # The input is a flattened array with 784 elements,
        # but the convolutional layers expect images with shape (28, 28, 1)
        self.model.add(Reshape(self.img_shape_full))

        # First convolutional layer with ReLU-activation and max-pooling.
        self.model.add(
            Conv2D(kernel_size=5, strides=1, filters=16, padding='same',
                   activation='relu', name='layer_conv1'))
        self.model.add(MaxPooling2D(pool_size=2, strides=2))

        # Second convolutional layer with ReLU-activation and max-pooling.
        self.model.add(
            Conv2D(kernel_size=5, strides=1, filters=36, padding='same',
                   activation='relu', name='layer_conv2'))
        self.model.add(MaxPooling2D(pool_size=2, strides=2))

        # Flatten the 4-rank output of the convolutional layers
        # to 2-rank that can be input to a fully-connected / dense layer.
        self.model.add(Flatten())

        # First fully-connected / dense layer with ReLU-activation.
        self.model.add(Dense(128, activation='relu'))

        # Last fully-connected / dense layer with softmax-activation
        # for use in classification.
        self.model.add(Dense(self.num_classes, activation='softmax'))

        # Model Compilation

        '''
        The Neural Network has now been defined and must be finalized by adding
        a loss-function, optimizer and performance metrics. This is called
        model "compilation" in Keras.

        We can either define the optimizer using a string, or if we want more
        control of its parameters then we need to instantiate an object. For
        example, we can set the learning-rate.
        '''

        optimizer = Adam(lr=1e-3)

        '''
        For a classification-problem such as MNIST which has 10 possible
        classes, we need to use the loss-function called
        categorical_crossentropy. The performance metric we are interested in
        is the classification accuracy.
        '''

        self.model.compile(
            optimizer=optimizer,
            loss='categorical_crossentropy',
            metrics=['accuracy'])

        # Training

        '''
        Now that the model has been fully defined with loss-function and
        optimizer, we can train it. This function takes numpy-arrays and
        performs the given number of training epochs using the given
        batch-size. An epoch is one full use of the entire training-set. So for
        10 epochs we would iterate randomly over the entire training-set 10
        times.
        '''

        self.model.fit(x=self.data.x_train,
                       y=self.data.y_train,
                       epochs=epochs, batch_size=128)

        # Evaluation

        '''
        Now that the model has been trained we can test its performance on the
        test-set. This also uses numpy-arrays as input.
        '''

        result = self.model.evaluate(x=self.data.x_test, y=self.data.y_test)

        '''
        Print actual versus predicted values
        '''

        print('\nActual vs Predicted X values')
        start = 0
        stop = 300
        predictions = self.model.predict(self.data.x_test[start:stop])
        for pointer in range(start, stop):
            predicted = np.argmax(predictions[pointer])
            actual = np.argmax(self.data.y_test[pointer])
            print(
                '{}: Actual: {}\tPredicted: {}\tMatch: {}'.format(
                    str(pointer).zfill(3),
                    predicted, actual, predicted == actual))

        '''
        We can print all the performance metrics for the test-set.
        '''
        print('\nPerfomance metrics')
        for name, value in zip(self.model.metrics_names, result):
            print('{} {}'.format(name, value))

        '''
        Print the model summary
        '''

        print('\n\nModel Summary\n\n{}'.format(self.model.summary()))
def get_model_1(x,y,Vocab_size,maxlen):
    model = Sequential()
    model.add(Embedding(Vocab_size, 30)) #input_length=maxlen))
    model.add(LSTM(60, return_sequences=True))
    model.add(Dense(60, activation='relu'))
    model.add(Dense(Vocab_size))
    model.add(Activation("softmax"))
    print(model.summary())
    optimizer = RMSprop(lr=0.01)
    model.compile(loss="sparse_categorical_crossentropy", optimizer='adam')
    return model
(x_train, y_train), (x_test, y_test) = mnist.load_data()

x_train = x_train.reshape(60000, 784)
x_test = x_test.reshape(10000, 784)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')

# convert class vectors to binary class matrices
y_train = tf.keras.utils.to_categorical(y_train, num_classes)
y_test = tf.keras.utils.to_categorical(y_test, num_classes)

model = Sequential()
model.add(Dense(512, activation='relu', input_shape=(784,)))
model.add(Dropout(0.2))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(num_classes, activation='softmax'))

model.summary()

model.compile(loss='categorical_crossentropy',
              optimizer=RMSprop(),
              metrics=['accuracy'])

history = model.fit(x_train, y_train,
                    batch_size=batch_size,
                    epochs=epochs,
示例#44
0
    embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))
    not_in_model = 0
    in_model = 0
    for word, i in word_index.items():
        if unicode(word) in w2v_model:
            in_model += 1
            embedding_matrix[i] = np.asarray(w2v_model[unicode(word)], dtype='float32')
        else:
            not_in_model += 1
    print(str(not_in_model)+' words not in w2v model')
    embedding_layer = Embedding(len(word_index) + 1, EMBEDDING_DIM, weights=[embedding_matrix],
                                input_length=MAX_SEQUENCE_LENGTH, trainable=False)


    print('(5) training model...')
    model = Sequential()
    model.add(embedding_layer)
    model.add(LSTM(200, dropout=0.2, recurrent_dropout=0.2))
    model.add(Dropout(0.2))
    model.add(Dense(labels.shape[1], activation='softmax'))
    model.summary()
    plot_model(model, to_file=os.path.join(ckpt_path, 'word_vector_lstm_model.png'), show_shapes=True)

    model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['acc'])
    print(model.metrics_names)
    model.fit(x_train, y_train, validation_data=(x_val, y_val), epochs=2, batch_size=128)
    model.save(os.path.join(ckpt_path, 'word_vector_lstm.h5'))


    print('(6) testing model...')
    print(model.evaluate(x_test, y_test))
示例#45
0
# ### Plot a few images to see if data is correct
# Get the first images from the test-set.
images = x_test[9:18]
# Get the true classes for those images.
cls_true = y_test[9:18]
# Plot the images and labels using our helper-function above.
plot_images(images=images, cls_true=cls_true)

# from tf.keras.models import Sequential  # This does not work!

from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import InputLayer, Input
from tensorflow.python.keras.layers import Reshape, MaxPooling2D
from tensorflow.python.keras.layers import Conv2D, Dense, Flatten

model = Sequential()

#Añadir el primer layer
model.add(InputLayer(input_shape=(img_size_flat, )))
model.add(Reshape(img_shape_full))
model.add(
    Conv2D(kernel_size=5,
           strides=1,
           filters=16,
           padding='same',
           activation='relu',
           name='layer_conv1'))
model.add(MaxPooling2D(pool_size=2, strides=2))
model.add(
    Conv2D(kernel_size=5,
           strides=1,
示例#46
0
class KerasDNNRegressor:
    def __init__(self, input_dropout=0.2, hidden_layers=2, hidden_units=64, 
                hidden_activation="relu", hidden_dropout=0.5, batch_norm=None, 
                optimizer="adadelta", nb_epoch=10, batch_size=64):
        self.input_dropout = input_dropout
        self.hidden_layers = hidden_layers
        self.hidden_units = hidden_units
        self.hidden_activation = hidden_activation
        self.hidden_dropout = hidden_dropout
        self.batch_norm = batch_norm
        self.optimizer = optimizer
        self.nb_epoch = nb_epoch
        self.batch_size = batch_size
        self.scaler = None
        self.model = None

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        return ("%s(input_dropout=%f, hidden_layers=%d, hidden_units=%d, \n"
                    "hidden_activation=\'%s\', hidden_dropout=%f, batch_norm=\'%s\', \n"
                    "optimizer=\'%s\', nb_epoch=%d, batch_size=%d)" % (
                    self.__class__.__name__,
                    self.input_dropout,
                    self.hidden_layers,
                    self.hidden_units,
                    self.hidden_activation,
                    self.hidden_dropout,
                    str(self.batch_norm),
                    self.optimizer,
                    self.nb_epoch,
                    self.batch_size,
                ))


    def fit(self, X, y):
        ## scaler
        self.scaler = StandardScaler()
        X = self.scaler.fit_transform(X)

        #### build model
        self.model = Sequential()
        ## input layer
        self.model.add(Dropout(self.input_dropout, input_shape=(X.shape[1],)))
        ## hidden layers
        first = True
        hidden_layers = self.hidden_layers
        while hidden_layers > 0:
            self.model.add(Dense(self.hidden_units))
            if self.batch_norm == "before_act":
                self.model.add(BatchNormalization())
            if self.hidden_activation == "prelu":
                self.model.add(PReLU())
            elif self.hidden_activation == "elu":
                self.model.add(ELU())
            else:
                self.model.add(Activation(self.hidden_activation))
            if self.batch_norm == "after_act":
                self.model.add(BatchNormalization())
            self.model.add(Dropout(self.hidden_dropout))
            hidden_layers -= 1

        ## output layer
        output_dim = 1
        output_act = "linear"
        self.model.add(Dense(output_dim))
        self.model.add(Activation(output_act))
        
        ## loss
        if self.optimizer == "sgd":
            sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
            self.model.compile(loss="mse", optimizer=sgd)
        else:
            self.model.compile(loss="mse", optimizer=self.optimizer)

        ## fit
        self.model.fit(X, y,
                    epochs=self.nb_epoch, 
                    batch_size=self.batch_size,
                    validation_split=0, verbose=1)
        return self

    def predict(self, X):
        X = self.scaler.transform(X)
        y_pred = self.model.predict(X)
        y_pred = y_pred.flatten()
        return y_pred
示例#47
0
    def model(self, params=None):
        """Create the Recurrent Neural Network.

        Args:
            None

        Returns:
            _model: RNN model

        """
        # Initialize key variables
        if params is None:
            _hyperparameters = self.hyperparameters
        else:
            _hyperparameters = params

        # Calculate the steps per epoch
        epoch_steps = int(
            self.training_rows / _hyperparameters['batch_size']) + 1

        # Create the model object
        _model = Sequential()

        '''
        We can now add a Gated Recurrent Unit (GRU) to the network. This will
        have 512 outputs for each time-step in the sequence.

        Note that because this is the first layer in the model, Keras needs to
        know the shape of its input, which is a batch of sequences of arbitrary
        length (indicated by None), where each observation has a number of
        input-signals (num_x_signals).
        '''

        _model.add(GRU(
            units=_hyperparameters['units'],
            return_sequences=True,
            recurrent_dropout=_hyperparameters['dropout'],
            input_shape=(None, self._training_vector_count,)))

        for _ in range(1, _hyperparameters['layers']):
            _model.add(GRU(
                units=_hyperparameters['units'],
                recurrent_dropout=_hyperparameters['dropout'],
                return_sequences=True))

        '''
        The GRU outputs a batch of sequences of 512 values. We want to predict
        3 output-signals, so we add a fully-connected (or dense) layer which
        maps 512 values down to only 3 values.

        The output-signals in the data-set have been limited to be between 0
        and 1 using a scaler-object. So we also limit the output of the neural
        network using the Sigmoid activation function, which squashes the
        output to be between 0 and 1.
        '''

        _model.add(
            Dense(self._training_class_count, activation='sigmoid'))

        '''
        A problem with using the Sigmoid activation function, is that we can
        now only output values in the same range as the training-data.

        For example, if the training-data only has values between -20 and +30,
        then the scaler-object will map -20 to 0 and +30 to 1. So if we limit
        the output of the neural network to be between 0 and 1 using the
        Sigmoid function, this can only be mapped back to values between
        -20 and +30.

        We can use a linear activation function on the output instead. This
        allows for the output to take on arbitrary values. It might work with
        the standard initialization for a simple network architecture, but for
        more complicated network architectures e.g. with more layers, it might
        be necessary to initialize the weights with smaller values to avoid
        NaN values during training. You may need to experiment with this to
        get it working.
        '''

        if False:
            # Maybe use lower init-ranges.
            init = RandomUniform(minval=-0.05, maxval=0.05)

            _model.add(Dense(
                self._training_class_count,
                activation='linear',
                kernel_initializer=init))

        # Compile Model

        '''
        This is the optimizer and the beginning learning-rate that we will use.
        We then compile the Keras model so it is ready for training.
        '''
        optimizer = RMSprop(lr=1e-3)
        _model.compile(
            loss=self._loss_mse_warmup,
            optimizer=optimizer,
            metrics=['accuracy'])

        '''
        This is a very small model with only two layers. The output shape of
        (None, None, 3) means that the model will output a batch with an
        arbitrary number of sequences, each of which has an arbitrary number of
        observations, and each observation has 3 signals. This corresponds to
        the 3 target signals we want to predict.
        '''
        print('\n> Model Summary:\n')
        print(_model.summary())

        # Create the batch-generator.
        generator = self._batch_generator(
            _hyperparameters['batch_size'],
            _hyperparameters['sequence_length'])

        # Validation Set

        '''
        The neural network trains quickly so we can easily run many training
        epochs. But then there is a risk of overfitting the model to the
        training-set so it does not generalize well to unseen data. We will
        therefore monitor the model's performance on the test-set after each
        epoch and only save the model's weights if the performance is improved
        on the test-set.

        The batch-generator randomly selects a batch of short sequences from
        the training-data and uses that during training. But for the
        validation-data we will instead run through the entire sequence from
        the test-set and measure the prediction accuracy on that entire
        sequence.
        '''

        validation_data = (np.expand_dims(self._x_validation_scaled, axis=0),
                           np.expand_dims(self._y_validation_scaled, axis=0))

        # Callback Functions

        '''
        During training we want to save checkpoints and log the progress to
        TensorBoard so we create the appropriate callbacks for Keras.

        This is the callback for writing checkpoints during training.
        '''

        callback_checkpoint = ModelCheckpoint(filepath=self._path_checkpoint,
                                              monitor='val_loss',
                                              verbose=1,
                                              save_weights_only=True,
                                              save_best_only=True)

        '''
        This is the callback for stopping the optimization when performance
        worsens on the validation-set.
        '''

        callback_early_stopping = EarlyStopping(
            monitor='val_loss',
            patience=_hyperparameters['patience'],
            verbose=1)

        '''
        This is the callback for writing the TensorBoard log during training.
        '''

        callback_tensorboard = TensorBoard(log_dir='/tmp/23_logs/',
                                           histogram_freq=0,
                                           write_graph=False)

        '''
        This callback reduces the learning-rate for the optimizer if the
        validation-loss has not improved since the last epoch
        (as indicated by patience=0). The learning-rate will be reduced by
        multiplying it with the given factor. We set a start learning-rate of
        1e-3 above, so multiplying it by 0.1 gives a learning-rate of 1e-4.
        We don't want the learning-rate to go any lower than this.
        '''

        callback_reduce_lr = ReduceLROnPlateau(monitor='val_loss',
                                               factor=0.1,
                                               min_lr=1e-4,
                                               patience=0,
                                               verbose=1)

        callbacks = [callback_early_stopping,
                     callback_checkpoint,
                     callback_tensorboard,
                     callback_reduce_lr]

        # Train the Recurrent Neural Network

        '''We can now train the neural network.

        Note that a single "epoch" does not correspond to a single processing
        of the training-set, because of how the batch-generator randomly
        selects sub-sequences from the training-set. Instead we have selected
        steps_per_epoch so that one "epoch" is processed in a few minutes.

        With these settings, each "epoch" took about 2.5 minutes to process on
        a GTX 1070. After 14 "epochs" the optimization was stopped because the
        validation-loss had not decreased for 5 "epochs". This optimization
        took about 35 minutes to finish.

        Also note that the loss sometimes becomes NaN (not-a-number). This is
        often resolved by restarting and running the Notebook again. But it may
        also be caused by your neural network architecture, learning-rate,
        batch-size, sequence-length, etc. in which case you may have to modify
        those settings.
        '''

        print('\n> Parameters for training\n')
        pprint(_hyperparameters)
        print('\n> Starting data training\n')

        _model.fit_generator(
            generator=generator,
            epochs=_hyperparameters['epochs'],
            steps_per_epoch=epoch_steps,
            validation_data=validation_data,
            callbacks=callbacks)

        # Return
        return _model
示例#48
0
# 准备target向量,[1860, 3073, 2872, 4106, 2395],anger:0.,disgus:1.,happiness:2.,like:3.,sadness:4.
train_target = np.concatenate(
    (np.zeros(1860), np.ones(3073), np.ones(2872) * 2, np.ones(4106) * 3,
     np.ones(2395) * 4))

# 进行训练和测试样本的分割
from sklearn.model_selection import train_test_split

# 90%的样本用来训练,剩余10%用来测试
X_train, X_test, y_train, y_test = train_test_split(train_pad,
                                                    train_target,
                                                    test_size=0.1,
                                                    random_state=12)

# 用LSTM对样本进行分类
model = Sequential()

# 模型第一层为embedding
model.add(
    Embedding(num_words,
              embedding_dim,
              weights=[embedding_matrix],
              input_length=max_tokens,
              trainable=False))

# model.add(Bidirectional(CuDNNLSTM(units=32, return_sequences=True)))
model.add(Bidirectional(LSTM(units=32, return_sequences=True)))
# model.add(CuDNNLSTM(units=16, return_sequences=False))
model.add(LSTM(units=16, return_sequences=False))

model.add(Dense(5, activation='softmax'))
示例#49
0
    def _model(self):
        """Create the Recurrent Neural Network.

        Args:
            None

        Returns:
            _model: RNN model

        """
        # Create the model object
        _model = Sequential()

        '''
        We can now add a Gated Recurrent Unit (GRU) to the network. This will
        have 512 outputs for each time-step in the sequence.

        Note that because this is the first layer in the model, Keras needs to
        know the shape of its input, which is a batch of sequences of arbitrary
        length (indicated by None), where each observation has a number of
        input-signals (num_x_signals).
        '''

        _model.add(GRU(
            units=self._units,
            return_sequences=True,
            recurrent_dropout=self._dropout,
            input_shape=(None, self._training_vector_count,)))

        for _ in range(0, self._layers):
            _model.add(GRU(
                units=self._units,
                recurrent_dropout=self._dropout,
                return_sequences=True))

        '''
        The GRU outputs a batch of sequences of 512 values. We want to predict
        3 output-signals, so we add a fully-connected (or dense) layer which
        maps 512 values down to only 3 values.

        The output-signals in the data-set have been limited to be between 0
        and 1 using a scaler-object. So we also limit the output of the neural
        network using the Sigmoid activation function, which squashes the
        output to be between 0 and 1.'''

        _model.add(
            Dense(self._training_class_count, activation='sigmoid'))

        '''
        A problem with using the Sigmoid activation function, is that we can
        now only output values in the same range as the training-data.

        For example, if the training-data only has temperatures between -20
        and +30 degrees, then the scaler-object will map -20 to 0 and +30 to 1.
        So if we limit the output of the neural network to be between 0 and 1
        using the Sigmoid function, this can only be mapped back to temperature
        values between -20 and +30.

        We can use a linear activation function on the output instead. This
        allows for the output to take on arbitrary values. It might work with
        the standard initialization for a simple network architecture, but for
        more complicated network architectures e.g. with more layers, it might
        be necessary to initialize the weights with smaller values to avoid
        NaN values during training. You may need to experiment with this to
        get it working.
        '''

        if False:
            # Maybe use lower init-ranges.
            # init = RandomUniform(minval=-0.05, maxval=0.05)
            init = RandomUniform(minval=-0.05, maxval=0.05)

            _model.add(Dense(
                self._training_class_count,
                activation='linear',
                kernel_initializer=init))

        # Compile Model

        '''
        This is the optimizer and the beginning learning-rate that we will use.
        We then compile the Keras model so it is ready for training.
        '''
        optimizer = RMSprop(lr=1e-3)
        _model.compile(
            loss=self._loss_mse_warmup,
            optimizer=optimizer,
            metrics=['accuracy'])

        '''
        This is a very small model with only two layers. The output shape of
        (None, None, 3) means that the model will output a batch with an
        arbitrary number of sequences, each of which has an arbitrary number of
        observations, and each observation has 3 signals. This corresponds to
        the 3 target signals we want to predict.
        '''
        print('> Model Summary:\n')
        print(_model.summary())

        # Return
        return _model
class RNNGRU(object):
    """Support vector machine class."""

    def __init__(self,
                 batch_size=64, sequence_length=20, warmup_steps=50,
                 epochs=20, display=False):
        """Instantiate the class.

        Args:
            batch_size: Size of batch
            sequence_length: Length of vectors for for each target
            save: Save charts if True

        Returns:
            None

        """
        # Initialize key variables
        self.target_names = ['Temp', 'WindSpeed', 'Pressure']
        self.warmup_steps = warmup_steps
        self.epochs = epochs
        self.batch_size = batch_size
        self.display = display

        # Get data
        x_data, y_data = self.data()

        print('\n> Numpy Data Type: {}'.format(type(x_data)))
        print("> Numpy Data Shape: {}".format(x_data.shape))
        print("> Numpy Data Row[0]: {}".format(x_data[0]))
        print('> Numpy Targets Type: {}'.format(type(y_data)))
        print("> Numpy Targets Shape: {}".format(y_data.shape))

        '''
        This is the number of observations (aka. data-points or samples) in
        the data-set:
        '''

        num_data = len(x_data)

        '''
        This is the fraction of the data-set that will be used for the
        training-set:
        '''

        train_split = 0.9

        '''
        This is the number of observations in the training-set:
        '''

        self.num_train = int(train_split * num_data)

        '''
        This is the number of observations in the test-set:
        '''

        num_test = num_data - self.num_train

        print('> Number of Samples: {}'.format(num_data))
        print("> Number of Training Samples: {}".format(self.num_train))
        print("> Number of Test Samples: {}".format(num_test))
        print("> Batch Size: {}".format(batch_size))
        steps_per_epoch = int(self.num_train/batch_size)
        print("> Recommended Epoch Steps: {:.2f}".format(steps_per_epoch))

        # Create test and training data
        x_train = x_data[0:self.num_train]
        x_test = x_data[self.num_train:]
        self.y_train = y_data[0:self.num_train]
        self.y_test = y_data[self.num_train:]
        self.num_x_signals = x_data.shape[1]
        self.num_y_signals = y_data.shape[1]

        print("> Training Minimum Value:", np.min(x_train))
        print("> Training Maximum Value:", np.max(x_train))

        '''
        The neural network works best on values roughly between -1 and 1, so we
        need to scale the data before it is being input to the neural network.
        We can use scikit-learn for this.

        We first create a scaler-object for the input-signals.

        Then we detect the range of values from the training-data and scale
        the training-data.
        '''

        x_scaler = MinMaxScaler()
        self.x_train_scaled = x_scaler.fit_transform(x_train)

        print('> Scaled Training Minimum Value: {}'.format(
            np.min(self.x_train_scaled)))
        print('> Scaled Training Maximum Value: {}'.format(
            np.max(self.x_train_scaled)))

        self.x_test_scaled = x_scaler.transform(x_test)

        '''
        The target-data comes from the same data-set as the input-signals,
        because it is the weather-data for one of the cities that is merely
        time-shifted. But the target-data could be from a different source with
        different value-ranges, so we create a separate scaler-object for the
        target-data.
        '''

        self.y_scaler = MinMaxScaler()
        self.y_train_scaled = self.y_scaler.fit_transform(self.y_train)
        y_test_scaled = self.y_scaler.transform(self.y_test)

        # Data Generator

        '''
        The data-set has now been prepared as 2-dimensional numpy arrays. The
        training-data has almost 300k observations, consisting of 20
        input-signals and 3 output-signals.

        These are the array-shapes of the input and output data:
        '''

        print('> Scaled Training Data Shape: {}'.format(
            self.x_train_scaled.shape))
        print('> Scaled Training Targets Shape: {}'.format(
            self.y_train_scaled.shape))

        # We then create the batch-generator.

        generator = self.batch_generator(batch_size, sequence_length)

        # Validation Set

        '''
        The neural network trains quickly so we can easily run many training
        epochs. But then there is a risk of overfitting the model to the
        training-set so it does not generalize well to unseen data. We will
        therefore monitor the model's performance on the test-set after each
        epoch and only save the model's weights if the performance is improved
        on the test-set.

        The batch-generator randomly selects a batch of short sequences from
        the training-data and uses that during training. But for the
        validation-data we will instead run through the entire sequence from
        the test-set and measure the prediction accuracy on that entire
        sequence.
        '''

        validation_data = (np.expand_dims(self.x_test_scaled, axis=0),
                           np.expand_dims(y_test_scaled, axis=0))

        # Create the Recurrent Neural Network

        self.model = Sequential()

        '''
        We can now add a Gated Recurrent Unit (GRU) to the network. This will
        have 512 outputs for each time-step in the sequence.

        Note that because this is the first layer in the model, Keras needs to
        know the shape of its input, which is a batch of sequences of arbitrary
        length (indicated by None), where each observation has a number of
        input-signals (num_x_signals).
        '''

        self.model.add(GRU(
            units=512,
            return_sequences=True,
            input_shape=(None, self.num_x_signals,)))

        '''
        The GRU outputs a batch of sequences of 512 values. We want to predict
        3 output-signals, so we add a fully-connected (or dense) layer which
        maps 512 values down to only 3 values.

        The output-signals in the data-set have been limited to be between 0
        and 1 using a scaler-object. So we also limit the output of the neural
        network using the Sigmoid activation function, which squashes the
        output to be between 0 and 1.'''

        self.model.add(Dense(self.num_y_signals, activation='sigmoid'))

        '''
        A problem with using the Sigmoid activation function, is that we can
        now only output values in the same range as the training-data.

        For example, if the training-data only has temperatures between -20
        and +30 degrees, then the scaler-object will map -20 to 0 and +30 to 1.
        So if we limit the output of the neural network to be between 0 and 1
        using the Sigmoid function, this can only be mapped back to temperature
        values between -20 and +30.

        We can use a linear activation function on the output instead. This
        allows for the output to take on arbitrary values. It might work with
        the standard initialization for a simple network architecture, but for
        more complicated network architectures e.g. with more layers, it might
        be necessary to initialize the weights with smaller values to avoid
        NaN values during training. You may need to experiment with this to
        get it working.
        '''

        if False:
            # Maybe use lower init-ranges.
            init = RandomUniform(minval=-0.05, maxval=0.05)

            self.model.add(Dense(
                self.num_y_signals,
                activation='linear',
                kernel_initializer=init))

        # Compile Model

        '''
        This is the optimizer and the beginning learning-rate that we will use.
        We then compile the Keras model so it is ready for training.
        '''
        optimizer = RMSprop(lr=1e-3)
        self.model.compile(loss=self.loss_mse_warmup, optimizer=optimizer)

        '''
        This is a very small model with only two layers. The output shape of
        (None, None, 3) means that the model will output a batch with an
        arbitrary number of sequences, each of which has an arbitrary number of
        observations, and each observation has 3 signals. This corresponds to
        the 3 target signals we want to predict.
        '''
        print('> Model Summary:\n')
        print(self.model.summary())

        # Callback Functions

        '''
        During training we want to save checkpoints and log the progress to
        TensorBoard so we create the appropriate callbacks for Keras.

        This is the callback for writing checkpoints during training.
        '''

        path_checkpoint = '/tmp/23_checkpoint.keras'
        callback_checkpoint = ModelCheckpoint(filepath=path_checkpoint,
                                              monitor='val_loss',
                                              verbose=1,
                                              save_weights_only=True,
                                              save_best_only=True)

        '''
        This is the callback for stopping the optimization when performance
        worsens on the validation-set.
        '''

        callback_early_stopping = EarlyStopping(monitor='val_loss',
                                                patience=5, verbose=1)

        '''
        This is the callback for writing the TensorBoard log during training.
        '''

        callback_tensorboard = TensorBoard(log_dir='/tmp/23_logs/',
                                           histogram_freq=0,
                                           write_graph=False)

        '''
        This callback reduces the learning-rate for the optimizer if the
        validation-loss has not improved since the last epoch
        (as indicated by patience=0). The learning-rate will be reduced by
        multiplying it with the given factor. We set a start learning-rate of
        1e-3 above, so multiplying it by 0.1 gives a learning-rate of 1e-4.
        We don't want the learning-rate to go any lower than this.
        '''

        callback_reduce_lr = ReduceLROnPlateau(monitor='val_loss',
                                               factor=0.1,
                                               min_lr=1e-4,
                                               patience=0,
                                               verbose=1)

        callbacks = [callback_early_stopping,
                     callback_checkpoint,
                     callback_tensorboard,
                     callback_reduce_lr]

        # Train the Recurrent Neural Network

        '''We can now train the neural network.

        Note that a single "epoch" does not correspond to a single processing
        of the training-set, because of how the batch-generator randomly
        selects sub-sequences from the training-set. Instead we have selected
        steps_per_epoch so that one "epoch" is processed in a few minutes.

        With these settings, each "epoch" took about 2.5 minutes to process on
        a GTX 1070. After 14 "epochs" the optimization was stopped because the
        validation-loss had not decreased for 5 "epochs". This optimization
        took about 35 minutes to finish.

        Also note that the loss sometimes becomes NaN (not-a-number). This is
        often resolved by restarting and running the Notebook again. But it may
        also be caused by your neural network architecture, learning-rate,
        batch-size, sequence-length, etc. in which case you may have to modify
        those settings.
        '''

        self.model.fit_generator(
            generator=generator,
            epochs=self.epochs,
            steps_per_epoch=steps_per_epoch,
            validation_data=validation_data,
            callbacks=callbacks)

        # Load Checkpoint

        '''
        Because we use early-stopping when training the model, it is possible
        that the model's performance has worsened on the test-set for several
        epochs before training was stopped. We therefore reload the last saved
        checkpoint, which should have the best performance on the test-set.
        '''

        try:
            self.model.load_weights(path_checkpoint)
        except Exception as error:
            print('\n> Error trying to load checkpoint.\n\n{}'.format(error))
            sys.exit(0)

        # Performance on Test-Set

        '''
        We can now evaluate the model's performance on the test-set. This
        function expects a batch of data, but we will just use one long
        time-series for the test-set, so we just expand the
        array-dimensionality to create a batch with that one sequence.
        '''

        result = self.model.evaluate(
            x=np.expand_dims(self.x_test_scaled, axis=0),
            y=np.expand_dims(y_test_scaled, axis=0))

        print('> Loss (test-set): {}'.format(result))

        # If you have several metrics you can use this instead.
        if False:
            for res, metric in zip(result, self.model.metrics_names):
                print('{0}: {1:.3e}'.format(metric, res))

    def batch_generator(self, batch_size, sequence_length):
        """Generator function for creating random batches of training-data.

        Args:
            batch_size: Size of batch
            sequence_length: Length of sequence

        Returns:
            (x_batch, y_batch)

        """
        # Infinite loop.
        while True:
            # Allocate a new array for the batch of input-signals.
            x_shape = (batch_size, sequence_length, self.num_x_signals)
            x_batch = np.zeros(shape=x_shape, dtype=np.float16)

            # Allocate a new array for the batch of output-signals.
            y_shape = (batch_size, sequence_length, self.num_y_signals)
            y_batch = np.zeros(shape=y_shape, dtype=np.float16)

            # Fill the batch with random sequences of data.
            for i in range(batch_size):
                # Get a random start-index.
                # This points somewhere into the training-data.
                idx = np.random.randint(self.num_train - sequence_length)

                # Copy the sequences of data starting at this index.
                x_batch[i] = self.x_train_scaled[idx:idx+sequence_length]
                y_batch[i] = self.y_train_scaled[idx:idx+sequence_length]

            yield (x_batch, y_batch)

    def plot_comparison(self, start_idx, length=100, train=True):
        """Plot the predicted and true output-signals.

        Args:
            start_idx: Start-index for the time-series.
            length: Sequence-length to process and plot.
            train: Boolean whether to use training- or test-set.

        Returns:
            None

        """

        if train:
            # Use training-data.
            x_values = self.x_train_scaled
            y_true = self.y_train
            shim = 'Train'
        else:
            # Use test-data.
            x_values = self.x_test_scaled
            y_true = self.y_test
            shim = 'Test'

        # End-index for the sequences.
        end_idx = start_idx + length

        # Select the sequences from the given start-index and
        # of the given length.
        x_values = x_values[start_idx:end_idx]
        y_true = y_true[start_idx:end_idx]

        # Input-signals for the model.
        x_values = np.expand_dims(x_values, axis=0)

        # Use the model to predict the output-signals.
        y_pred = self.model.predict(x_values)

        # The output of the model is between 0 and 1.
        # Do an inverse map to get it back to the scale
        # of the original data-set.
        y_pred_rescaled = self.y_scaler.inverse_transform(y_pred[0])

        # For each output-signal.
        for signal in range(len(self.target_names)):
            # Create a filename
            filename = (
                '/tmp/batch_{}_epochs_{}_training_{}_{}_{}_{}.png').format(
                    self.batch_size, self.epochs, self.num_train, signal,
                    int(time.time()), shim)

            # Get the output-signal predicted by the model.
            signal_pred = y_pred_rescaled[:, signal]

            # Get the true output-signal from the data-set.
            signal_true = y_true[:, signal]

            # Make the plotting-canvas bigger.
            plt.figure(figsize=(15, 5))

            # Plot and compare the two signals.
            plt.plot(signal_true, label='true')
            plt.plot(signal_pred, label='pred')

            # Plot grey box for warmup-period.
            _ = plt.axvspan(
                0, self.warmup_steps, facecolor='black', alpha=0.15)

            # Plot labels etc.
            plt.ylabel(self.target_names[signal])
            plt.legend()

            # Show and save the image
            if self.display is True:
                plt.savefig(filename, bbox_inches='tight')
                plt.show()
            else:
                plt.savefig(filename, bbox_inches='tight')
            print('> Saving file: {}'.format(filename))

    def data(self):
        """Get data to analyze.

        Args:
            None

        Returns:
            (x_data, y_data): X and Y values as numpy arrays

        """
        # Download data
        weather.maybe_download_and_extract()

        # Import data into Pandas dataframe
        pandas_df = weather.load_resampled_data()
        print('\n> First Rows of Data:\n\n{}'.format(pandas_df.head(3)))

        # Print the cities
        cities = weather.cities
        print('\n> Cities: {}'.format(cities))

        # Print dataframe shape
        print(
            '> Dataframe shape (Original): {}'.format(pandas_df.values.shape))

        # The two signals that have missing data. (Columns with Nans)
        pandas_df.drop(('Esbjerg', 'Pressure'), axis=1, inplace=True)
        pandas_df.drop(('Roskilde', 'Pressure'), axis=1, inplace=True)

        # Print dataframe shape
        print('> Dataframe shape (New): {}'.format(pandas_df.values.shape))

        # Verify that the columns have been dropped
        print(
            '\n> First Rows of Updated Data:\n\n{}'.format(pandas_df.head(1)))

        # Add Data

        '''
        We can add some input-signals to the data that may help our model in
        making predictions.

        For example, given just a temperature of 10 degrees Celcius the model
        wouldn't know whether that temperature was measured during the day or
        the night, or during summer or winter. The model would have to infer
        this from the surrounding data-points which might not be very accurate
        for determining whether it's an abnormally warm winter, or an
        abnormally cold summer, or whether it's day or night. So having this
        information could make a big difference in how accurately the model can
        predict the next output.

        Although the data-set does contain the date and time information for
        each observation, it is only used in the index so as to order the data.
        We will therefore add separate input-signals to the data-set for the
        day-of-year (between 1 and 366) and the hour-of-day (between 0 and 23).
        '''

        pandas_df['Various', 'Day'] = pandas_df.index.dayofyear
        pandas_df['Various', 'Hour'] = pandas_df.index.hour

        # Target Data for Prediction

        '''
        We will try and predict the future weather-data for this city.
        '''

        target_city = 'Odense'

        '''
        We will try and predict these signals.
        '''

        self.target_names = ['Temp', 'WindSpeed', 'Pressure']

        '''
        The following is the number of time-steps that we will shift the
        target-data. Our data-set is resampled to have an observation for each
        hour, so there are 24 observations for 24 hours.

        If we want to predict the weather 24 hours into the future, we shift
        the data 24 time-steps. If we want to predict the weather 7 days into
        the future, we shift the data 7 * 24 time-steps.
        '''

        shift_days = 1
        shift_steps = shift_days * 24  # Number of hours.

        # Create a new data-frame with the time-shifted data.

        '''
        Note the negative time-shift!

        We want the future state targets to line up with the timestamp of the
        last value of each sample set.
        '''

        df_targets = pandas_df[
            target_city][self.target_names].shift(-shift_steps)

        '''
        WARNING! You should double-check that you have shifted the data in the
        right direction! We want to predict the future, not the past!

        The shifted data-frame is confusing because Pandas keeps the original

        This is the first shift_steps + 5 rows of the original data-frame:
        '''

        explanatory_hours = shift_steps + 5
        print('\n> First Rows of Updated Data ({} hours):\n\n{}'.format(
            explanatory_hours,
            pandas_df[target_city][self.target_names].head(explanatory_hours)))

        '''
        The following is the first 5 rows of the time-shifted data-frame. This
        should be identical to the last 5 rows shown above from the original
        data, except for the time-stamp.
        '''
        print('\n> First Rows of Shifted Data - Target Labels '
              '(Notice 1980 Dates):\n\n{}'.format(df_targets.head(5)))

        '''
        The time-shifted data-frame has the same length as the original
        data-frame, but the last observations are NaN (not a number) because
        the data has been shifted backwards so we are trying to shift data that
        does not exist in the original data-frame.
        '''
        print('\n> Last Rows of Shifted Data - Target Labels '
              '(Notice 2018 Dates):\n\n{}'.format(df_targets.tail()))

        # NumPy Arrays

        '''
        We now convert the Pandas data-frames to NumPy arrays that can be input
        to the neural network. We also remove the last part of the numpy
        arrays, because the target-data has NaN for the shifted period, and we
        only want to have valid data and we need the same array-shapes for the
        input- and output-data.

        These are the input-signals:
        '''

        x_data = pandas_df.values[0:-shift_steps]
        y_data = df_targets.values[:-shift_steps]

        # Return
        return (x_data, y_data)

    def loss_mse_warmup(self, y_true, y_pred):
        """Calculate the Mean Squared Errror.

        Calculate the Mean Squared Error between y_true and y_pred,
        but ignore the beginning "warmup" part of the sequences.

        We will use Mean Squared Error (MSE) as the loss-function that will be
        minimized. This measures how closely the model's output matches the
        true output signals.

        However, at the beginning of a sequence, the model has only seen
        input-signals for a few time-steps, so its generated output may be very
        inaccurate. Using the loss-value for the early time-steps may cause the
        model to distort its later output. We therefore give the model a
        "warmup-period" of 50 time-steps where we don't use its accuracy in the
        loss-function, in hope of improving the accuracy for later time-steps

        Args:
            y_true: Desired output.
            y_pred: Model's output.

        Returns:
            loss_mean: Mean Squared Error

        """
        warmup_steps = self.warmup_steps

        # The shape of both input tensors are:
        # [batch_size, sequence_length, num_y_signals].

        # Ignore the "warmup" parts of the sequences
        # by taking slices of the tensors.
        y_true_slice = y_true[:, warmup_steps:, :]
        y_pred_slice = y_pred[:, warmup_steps:, :]

        # These sliced tensors both have this shape:
        # [batch_size, sequence_length - warmup_steps, num_y_signals]

        # Calculate the MSE loss for each value in these tensors.
        # This outputs a 3-rank tensor of the same shape.
        loss = tf.losses.mean_squared_error(labels=y_true_slice,
                                            predictions=y_pred_slice)

        # Keras may reduce this across the first axis (the batch)
        # but the semantics are unclear, so to be sure we use
        # the loss across the entire tensor, we reduce it to a
        # single scalar with the mean function.
        loss_mean = tf.reduce_mean(loss)

        return loss_mean