def compression(tr_data,test_data):
    #normalize data
    #Create autoencoder models
    model=autoencoderKeras.create_model(num_of_hidden_layers,len(tr_data[0,:]),num_of_neurons,layer_activation,output_activation)
    #pretraining
    model=autoencoderKeras.pretrain(model,tr_data,0.8,layer_activation,output_activation,'RMSprop')
    weights1=model.get_weights()
    #training
    model=autoencoderKeras.overall_train(model,tr_data,0.1,'RMSprop')
    #test of ae on training set
    tr_pred = model.predict(tr_data)
    #test of ae on test set
    test_pred = model.predict(test_data)
    #coding model
    code_model=Sequential()
    for i in range(num_of_hidden_layers):
        if i==0:
            code_model.add(Dense(num_of_neurons[i],activation='relu',input_dim=len(tr_data[0,:])))
        else:
            code_model.add(Dense(num_of_neurons[i],activation='relu'))

    #copy trained weights
    weights=model.get_weights()
    code_model.set_weights(weights[0:2*num_of_hidden_layers])
    #codes
    test_code=code_model.predict(test_data)
    tr_code=code_model.predict(tr_data)
    return tr_pred,test_pred,tr_code,test_code,weights1
Example #2
0
def build_overkill_stacked_lstm_regularized_dropout(dx, dh, do, length, weights=None):
    model = Sequential()
    model.add(LSTM(
        dh,
        input_dim=dx,
        return_sequences=True,
        W_regularizer='l2',
        U_regularizer='l2',
        b_regularizer='l2'
    ))
    model.add(Dropout(0.2))
    model.add(LSTM(
        512,
        input_dim=dh,
        return_sequences=True,
        W_regularizer='l2',
        U_regularizer='l2',
        b_regularizer='l2'
    ))
    model.add(Dropout(0.2))
    model.add(LSTM(
        do,
        input_dim=512,
        return_sequences=True,
        activation='linear',
        W_regularizer='l2',
        U_regularizer='l2',
        b_regularizer='l2'
    ))
    if weights is not None:
        model.set_weights(weights)
    return model
Example #3
0
def test_saving_overwrite_option_gcs():
    model = Sequential()
    model.add(Dense(2, input_shape=(3,)))
    org_weights = model.get_weights()
    new_weights = [np.random.random(w.shape) for w in org_weights]

    with tf_file_io_proxy('keras.engine.saving.tf_file_io') as file_io_proxy:
        gcs_filepath = file_io_proxy.get_filepath(
            filename='test_saving_overwrite_option_gcs.h5')
        # we should not use same filename in several tests to allow for parallel
        # execution
        save_model(model, gcs_filepath)
        model.set_weights(new_weights)

        with patch('keras.engine.saving.ask_to_proceed_with_overwrite') as ask:
            ask.return_value = False
            save_model(model, gcs_filepath, overwrite=False)
            ask.assert_called_once()
            new_model = load_model(gcs_filepath)
            for w, org_w in zip(new_model.get_weights(), org_weights):
                assert_allclose(w, org_w)

            ask.return_value = True
            save_model(model, gcs_filepath, overwrite=False)
            assert ask.call_count == 2
            new_model = load_model(gcs_filepath)
            for w, new_w in zip(new_model.get_weights(), new_weights):
                assert_allclose(w, new_w)

        file_io_proxy.delete_file(gcs_filepath)  # cleanup
Example #4
0
def test_saving_overwrite_option():
    model = Sequential()
    model.add(Dense(2, input_shape=(3,)))
    org_weights = model.get_weights()
    new_weights = [np.random.random(w.shape) for w in org_weights]

    _, fname = tempfile.mkstemp('.h5')
    save_model(model, fname)
    model.set_weights(new_weights)

    with patch('keras.engine.saving.ask_to_proceed_with_overwrite') as ask:
        ask.return_value = False
        save_model(model, fname, overwrite=False)
        ask.assert_called_once()
        new_model = load_model(fname)
        for w, org_w in zip(new_model.get_weights(), org_weights):
            assert_allclose(w, org_w)

        ask.return_value = True
        save_model(model, fname, overwrite=False)
        assert ask.call_count == 2
        new_model = load_model(fname)
        for w, new_w in zip(new_model.get_weights(), new_weights):
            assert_allclose(w, new_w)

    os.remove(fname)
Example #5
0
def build_train_lstm_mse(dx, dh, do, span=1, weights=None, batch_size=2):
    model = Sequential()
    model.add(LSTM(
        dh,
        input_dim=dx,
        return_sequences=False
    ))
    model.add(Dense(do))
    if weights is not None:
        model.set_weights(weights)
    return model
Example #6
0
def build_test_rnn_mse(dx, dh, do, weights=None):
    model = Sequential()
    model.add(SimpleRNN(
        dh,
        input_dim=dx,
        return_sequences=True
    ))
    model.add(TimeDistributed(Dense(do)))
    if weights is not None:
        model.set_weights(weights)
    return model
Example #7
0
def build_simple_rnn_stateful(dx, dh, do, length, weights=None, batch_size=1):
    model = Sequential()
    model.add(SimpleRNN(
        dh,
        batch_input_shape=(batch_size, 1, dx),
        return_sequences=True,
        stateful=True
    ))
    model.add(TimeDistributed(Dense(do)))
    if weights is not None:
        model.set_weights(weights)
    return model
Example #8
0
def build_softmax_rnn(dx, dh, do, length, weights=None):
    model = Sequential()
    model.add(SimpleRNN(
        dh,
        input_dim=dx,
        return_sequences=True
    ))
    model.add(TimeDistributed(Dense(do), activation='softmax'))

    if weights is not None:
        model.set_weights(weights)
    return model
Example #9
0
def build_test_lstm_softmax(dx, dh, do, weights=None):
    model = Sequential()
    model.add(LSTM(
        dh,
        input_dim=dx,
        return_sequences=True
    ))
    model.add(TimeDistributed(Dense(do)))
    model.add(TimeDistributed(Activation('softmax')))
    if weights is not None:
        model.set_weights(weights)
    return model
Example #10
0
def build_test_lstm_mse(dx, dh, do, weights=None):
    model = Sequential()
    model.add(LSTM(
        dh,
        input_dim=dx,
        return_sequences=True
    ))
    model.add(TimeDistributed(Dense(do)))
    if weights is not None:
        print(len(weights))
        model.set_weights(weights)
    return model
Example #11
0
def build_lstm_stateful_softmax(dx, dh, do, length=1, weights=None, batch_size=1):
    model = Sequential()
    model.add(LSTM(
        dh,
        batch_input_shape=(batch_size, length, dx),
        return_sequences=False,
        stateful=True
    ))
    model.add(Dense(do))
    model.add(Activation('softmax'))
    if weights is not None:
        model.set_weights(weights)
    return model
Example #12
0
def test():
    with open("save_weight.pickle", mode="rb") as f:
        weights = pickle.load(f)

    model = Sequential()
    model.add(Dense(output_dim=100, input_dim=28*28))
    model.add(Activation("relu"))
    model.set_weights(weights)

    layey1_value = model.predict(X_test[:5])
    y_pred = np_utils.categorical_probas_to_classes(y)
    Y = np_utils.categorical_probas_to_classes(y_test)
    print np_utils.accuracy(y_pred,Y)
    print y_pred.shape
Example #13
0
def build_stacked_rnn(dx, dh, do, length, weights=None):
    model = Sequential()
    model.add(SimpleRNN(
        dh,
        input_dim=dx,
        return_sequences=True
    ))
    model.add(SimpleRNN(
        do,
        input_dim=dh,
        return_sequences=True,
    ))
    if weights is not None:
        model.set_weights(weights)
    return model
Example #14
0
def build_stacked_lstm(dx, dh, do, length, weights=None):
    model = Sequential()
    model.add(LSTM(
        dh,
        input_dim=dx,
        return_sequences=True
    ))
    model.add(LSTM(
        do,
        input_dim=dh,
        return_sequences=True
    ))
    model.add(TimeDistributed(Dense(do)))
    if weights is not None:
        model.set_weights(weights)
    return model
Example #15
0
def build_stacked_lstm_mse_stateful(dx, dh, do, length, weights=None, batch_size=5):
    model = Sequential()
    model.add(LSTM(
        dh,
        batch_input_shape=(batch_size, 1, dx),
        return_sequences=True,
        stateful=True
    ))
    model.add(LSTM(
        do,
        batch_input_shape=(batch_size, 1, dh),
        return_sequences=True,
        stateful=True
    ))
    model.add(TimeDistributed(Dense(do)))
    if weights is not None:
        model.set_weights(weights)
    return model
Example #16
0
def build_train_stacked_lstm_dropout_softmax(dx, dh, do, span=1, weights=None, batch_size=2):
    model = Sequential()
    model.add(LSTM(
        dh,
        input_dim=dx,
        return_sequences=True
    ))
    model.add(Dropout(0.2))
    model.add(LSTM(
        dh,
        input_dim=dh,
        return_sequences=False
    ))
    model.add(Dense(do))
    model.add(Activation('softmax'))
    if weights is not None:
        model.set_weights(weights)
    return model
Example #17
0
def test_EarlyStopping_reuse():
    patience = 3
    data = np.random.random((100, 1))
    labels = np.where(data > 0.5, 1, 0)
    model = Sequential((
        Dense(1, input_dim=1, activation='relu'),
        Dense(1, activation='sigmoid'),
    ))
    model.compile(optimizer='sgd', loss='binary_crossentropy', metrics=['accuracy'])
    stopper = callbacks.EarlyStopping(monitor='acc', patience=patience)
    weights = model.get_weights()

    hist = model.fit(data, labels, callbacks=[stopper])
    assert len(hist.epoch) >= patience

    # This should allow training to go for at least `patience` epochs
    model.set_weights(weights)
    hist = model.fit(data, labels, callbacks=[stopper])
    assert len(hist.epoch) >= patience
Example #18
0
class brain:
    def __init__(self, model):
        if (model == None):
            self.model = Sequential()
            self.model.add(
                Dense(8, activation="tanh", input_dim=6,
                      kernel_initializer=initializers.RandomUniform(minval=-1, maxval=1, seed=None)))
            self.model.add(
                Dense(3, activation="tanh",
                      kernel_initializer=initializers.RandomUniform(minval=-1, maxval=1, seed=None)))
            self.model.compile(loss='mean_squared_error', optimizer='adam')
        else:
            self.model = model

    def getOutputs(self, inputs):
        inputs.append(1)
        return self.model.predict(np.asarray([inputs]))

    def mutate(self, brain1, brain2):
        newBrain = []
        for i in range(0, len(self.model.get_weights()), 2):
            newWeights = []
            b1weights = brain1.get_weights()[i]
            b2weights = brain2.get_weights()[i]
            for n in range(len(b1weights)):
                w = []
                for m in range(len(b1weights[0])):
                    r = random()
                    k = 0
                    if random() < 0.1:
                        k = randint(-100, 100) / 100

                    if (r < 0.4):
                        w.append(b1weights[n][m] + k)
                    elif r > 0.6:
                        w.append(b2weights[n][m] + k)
                    else:
                        w.append((b1weights[n][m] + b2weights[n][m]) / 2 + k)

                newWeights.append(w)
            newBrain.append(newWeights)
            newBrain.append(self.model.get_weights()[i + 1])
        self.model.set_weights(newBrain)
Example #19
0
def test_save_load_weights_gcs():
    model = Sequential()
    model.add(Dense(2, input_shape=(3,)))
    org_weights = model.get_weights()

    with tf_file_io_proxy('keras.engine.saving.tf_file_io') as file_io_proxy:
        gcs_filepath = file_io_proxy.get_filepath(
            filename='test_save_load_weights_gcs.h5')
        # we should not use same filename in several tests to allow for parallel
        # execution
        model.save_weights(gcs_filepath)
        model.set_weights([np.random.random(w.shape) for w in org_weights])
        for w, org_w in zip(model.get_weights(), org_weights):
            assert not (w == org_w).all()
        model.load_weights(gcs_filepath)
        for w, org_w in zip(model.get_weights(), org_weights):
            assert_allclose(w, org_w)

        file_io_proxy.delete_file(gcs_filepath)  # cleanup
def train_model(feature_layers, classification_layers, image_list, nb_epoch, nb_classes, img_rows, img_cols, weights=None): 
    # Create testset data for cross-val
    num_images = len(image_list)
    test_size = int(0.2 * num_images)
    print("Train size: ", num_images-test_size)
    print("Test size: ", test_size)

    model = Sequential()
    for l in feature_layers + classification_layers:
        model.add(l)

    if not(weights is None):
        model.set_weights(weights)

    # let's train the model using SGD + momentum (how original).
    sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
    model.compile(loss='categorical_crossentropy', optimizer=sgd)
    
    print('Using real time data augmentation')
    for e in range(nb_epoch):
        print('-'*40)
        print('Epoch', e)
        print('-'*40)
        print('Training...')
        # batch train with realtime data augmentation
        progbar = generic_utils.Progbar(num_images-test_size)
        for X_batch, Y_batch in flow(image_list[0:-test_size]):
            X_batch = X_batch.reshape(X_batch.shape[0], 3, img_rows, img_cols)
            Y_batch = np_utils.to_categorical(Y_batch, nb_classes)
            loss = model.train_on_batch(X_batch, Y_batch)
            progbar.add(X_batch.shape[0], values=[('train loss', loss)])

        print('Testing...')
        # test time!
        progbar = generic_utils.Progbar(test_size)
        for X_batch, Y_batch in flow(image_list[-test_size:]):
            X_batch = X_batch.reshape(X_batch.shape[0], 3, img_rows, img_cols)
            Y_batch = np_utils.to_categorical(Y_batch, nb_classes)
            score = model.test_on_batch(X_batch, Y_batch)
            progbar.add(X_batch.shape[0], values=[('test loss', score)])
    return model, model.get_weights()
Example #21
0
class Brain:
    def __init__(self, model):
        if (model == None):
            self.model = Sequential()
            self.model.add(Dense(12, input_dim=6, activation="tanh",
                                 kernel_initializer=initializers.RandomUniform(minval=-1, maxval=1, seed=None)))

            # self.model.add(Dense(20, activation="tanh",
            #                      kernel_initializer=initializers.RandomUniform(minval=-1, maxval=1, seed=None)))
            # self.model.add(Dense(20, activation="tanh",
            #                      kernel_initializer=initializers.RandomUniform(minval=-1, maxval=1, seed=None)))
            # self.model.add(Dense(20, activation="tanh",
            #                      kernel_initializer=initializers.RandomUniform(minval=-1, maxval=1, seed=None)))
            self.model.add(Dense(3, activation="tanh",
                                 kernel_initializer=initializers.RandomUniform(minval=-1, maxval=1, seed=None)))
            self.model.compile(optimizer='sgd', loss='mean_squared_error')
        else:
            self.model = model

    def getOutputs(self, inputs):
        return self.model.predict(np.asarray([inputs]))

    def breed(self, brain1, brain2):
        newBrain = []
        for i in range(0, len(self.model.get_weights()), 2):
            newWeights = []
            b1weights = brain1.model.get_weights()[i]
            b2weights = brain2.model.get_weights()[i]
            for j in range(len(b1weights)):
                w = []
                for k in range(len(b1weights[0])):
                    r = random()
                    if r > 0.8:
                        genome = choice([b1weights[j][k], b2weights[j][k]])
                        w.append(genome + randint(-200, 200)/1000)
                    else:
                        w.append(choice([b1weights[j][k], b2weights[j][k]]))
                newWeights.append(w)
            newBrain.append(newWeights)
            newBrain.append(self.model.get_weights()[i + 1])
        self.model.set_weights(newBrain)
Example #22
0
File: nn.py Project: deesatz/dlopt
class RNNBuilder(NNBuilder):
    """ Recurrent neural network builder
    """
    def build_model(self,
                    layers,
                    cell=LSTM,
                    weights=None,
                    dense_activation='tanh',
                    verbose=0,
                    **kwargs):
        # self.hidden_layers=len(layers) - 2
        # self.layers=layers
        # self.input_dim=layers[0]
        # self.output_dim=layers[-1]
        self.model = Sequential()
        for i in range(len(layers) - 2):
            self.model.add(cell(
                # Keras API 2
                input_shape=(None, layers[i]),
                units=layers[i+1],
                # Keras API 1
                # input_dim=layers[i],
                # output_dim=layers[i+1],
                kernel_initializer='zeros',
                recurrent_initializer='zeros',
                bias_initializer='zeros',
                # Uncomment to use last batch state to init next training step.
                # Specify shuffle=False when calling fit()
                # batch_size=batch_size, stateful=True,
                return_sequences=True if i < (len(layers) - 3) else False))
        self.model.add(Dense(layers[-1],
                       activation=dense_activation,
                       kernel_initializer='zeros',
                       bias_initializer='zeros'))
        if weights:
            self.model.set_weights(weights)
        self.trainable_params = int(np.sum(
              [K.count_params(p) for p in set(self.model.trainable_weights)]))
        if verbose > 1:
            self.model.summary()
        return self.model
Example #23
0
def _test_equivalence(channel_order=None):

    from kfs.layers.convolutional import Convolution2DEnergy_TemporalBasis
    from keras.models import Sequential
    #from keras.layers import Flatten, Dense
    input_shape = (12, 3, 64, 64)
    if channel_order is None:
        channel_order = K.image_data_format()
    if channel_order == 'channels_last':
        input_shape = (12, 64, 64, 3)


    nn = Sequential()
    nn.add(Convolution2DEnergy_TemporalBasis(8, 16, 4, (5, 5), 7,
                                            padding='same',
                                            input_shape=input_shape,
                                            data_format=channel_order))

    rng = np.random.RandomState(42)
    datums = rng.randn(6, 12, 3, 64, 64).astype('float32')
    if channel_order == 'channels_last':
        datums = datums.transpose(0, 1, 3, 4, 2)


    nn.compile(loss='mse', optimizer='sgd')

    nn2 = Sequential()
    nn2.add(Convolution2DEnergy_TemporalCorrelation(8, 16, 4, (5, 5), 7,
                                            padding='same',
                                            input_shape=input_shape,
                                            data_format=channel_order))
    nn2.compile(loss='mse', optimizer='sgd')
    nn2.set_weights(nn.get_weights())

    pred1 = nn.predict(datums)
    pred2 = nn2.predict(datums)
    assert ((pred1 - pred2) == 0.).all()

    return nn, nn.predict(datums), nn2, nn2.predict(datums)
print("Train...")
model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=5, validation_split=0.1, show_accuracy=True)
score = model.evaluate(X_test, Y_test, batch_size=batch_size)
print('Test score:', score)

classes = model.predict_classes(X_test, batch_size=batch_size)
acc = np_utils.accuracy(classes, Y_test)

print('Test accuracy:', acc)

store_weights = {}
for layer in model.layers :
    store_weights[layer] = layer.get_weights() 

# create a new model of same structure minus last layers, to explore intermediate outputs
print('Build truncated model')
chopped_model = Sequential()
chopped_model.add(Embedding(max_features, 256))
chopped_model.add(LSTM(256, 128))
chopped_model.add(Dense(128, nb_classes))
chopped_model.set_weights(model.get_weights())
chopped_model.compile(loss='categorical_crossentropy', optimizer='adam', class_mode="categorical")

# pickle intermediate activations, model weights, accuracy
train_activations = chopped_model.predict(X_train, batch_size=batch_size)
test_activations = chopped_model.predict(X_test, batch_size=batch_size)
outputs = dict(final=classes, train_activations=train_activations, test_activations=test_activations, acc=acc) 
pkl.dump(outputs, open('results/predicted_activations_categories.pkl', 'wb'), 
    protocol=pkl.HIGHEST_PROTOCOL)
Example #25
0
# model.add(Dropout(0.2))
model.add(LGRU2(80, 
    return_sequences=True,
    inner_activation='sigmoid',
    activation='tanh'
    )
)
# # model.add(Dropout(0.2))
model.add(LGRU2(90, 
    return_sequences=True,
    inner_activation='sigmoid',
    activation='tanh'
    )
)
model.add(TimeDistributedDense(outputsize))
model.add(Activation('softmax'))

opt = RMSprop(lr=learning_rate, rho=0.9, epsilon=1e-6, clipvalue=clipval)
model.compile(loss='categorical_crossentropy', optimizer=opt)


res = pickle.load(gzip.open(paramsfile,'r'))
W = res['weights']
model.set_weights(W)

print(' -- Text sampling ---')
temperatures = [0.7, 1]
generated = text_sampling_char(
    model,vocabs,
    temperatures, 
    ns=400)
                else:
                    raise Exception("language feature dim error!")
                totalloss += loss[0]
                if (j+1) % 100 == 0:
                    print 'epoch #{:03d}, batch #{:03d}, current avg loss = {:.3f}'.format(i+1, j+1, totalloss/(j+1))
                    logfile.write('epoch #{:03d}, batch #{:03d}, current avg loss = {:.3f}\n'.format(i+1, j+1, totalloss/(j+1)))
            if (i+1) % 5 == 0:
                model.save_weights(model_file_name + '_epoch_{:05d}_loss_{:.3f}.hdf5'.format(i+1, totalloss/batchNum))
    else:
        # cross valid & training
        dataSize = len(idList)
        setSize = dataSize / arg.cross_valid
        crossvalidList = []
        for k in xrange(arg.cross_valid):
            #reset the weights to initial
            model.set_weights(weights_save)

            # cut train and valid id list
            if k == arg.cross_valid -1:
                validIdList = idList[k * setSize:]
                trainIdList = idList[:k * setSize]
            else:
                validIdList = idList[k * setSize : (k + 1) * setSize]
                trainIdList = idList[:k * setSize] + idList[(k + 1) * setSize:]
            
            # for save avg totalerr in each cross validation
            totalerror = 0
            for i in xrange(arg.epochs):
                #print 'valid #{:02d}, epoch #{:03d}'.format(k+1, i+1)
                #logfile.write('valid #{:02d}, epoch #{:03d}\n'.format(k+1, i+1))
                # training
#training
model=autoencoderKeras.overall_train(model,tr_set_st,0.5)
#test of ae on training set
tr = model.predict(tr_set_st)
tr_pred=denormalize(tr,tr_data_stands)
#test of ae on test set
test = model.predict(test_set_st)
test_pred=denormalize(test,test_data_stands)
#coding model
code_model=Sequential()
code_model.add(Dense(4096,activation='sigmoid',input_dim=len(tr_set_st[0,:])))
code_model.add(Dense(512,activation='sigmoid'))
code_model.add(Dense(64,activation='sigmoid'))
#copy trained weighs
weights=model.get_weights()
code_model.set_weights(weights[0:6])
#codes
test_code=code_model.predict(test_set_st)
tr_code=code_model.predict(tr_set_st)
#linear svm on raw training
clf =  SVC(class_weight='balanced')
clf.fit(tr_code, tr_labels)
predicted_labels=clf.predict(tr_code)
conf_mat=confusion_matrix(tr_labels,predicted_labels)
print(conf_mat)

predicted_labels=clf.predict(test_code)
conf_mat=confusion_matrix(test_labels,predicted_labels)
print(conf_mat)

Example #28
0
class KerasRegressionModel(RegressionModel):
    def __init__(self, arity=1, network_structure=(1,), activation_function="tanh",
                 error_metric="rmse",
                 optimizer_type="nadam", learning_rate=None, loss_function="mse", nb_epoch=1, batch_size=100,
                 early_stopping=False,
                 weight_init_method="normal", graphical_verbose=False,
                 validation_split=0.1, dropout=False, dropout_input_layer_fraction=0.2,
                 dropout_hidden_layer_fraction=0.5, batch_normalization=False, verbose=False, weight_decay=False,
                 weigt_decay_parameter=0.001,
                 **kwargs):
        """

        A class to construct arbitrary artifical neural networks using Keras library (http://keras.io/).
        The module supports state-of-the-art technologies for optimization and regularization of ANNs.

        :param network_structure: A tuple which specifies the number of neurons for each layer
        :param activation_function: Activation function used, cf. http://keras.io/activations/
        :param error_metric: Error metric
        :param optimizer_type: Specifies the optimization method used
        :param loss_function: Loss function (given by Keras or custom loss functions), cf. http://keras.io/objectives/
        :param nb_epoch: Number of training epochs
        :param batch_size: Batch size used for mini-batch learning
        :param early_stopping: If set True, training will be interruptped when the loss isn't decaying anymore
        :param init: Method of weight initialization, e.g normal, glorot_normal, uniform
        :param arity: Input dimension
        :param verbose: Verbose mode, verbose=1 show progress bar logging, verbose=2 show console logging
        :param graphical_verbose: If True,
        :param dropout: Use dropout layers for regularization
        :param dropout_input_layer_fraction: Fraction of input units to drop
        :param dropout_hidden_layer_fraction: Fraction hidden layer units to drop
        :param batch_normalization: Activate batch normalization
        :param weight_decay: Activate weight decay regularization method
        :param weight_decay_parameter: Sets the weight decay regularization parameter
        :param kwargs:
        """

        super(RegressionModel, self).__init__()
        #self.logger.info("Compiling ANN...")
        self.__dict__.update(locals())

        # Initialize ANN structure

        self.__model = Sequential()

        self.input_layer_params = {"input_shape": (self.arity,), "activation": self.activation_function,
                                       "output_dim": self.network_structure[0], "init": self.weight_init_method}

        self.hidden_layer_params = {"activation": self.activation_function, "init": self.weight_init_method}
        if self.weight_decay:
            self.hidden_layer_params["W_regularizer"] = l2(weigt_decay_parameter)

        self.output_layer_params = {"activation": "linear", "init": self.weight_init_method}

        self.create_input_layer()  # stack up remaining layers.
        self.create_hidden_layers()
        self.create_output_layer()


        # compile the neural network
        self.__model.compile(optimizer=RMSprop(lr=0.001), loss=self.loss_function)
        #self.logger.info("Compilation completed...")
        self.func = self.__model.predict


    def add_layer(self, num_nodes, layer_params, dropout=False):
        self.__model.add(Dense(num_nodes, **layer_params))
        if (dropout):
            self.__model.add(Dropout(self.dropout_hidden_layer_fraction))

    def create_input_layer(self):
        if self.dropout:
            self.__model.add(Dropout(self.dropout_input_layer_fraction, input_shape=(self.arity,)))
            del self.input_layer_params["input_shape"]
        self.__model.add(Dense(**self.input_layer_params))
        if self.batch_normalization:
            self.__model.add(BatchNormalization())

    def create_hidden_layers(self):
        for num_nodes in self.network_structure[1:-1]:
            self.add_layer(num_nodes, self.hidden_layer_params, dropout=self.dropout)
            if self.batch_normalization:
                self.__model.add(BatchNormalization())

    def create_output_layer(self):
        self.add_layer(self.network_structure[-1], self.output_layer_params)
        if self.batch_normalization:
            self.__model.add(BatchNormalization())


    @property
    def weights(self):
        return self.__model.get_weights()

    #@doc_inherit
    def fit(self, xfit, yfit):
        self.hist = self.__model.fit(xfit, yfit, nb_epoch=self.nb_epoch, batch_size=self.batch_size,
                                     verbose=self.verbose, validation_split=self.validation_split)#, callbacks=self.callbacks)
        return self

    def __getstate__(self):
        """
        Function to make ANNRegressionModel pickable.
        The weights, the architecture as also the ANN compilation settings are stored in a dictionary.
        :return: The dictionary containing ANN architecture in json format, weight and ANN compilation setting
        """

        state = copy(self.__dict__)
        del state["func"]
        #del state["logger"]
        #del state["_ANNRegressionModel__model"]
        del state["hist"]
        return dict(json_model=self.__model.to_json(), weights=self.__model.get_weights(), config=state)

    def __setstate__(self, d):
        """
        Function to make ANNRegressionModel pickable
        :param d:
        :return:
        """

        self.__dict__ = d["config"]
        self.__model = model_from_json(d["json_model"])
        self.__model.set_weights(d["weights"])
        self.func = self.__model.predict

    def print_summary(self):
        """
        Print summary of the neural network, includes architecture and compiling setting.
        :return:
        """
        self.__model.summary()
Example #29
0
base_path = "../../../../../../../../resources/weights/"
backend = K.backend()
version = keras.__version__
major_version = int(version[0])

if major_version == 2:
    from keras.layers import Conv2D
else:
    from keras.layers import Convolution2D as Conv2D

input_shape=(5, 5, 5)
n_out = 6
kernel_size = (3, 3)

weights = np.arange(0, kernel_size[0] * kernel_size[1] * input_shape[0] * n_out)
weights = weights.reshape((kernel_size[0], kernel_size[1], input_shape[0], n_out))
bias = np.arange(0, n_out)

model = Sequential()
if major_version == 2:
    model.add(Conv2D(n_out, kernel_size, input_shape=input_shape))
else:
    model.add(Conv2D(n_out, kernel_size[0], kernel_size[1], input_shape=input_shape))

model.set_weights([weights, bias])

model.compile(loss='mse', optimizer='adam')

print("Saving model with single 2D convolution layer for backend {} and keras major version {}".format(backend, major_version))
model.save("{}conv2d_{}_{}.h5".format(base_path, backend, major_version))
def evaluate_fold(fold_ix, use_pretrained_embedding, bi_directional, num_rnns, merge_mode, hidden_size):
    if use_pretrained_embedding:
        embedding_matrix = get_embedding_matrix(unique_words, generator, max_features, init='uniform',
                                                unit_length=False)
        embedding_layer = Embedding(max_features,
                                    EMBEDDING_DIM,
                                    weights=[embedding_matrix],
                                    input_length=maxlen,
                                    trainable=True,
                                    mask_zero=True)  # If false, initialize unfound words with all 0's
    else:
        embedding_layer = Embedding(max_features, embedding_size, input_length=maxlen, trainable=True, mask_zero=True)

    if bi_directional:
        rnn_layer_fact = lambda: Bidirectional(GRU(hidden_size, return_sequences=True, consume_less="cpu"),
                                               merge_mode=merge_mode)
    else:
        rnn_layer_fact = lambda: GRU(hidden_size, return_sequences=True, consume_less="cpu")

    model = Sequential()
    model.add(embedding_layer)
    for i in range(num_rnns):
        model.add(rnn_layer_fact())

    model.add(TimeDistributedDense(out_size))
    model.add(Activation('softmax'))

    model.compile(loss='categorical_crossentropy', optimizer='adam', sample_weight_mode="temporal")

    X_train, y_train, train_ys_by_tag, seq_len_train = fold2training_data[fold_ix]
    X_dev, y_dev, dev_ys_by_tag, seq_len_dev = fold2dev_data[fold_ix]
    X_test, y_test, test_ys_by_tag, seq_len_test = fold2test_data[fold_ix]

    # init loop vars
    f1_scores = [-1]
    num_since_best_score = 0
    patience = 3
    best_weights = None

    for i in range(30):
        print("{ts}: Epoch={epoch}".format(ts=get_ts(), epoch=i))
        epochs = 1  # epochs per training instance
        results = model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=epochs, validation_split=0.0, verbose=0)
        micro_metrics, _ = score_predictions(model, X_dev, dev_ys_by_tag, seq_len_dev)
        print(micro_metrics)

        f1_score = micro_metrics.f1_score
        best_f1_score = max(f1_scores)
        if f1_score <= best_f1_score:
            num_since_best_score += 1
        else:  # score improved
            num_since_best_score = 0
            best_weights = model.get_weights()

        f1_scores.append(f1_score)
        if num_since_best_score >= patience:
            break

    # load best weights
    model.set_weights(best_weights)
    train_predictions_by_tag = get_predictions(model, X_train, train_ys_by_tag, seq_len_train)
    test_predictions_by_tag = get_predictions(model, X_test, test_ys_by_tag, seq_len_test)
    return train_predictions_by_tag, test_predictions_by_tag, train_ys_by_tag, test_ys_by_tag