def main(nb_units, depth, nb_epoch, filter_size, project_factor, nb_dense):
    h5_fname = "/home/leon/data/generated_tags_variable_depth_map_normed.hdf5"
    batch_size = 32
    output_dir = "models/holy/mask_generator_var_with_depth_n{}_d{}_e{}/" \
        .format(nb_units, depth, nb_epoch)
    use_l1_regularizer = False
    os.makedirs(output_dir)

    gen = generator(h5_fname, batch_size)
    nb_input = next(gen)[0][0].shape[1]

    x = Input(shape=(nb_input,))
    mask, depth_map = mask_generator_all_conv(x, nb_units=nb_units, depth=depth,
                                           filter_size=filter_size)
    if use_l1_regularizer:
        layers = collect_layers(x, [mask, depth_map])
        for l in layers:
            if hasattr(l, 'W_regularizer'):
                l.W_regularizer = l1(0.001)
            if hasattr(l, 'b_regularizer'):
                l.b_regularizer = l1(0.001)
    g = Model(x, [mask, depth_map])
    optimizer = Adam()


    mask_more_weight = 5
    total_pixels = mask_more_weight*64**2 + 16**2
    weight_depth_map = 16**2 / total_pixels
    weight_mask = mask_more_weight*64**2 / total_pixels
    g.compile(optimizer, [weight_mse(weight_mask),
                          weight_mse(weight_depth_map)])

    scheduler = AutomaticLearningRateScheduler(
        optimizer, 'loss', epoch_patience=4, min_improvment=0.0002)
    history = HistoryPerBatch()
    save = SaveModels({'{epoch:^03}_mask_generator.hdf5': g}, output_dir=output_dir)
    g.fit_generator(gen, samples_per_epoch=200*batch_size,
                    nb_epoch=nb_epoch, verbose=1, callbacks=[scheduler, save])

    ins = next(generator(h5_fname, batch_size))
    predict_masks, predict_depth_map = g.predict(ins[0])
    masks, depth_map = ins[1]

    def clip(x):
        return np.clip(x, 0, 1)

    zip_visualise_tiles(clip(masks),
                        clip(predict_masks), show=False)
    plt.savefig(output_dir + "predict_tags.png")

    zip_visualise_tiles(clip(depth_map),
                        clip(predict_depth_map), show=False)
    plt.savefig(output_dir + "predict_depth_map.png")

    g.save_weights(output_dir + "mask_generator.hdf5", overwrite=True)
    with open(output_dir + 'mask_generator.json', 'w+') as f:
        f.write(g.to_json())

    with open(output_dir + 'history.json', 'w+') as f:
        json.dump(history.history, f)
Example #2
0
    def __init__(self):
        X1 = T.tensor4()
        X2 = T.tensor4()
        X = [X1, X2]
        Y = [T.ivector()]
        
        model = Model()
        #conv1
        model.add(Conv(filter_shape = (32, 3, 3, 3), regularizers = {'W': l1(0.0001)},  w_shared = True, n_inputs = 2))
        model.add(Conv(filter_shape = (32, 32, 2, 2), regularizers = {'W': l1(0.0001)}, w_shared = True, n_inputs = 2))
        model.add(Pooling(pool_size = (2,2)))
        model.add(Activation(mode = 'tanh'))
        #conv2
        model.add(Conv(filter_shape = (32, 32, 3, 3), regularizers = {'W': l1(0.0001)}, w_shared = True, n_inputs = 2))
        model.add(Pooling(pool_size = (2,2)))
        model.add(Activation(mode = 'tanh'))
        #abs_diff
        model.add(Abs_diff())
        #conv3
        model.add(Conv(filter_shape = (32, 32, 3, 3), regularizers = {'W': l1(0.0001)}, w_shared = True))
        model.add(Pooling(pool_size = (2,2)))
        model.add(Activation(mode = 'tanh'))
        model.add(Flatten())

        self.f = theano.function(X, model.f(X, is_train = True))

        model.add(Fully((2880, 512)))
        model.add(Activation(mode = 'tanh'))
        model.add(Dropout(0.5))
        model.add(Fully((512, 2)))
        model.add(Activation(mode = 'softmax'))
        model.build(CostFunc.nll, RMSprop(), X, Y)
        self.model = model
Example #3
0
      def __init__(self):
          left = Sequential()
          left.add(Dense(784, 50))
          left.add(Activation('relu'))
          
          model = Sequential()
          model.add(Merge([left, left], mode='sum'))
          
          model.add(Dense(50, 10))
          model.add(Activation('softmax'))
          pdb.set_trace()

          model = Sequential()

          left = Sequential()
          num_kernel = 32
          l1_penalty = 0.0001
          b_mode = 'full'
          left.add(Convolution2D(num_kernel, 3, 2, 2,  W_regularizer=l1(l1_penalty), border_mode=b_mode))
          left.add(Convolution2D(num_kernel, num_kernel, 2, 2, W_regularizer=l1(l1_penalty), border_mode=b_mode))
          left.add(LeakyReLU(0.1))
          #left.add(Activation('relu'))
          left.add(MaxPooling2D(poolsize=(2, 2)))
          #left.add(Convolution2D(num_kernel, 3, 2, 2,  W_regularizer=l1(l1_penalty), border_mode=b_mode))
          #left.add(Convolution2D(num_kernel, num_kernel, 2, 2, W_regularizer=l1(l1_penalty), border_mode=b_mode))
          #left.add(LeakyReLU(0.1))
          ##left.add(Activation('relu'))
          #left.add(MaxPooling2D(poolsize=(2, 2)))

          model.add(Merge([left, left], mode='sum'))
          pdb.set_trace()
          self.f = theano.function(model.get_input(), model.get_output())
Example #4
0
def skip_layer(x, nb_layers=16):
    x1 = Dense(nb_layers, kernel_regularizer=l1(0.01))(x)
    x1 = Activation('relu')(x1)
    x2 = Dense(nb_layers, kernel_regularizer=l1(0.01))(x1)
    x2 = Activation('relu')(x2)
    x3 = Add()([x1, x2])
    return x3
Example #5
0
 def __init__(self, seq_length, use_RNN=False, num_tasks=1,
              num_filters=(15, 15, 15), conv_width=(15, 15, 15),
              pool_width=35, GRU_size=35, TDD_size=15,
              L1=0, dropout=0.0, num_epochs=100, verbose=1):
     self.saved_params = locals()
     self.seq_length = seq_length
     self.input_shape = (1, 4, self.seq_length)
     self.num_tasks = num_tasks
     self.num_epochs = num_epochs
     self.verbose = verbose
     self.model = Sequential()
     assert len(num_filters) == len(conv_width)
     for i, (nb_filter, nb_col) in enumerate(zip(num_filters, conv_width)):
         conv_height = 4 if i == 0 else 1
         self.model.add(Convolution2D(
             nb_filter=nb_filter, nb_row=conv_height,
             nb_col=nb_col, activation='linear',
             init='he_normal', input_shape=self.input_shape,
             W_regularizer=l1(L1), b_regularizer=l1(L1)))
         self.model.add(Activation('relu'))
         self.model.add(Dropout(dropout))
     self.model.add(MaxPooling2D(pool_size=(1, pool_width)))
     if use_RNN:
         num_max_pool_outputs = self.model.layers[-1].output_shape[-1]
         self.model.add(Reshape((num_filters[-1], num_max_pool_outputs)))
         self.model.add(Permute((2, 1)))
         self.model.add(GRU(GRU_size, return_sequences=True))
         self.model.add(TimeDistributedDense(TDD_size, activation='relu'))
     self.model.add(Flatten())
     self.model.add(Dense(output_dim=self.num_tasks))
     self.model.add(Activation('sigmoid'))
     self.model.compile(optimizer='adam', loss='binary_crossentropy')
     self.train_losses = None
     self.valid_losses = None
Example #6
0
def test_dense():
    layer_test(layers.Dense,
               kwargs={'units': 3},
               input_shape=(3, 2))

    layer_test(layers.Dense,
               kwargs={'units': 3},
               input_shape=(3, 4, 2))

    layer_test(layers.Dense,
               kwargs={'units': 3},
               input_shape=(None, None, 2))

    layer_test(layers.Dense,
               kwargs={'units': 3},
               input_shape=(3, 4, 5, 2))

    layer_test(layers.Dense,
               kwargs={'units': 3,
                       'kernel_regularizer': regularizers.l2(0.01),
                       'bias_regularizer': regularizers.l1(0.01),
                       'activity_regularizer': regularizers.L1L2(l1=0.01, l2=0.01),
                       'kernel_constraint': constraints.MaxNorm(1),
                       'bias_constraint': constraints.max_norm(1)},
               input_shape=(3, 2))

    layer = layers.Dense(3,
                         kernel_regularizer=regularizers.l1(0.01),
                         bias_regularizer='l1')
    layer.build((None, 4))
    assert len(layer.losses) == 2
Example #7
0
	def __init__(self, X_shape_1, n_h_nodes, alpha):
		in_model = Input(shape=(X_shape_1,))
		hidden_l = Dense(n_h_nodes, activation='relu', W_regularizer=l1(alpha))(in_model)
		out_model = Dense(1, activation='linear', W_regularizer=l1(alpha))(hidden_l)
		
		super().__init__(input = in_model, output=out_model)
		
		self.compile(optimizer='adadelta', loss='mse', metrics=['accuracy'])
Example #8
0
def test_regularization_shared_layer():
    dense_layer = Dense(num_classes,
                        kernel_regularizer=regularizers.l1(),
                        activity_regularizer=regularizers.l1())

    model = create_multi_input_model_from(dense_layer, dense_layer)
    model.compile(loss='categorical_crossentropy', optimizer='sgd')
    assert len(model.losses) == 6
Example #9
0
def buildModel(weight_decay1, weight_decay2, nodes_per_layer, activation_functions=['sigmoid','sigmoid'], 
               solver='Adadelta', regulization_method='l2'):
    
    # Sequential to get a linear layer structure as in feedforward networks
    model = Sequential()
    
    if(regulization_method is 'l1'):
            # input to hidden layer
            model.add(Dense(input_dim=X_train.shape[1], # input dimension
                      output_dim=nodes_per_layer, # output dimension
                      init='uniform', # initialization of the weights
                      activation=activation_functions[0], # activation function,
                      W_regularizer=l1(weight_decay1),  # weight decay on the first level conncetion weight
                      b_regularizer=l1(weight_decay1)   # weight decay for bias unit              
                      ))
            # hidden to output layer
            model.add(Dense(input_dim=nodes_per_layer, # input dimension must match the previous output_dim
                            output_dim=2, # output dimension must match the target dimension
                            init='uniform', # initialization of the weights
                            activation=activation_functions[1], # activation function,
                        W_regularizer=l1(weight_decay2),  # weight decay 
                        b_regularizer=l1(weight_decay2)   # weight decay for bias unit              
                        ))
    else:
        # input to hidden layer
        model.add(Dense(input_dim=X_train.shape[1], # input dimension
                        output_dim=nodes_per_layer, # output dimension
                        init='uniform', # initialization of the weights
                        activation=activation_functions[0], # activation function,
                        W_regularizer=l2(weight_decay1),  # weight decay on the first level conncetion weight
                        b_regularizer=l2(weight_decay1)   # weight decay for bias unit              
                        ))
        
        # hidden to output layer
        model.add(Dense(input_dim=nodes_per_layer, # input dimension must match the previous output_dim
                        output_dim=2, # output dimension must match the target dimension
                        init='uniform', # initialization of the weights
                        activation=activation_functions[1], # activation function,
                        W_regularizer=l2(weight_decay2),  # weight decay 
                        b_regularizer=l2(weight_decay2)   # weight decay for bias unit              
                        ))
    
    # setup solver 
    if(solver is 'Adam'):
        bp = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
    if(solver is 'Adadelta'):
        bp = Adadelta(lr=1.0, rho=0.95, epsilon=1e-08)
    if (solver is 'Adagrad'):
        bp = Adagrad(lr=0.01, epsilon=1e-08)
    else:
        bp = SGD(lr=0.01)  
        # learning rate recommended to tune, but was found to have no influenceas long it is between 0.01 and 0.1
    
    # compile model    
    model.compile(loss='binary_crossentropy', optimizer=bp, class_mode="binary")
    return model
Example #10
0
def test_regularizer(layer_class):
    layer = layer_class(units, return_sequences=False, weights=None,
                        batch_input_shape=(num_samples, timesteps, embedding_dim),
                        kernel_regularizer=regularizers.l1(0.01),
                        recurrent_regularizer=regularizers.l1(0.01),
                        activity_regularizer='l1',
                        bias_regularizer='l2')
    layer.build((None, None, 2))
    assert len(layer.losses) == 3
    layer(K.variable(np.ones((2, 3, 2))))
    assert len(layer.losses) == 4
Example #11
0
def test_regularization_shared_model():
    dense_layer = Dense(num_classes,
                        kernel_regularizer=regularizers.l1(),
                        activity_regularizer=regularizers.l1())

    input_tensor = Input(shape=(data_dim,))
    dummy_model = Model(input_tensor, dense_layer(input_tensor))

    model = create_multi_input_model_from(dummy_model, dummy_model)
    model.compile(loss='categorical_crossentropy', optimizer='sgd')
    assert len(model.losses) == 6
Example #12
0
def test_regularization_shared_layer_in_different_models():
    shared_dense = Dense(num_classes,
                         kernel_regularizer=regularizers.l1(),
                         activity_regularizer=regularizers.l1())
    models = []
    for _ in range(2):
        input_tensor = Input(shape=(data_dim,))
        unshared_dense = Dense(num_classes, kernel_regularizer=regularizers.l1())
        out = unshared_dense(shared_dense(input_tensor))
        models.append(Model(input_tensor, out))

    model = create_multi_input_model_from(*models)
    model.compile(loss='categorical_crossentropy', optimizer='sgd')
    assert len(model.losses) == 8
def main(nb_units, nb_epoch):
    h5_fname = "/home/leon/data/generated_tags_variable_depth_map.hdf5"
    batch_size = 32
    output_dir = "models/holy/mask_generator_var_with_depth_n{}_d3/".format(nb_units)
    use_l1_regularizer = False
    assert not os.path.exists(output_dir)
    gen = generator(h5_fname, batch_size)
    nb_input = next(gen)[0][0].shape[1]

    x = Input(shape=(nb_input,))
    out = mask_generator(x, nb_units=nb_units, dense_factor=7,
                         nb_dense_layers=3, nb_output_channels=2)
    if use_l1_regularizer:
        layers = collect_layers(x, out)
        for l in layers:
            if hasattr(l, 'W_regularizer'):
                l.W_regularizer = l1(0.001)
            if hasattr(l, 'b_regularizer'):
                l.b_regularizer = l1(0.001)
    g = Model(x, out)
    optimizer = Adam()
    g.compile(optimizer, mse)

    scheduler = AutomaticLearningRateScheduler(
        optimizer, 'loss', epoch_patience=4, min_improvment=0.0002)
    history = HistoryPerBatch()
    g.fit_generator(gen, samples_per_epoch=200*batch_size,
                    nb_epoch=nb_epoch, verbose=1, callbacks=[scheduler])

    ins = next(gen)
    out = g.predict(ins[0])

    def clip(x):
        return np.clip(x, 0, 1)

    os.makedirs(output_dir, exist_ok=True)

    zip_visualise_tiles(clip(ins[1][:, :1]), clip(out[:, :1]), show=False)
    plt.savefig(output_dir + "predict_tags.png")

    zip_visualise_tiles(clip(ins[1][:, 1:]), clip(out[:, 1:]), show=False)
    plt.savefig(output_dir + "predict_depth_map.png")

    g.save_weights(output_dir + "mask_generator.hdf5", overwrite=True)
    with open(output_dir + 'mask_generator.json', 'w+') as f:
        f.write(g.to_json())

    with open(output_dir + 'history.json', 'w+') as f:
        json.dump(history.history, f)
    def __init__(self, timesteps, attention_size, bias=True,
                 W_regularizer=regularizers.l1(0.01), W_constraint=None,
                 U_regularizer=regularizers.l1(0.01), U_constraint=None,
                 **kwargs):
        self.supports_masking = True
        self.init = initializers.get('glorot_uniform')
        self.bias = bias
        self.timesteps = timesteps
        self.attention_size = attention_size
        self.W_regularizer = regularizers.get(W_regularizer)
        self.W_constraint = constraints.get(W_constraint)
        self.U_regularizer = regularizers.get(U_regularizer)
        self.U_constraint = constraints.get(U_constraint)

        super(Attention, self).__init__(**kwargs)
Example #15
0
def test_activity_regularization():
    x_train, y_train = get_data()
    for reg in [regularizers.l1(), regularizers.l2()]:
        model = create_model(activity_regularizer=reg)
        model.compile(loss='categorical_crossentropy', optimizer='sgd')
        assert len(model.losses) == 1
        model.train_on_batch(x_train, y_train)
Example #16
0
def test_dense():
    from keras import regularizers
    from keras import constraints

    layer_test(core.Dense,
               kwargs={'output_dim': 3},
               input_shape=(3, 2))

    layer_test(core.Dense,
               kwargs={'output_dim': 3},
               input_shape=(3, 4, 2))

    layer_test(core.Dense,
               kwargs={'output_dim': 3},
               input_shape=(None, None, 2))

    layer_test(core.Dense,
               kwargs={'output_dim': 3},
               input_shape=(3, 4, 5, 2))

    layer_test(core.Dense,
               kwargs={'output_dim': 3,
                       'W_regularizer': regularizers.l2(0.01),
                       'b_regularizer': regularizers.l1(0.01),
                       'activity_regularizer': regularizers.activity_l2(0.01),
                       'W_constraint': constraints.MaxNorm(1),
                       'b_constraint': constraints.MaxNorm(1)},
               input_shape=(3, 2))
Example #17
0
def create_model(input_shape, n_classes):
    model = Sequential()
    model.add(Conv2D(32, (3, 3), padding='same', activation='relu', input_shape=input_shape))
    model.add(Conv2D(32, (3, 3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))
 
    model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
    model.add(Conv2D(64, (3, 3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    # model.add(Conv2D(32, (3, 3), padding='same', activation='relu'))
    # model.add(Conv2D(32, (3, 3), activation='relu'))
    # model.add(MaxPooling2D(pool_size=(2, 2)))
    # model.add(Dropout(0.25))
 
    model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
    model.add(Conv2D(64, (3, 3), activation='relu', kernel_regularizer=regularizers.l1(0.001)))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))
 
    model.add(Flatten())
    model.add(Dense(256, activation='relu', activity_regularizer=regularizers.l2(0.001)))
    model.add(Dropout(0.25))
    model.add(Dense(n_classes, activation='softmax'))
     
    return model
Example #18
0
def regularizer():
  if args.l2_reg:
    return l2(args.l2_reg)
  elif args.l1_reg:
    return l1(args.l1_reg)
  else:
    return None
Example #19
0
 def train(self, X_train, Y_train, X_test, iter_):
     '''
     output: predicted class: 0, 1, 2
     '''
     inputs = Input(shape=(12,))
     x1 = Dense(96, activation='relu', W_regularizer=l1(0.01))(inputs)
     x2 = Dense(96, activation='relu', W_regularizer=l1(0.01))(x1)
     #drop = Dropout(0.2)(x)
     prediction = Dense(3, activation='relu', W_regularizer=l1(0.01))(x2)
     model = Model(input=inputs, output=prediction)
     model.compile(optimizer='adagrad',
                   loss='poisson')
     model.fit(X_train, Y_train, nb_epoch=iter_, batch_size=100)
     pred = model.predict(X_test)
     return (np.argmax(pred, axis=1),
             np.max(pred, axis=1))
Example #20
0
def test_W_reg():
    for reg in [regularizers.identity(), regularizers.l1(), regularizers.l2(), regularizers.l1l2()]:
        model = create_model(weight_reg=reg)
        model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
        model.fit(X_train, Y_train, batch_size=batch_size,
                  nb_epoch=nb_epoch, verbose=0)
        model.evaluate(X_test[test_ids, :], Y_test[test_ids, :], verbose=0)
Example #21
0
def test_activity_regularization():
    (x_train, y_train), (x_test, y_test) = get_data()
    for reg in [regularizers.l1(), regularizers.l2()]:
        model = create_model(activity_regularizer=reg)
        model.compile(loss='categorical_crossentropy', optimizer='sgd')
        assert len(model.losses) == 1
        model.fit(x_train, y_train, batch_size=batch_size,
                  epochs=epochs, verbose=0)
Example #22
0
 def __init__(self, seq_length=None, keras_model=None,
              use_RNN=False, num_tasks=1,
              num_filters=(15, 15, 15), conv_width=(15, 15, 15),
              pool_width=35, GRU_size=35, TDD_size=15,
              L1=0, dropout=0.0, num_epochs=100, verbose=1):
     from keras.models import Sequential
     from keras.layers.core import (
         Activation, Dense, Dropout, Flatten,
         Permute, Reshape, TimeDistributedDense
     )
     from keras.layers.convolutional import Convolution2D, MaxPooling2D
     from keras.layers.recurrent import GRU
     from keras.regularizers import l1
     self.num_tasks = num_tasks
     self.num_epochs = num_epochs
     self.verbose = verbose
     self.train_metrics = []
     self.valid_metrics = []
     if keras_model is not None and seq_length is None:
         self.model = keras_model
         self.num_tasks = keras_model.layers[-1].output_shape[-1]
     elif seq_length is not None and keras_model is None:
         self.model = Sequential()
         assert len(num_filters) == len(conv_width)
         for i, (nb_filter, nb_col) in enumerate(zip(num_filters, conv_width)):
             conv_height = 4 if i == 0 else 1
             self.model.add(Convolution2D(
                 nb_filter=nb_filter, nb_row=conv_height,
                 nb_col=nb_col, activation='linear',
                 init='he_normal', input_shape=(1, 4, seq_length),
                 W_regularizer=l1(L1), b_regularizer=l1(L1)))
             self.model.add(Activation('relu'))
             self.model.add(Dropout(dropout))
         self.model.add(MaxPooling2D(pool_size=(1, pool_width)))
         if use_RNN:
             num_max_pool_outputs = self.model.layers[-1].output_shape[-1]
             self.model.add(Reshape((num_filters[-1], num_max_pool_outputs)))
             self.model.add(Permute((2, 1)))
             self.model.add(GRU(GRU_size, return_sequences=True))
             self.model.add(TimeDistributedDense(TDD_size, activation='relu'))
         self.model.add(Flatten())
         self.model.add(Dense(output_dim=self.num_tasks))
         self.model.add(Activation('sigmoid'))
         self.model.compile(optimizer='adam', loss='binary_crossentropy')
     else:
         raise ValueError("Exactly one of seq_length or keras_model must be specified!")
Example #23
0
def get_regularizer(lambda_l1=None, lambda_l2=None):
    regularizer = None
    if lambda_l1 is None and lambda_l2 is not None:
        regularizer = l2(l=lambda_l2)
    elif lambda_l1 is not None and lambda_l2 is None:
        regularizer = l1(l=lambda_l1)
    elif lambda_l1 is not None and lambda_l2 is not None:
        regularizer = l1l2(l1=lambda_l1, l2=lambda_l2)
    return regularizer
Example #24
0
def test_regularizer(layer_class):
    layer = layer_class(units, return_sequences=False, weights=None,
                        input_shape=(timesteps, embedding_dim),
                        kernel_regularizer=regularizers.l1(0.01),
                        recurrent_regularizer=regularizers.l1(0.01),
                        bias_regularizer='l2')
    layer.build((None, None, embedding_dim))
    assert len(layer.losses) == 3
    assert len(layer.cell.losses) == 3

    layer = layer_class(units, return_sequences=False, weights=None,
                        input_shape=(timesteps, embedding_dim),
                        activity_regularizer='l2')
    assert layer.activity_regularizer
    x = K.variable(np.ones((num_samples, timesteps, embedding_dim)))
    layer(x)
    assert len(layer.cell.get_losses_for(x)) == 0
    assert len(layer.get_losses_for(x)) == 1
Example #25
0
File: model.py Project: guker/mlp-1
def build_model(in_dim, out_dim=1,
                n_hidden=100, l1_norm=0.0,
                n_deep=5, drop=0.1,
                learning_rate=0.1):
    model = Sequential()
    # Input layer
    model.add(Dense(
        input_dim=in_dim,
        output_dim=n_hidden,
        init='glorot_uniform',
        activation='tanh',
        W_regularizer=l1(l1_norm)))

    # do X layers
    for layer in range(n_deep-1):
        model.add(Dropout(drop))
        model.add(Dense(
            output_dim=np.round(n_hidden/2**(layer+1)),
            init='glorot_uniform',
            activation='tanh',
            W_regularizer=l1(l1_norm)))

    # Output layer
    if out_dim == 1:
        activation = 'tanh'
    else:
        activation = 'softmax'

    model.add(Dense(out_dim,
                    init='glorot_uniform',
                    activation=activation))

    # Optimization algorithms
    opt = Adadelta(lr=learning_rate)
    if out_dim == 1:
        model.compile(loss='binary_crossentropy',
                      optimizer=opt,
                      class_mode='binary')
    else:
        model.compile(loss='categorical_crossentropy',
                      optimizer=opt,
                      class_mode='categorical')

    return model
def model_generator(latent_dim, input_shape, hidden_dim=512, reg=lambda: l1(1e-7)):
    return Sequential([
        Dense(hidden_dim, name="generator_h1", input_dim=latent_dim, W_regularizer=reg()),
        LeakyReLU(0.2),
        Dense(hidden_dim, name="generator_h2", W_regularizer=reg()),
        LeakyReLU(0.2),
        Dense(np.prod(input_shape), name="generator_x_flat", W_regularizer=reg()),
        Activation('sigmoid'),
        Reshape(input_shape, name="generator_x")],
        name="generator")
Example #27
0
def MLP_regression(layer1=None, layer2=None, layer3=None, input_shape=(64,)):
    model = Sequential()
    if layer1:    
        model.add(Dense(layer1, input_dim=input_shape[0], init='glorot_uniform',\
                  activation='sigmoid', W_regularizer=l1(0.00),\
                  ))
        # model.add(Dropout(0.5))
    if layer2:
        model.add(Dense(layer2, init='glorot_uniform',\
                  activation='sigmoid', W_regularizer=l1(0.01),\
              ))
    if layer3:
        model.add(Dense(layer3, init='glorot_uniform',\
          activation='sigmoid', W_regularizer=l1(0.01),\
         ))
    #model.add(Dropout(0.5))
    model.add(Dense(1, init='glorot_uniform', activation='sigmoid'))
    sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
    model.compile(class_mode='binary', loss='binary_crossentrob', optimizer=RMSprop())
    return model
Example #28
0
def make_3l_nn():
    model = Sequential()
    model.add(Dense(1024, input_dim=x_train.shape[1], W_regularizer=l1(0.001), init="normal"))
    model.add(Activation("relu"))
    model.add(Dropout(0.5))
    model.add(Dense(256))
    model.add(Activation("relu"))
    model.add(Dropout(0.5))
    model.add(Dense(6))
    model.add(Activation("sigmoid"))
    return model
Example #29
0
def test_W_reg():
    (X_train, Y_train), (X_test, Y_test), test_ids = get_data()
    for reg in [regularizers.l1(),
                regularizers.l2(),
                regularizers.l1l2()]:
        model = create_model(weight_reg=reg)
        model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
        assert len(model.losses) == 1
        model.fit(X_train, Y_train, batch_size=batch_size,
                  nb_epoch=nb_epoch, verbose=0)
        model.evaluate(X_test[test_ids, :], Y_test[test_ids, :], verbose=0)
Example #30
0
def NNet(TrainX, TrainY, TestX):

	## the network remained to be XJBplay

	model = Sequential()

	numNode = 20

	model.add(Dense(numNode, input_dim=len(TrainX[0]), W_regularizer=l1(0.1)))
	model.add(BatchNormalization())
	model.add(Activation('tanh'))
	model.add(Dropout(0.5))


	model.add(Dense(numNode, W_regularizer=l2(1)))
	model.add(BatchNormalization())
	model.add(Activation('tanh'))
	model.add(Dropout(0.2))


	model.add(Dense(numNode, W_regularizer=l1(0.1)))
	model.add(BatchNormalization())
	model.add(PReLU())
	model.add(Dropout(0.5))

	model.add(Dense(numNode, W_regularizer=l2(1)))
	model.add(BatchNormalization())
	model.add(Activation('tanh'))
	model.add(Dropout(0.2))


	model.add(Dense(output_dim=2))
	model.add(Activation("softmax"))
	model.compile(loss='sparse_categorical_crossentropy',
				optimizer="adam",
				metrics=["accuracy"])

	model.fit(np.array(TrainX), TrainY, batch_size=int(len(TrainX)*.9), nb_epoch = 1000, shuffle=True, verbose=0, validation_split=0.2)
	PredY = model.predict_classes(np.array(TestX), batch_size=int(len(TrainX)*.9),verbose=0,)

	return PredY
Example #31
0
from keras.regularizers import l1
from keras.layers import Activation
from matplotlib import pyplot
# generate 2d classification dataset
X, y = make_circles(n_samples=100, noise=0.1, random_state=1)
# split into train and test
n_train = 30
trainX, testX = X[:n_train, :], X[n_train:, :]
trainy, testy = y[:n_train], y[n_train:]
# define model
model = Sequential()
model.add(
    Dense(500,
          input_dim=2,
          activation='linear',
          activity_regularizer=l1(0.0001)))
model.add(Activation('relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])
# fit model
history = model.fit(trainX,
                    trainy,
                    validation_data=(testX, testy),
                    epochs=4000,
                    verbose=0)
# evaluate the model
_, train_acc = model.evaluate(trainX, trainy, verbose=0)
_, test_acc = model.evaluate(testX, testy, verbose=0)
print('Train: %.3f, Test: %.3f' % (train_acc, test_acc))
Example #32
0
X_full = np.transpose(X, (0, 2, 1))

X_train = X_train[..., np.newaxis]
X_cv = X_cv[..., np.newaxis]
X_test = X_test[..., np.newaxis]
X_full = X_full[..., np.newaxis]

model = Sequential()

model.add(
    Conv2D(filters=32,
           kernel_size=(3, 3),
           activation='relu',
           padding='same',
           kernel_regularizer=l1(0.01),
           input_shape=(501, 40, 1))),

model.add(
    Conv2D(filters=32,
           kernel_size=(3, 3),
           padding='same',
           kernel_regularizer=l1(0.01),
           activation='relu'))

model.add(MaxPool2D(2, 2))

model.add(
    Conv2D(filters=32,
           kernel_size=(3, 3),
           padding='same',
    def __init__(self,
                 max_sequence_length,
                 num_classes,
                 vocab_size,
                 embedding_size,
                 filter_sizes,
                 num_filters,
                 l2_reg_lambda=0.0):
        #
        # Placeholders for input, output and dropout
        # ----------------------------------------------------------------------------
        self.input_x = Input(shape=(max_sequence_length, ), dtype='int32')
        self.input_y = Input(shape=(num_classes, ), dtype='float32')
        self.dropout_keep_prob = Input(shape=1, dtype='float32')

        # Keeping track of l2 regularization loss (optional)
        #l2_loss = tf.constant(0.0)

        #
        # Embedding layer
        # ----------------------------------------------------------------------------
        embedding_layer = Embedding(vocab_size,
                                    embedding_size,
                                    embeddings_initializer='uniform',
                                    input_length=max_sequence_length,
                                    trainable=False)
        self.embedded_chars = embedding_layer()(self.input)
        self.embedded_chars_expanded = K.expand_dims(self.embedded_chars, -1)

        #
        # Create a convolution + maxpool layer for each filter size
        # ----------------------------------------------------------------------------
        pooled_outputs = []
        for filter_size, num_filter in zip(filter_sizes, num_filters):
            # Convolution Layer
            conv = Conv2D(filters=num_filter,
                          kernel_size=filter_size,
                          padding='valid',
                          activation="relu",
                          strides=1)(self.embedded_chars_expanded)
            # Max-pooling over the outputs
            conv = MaxPooling2D(pool_size=2)(conv)
            pooled_outputs.append(conv)

        #
        # Combine all the pooled features
        # ----------------------------------------------------------------------------
        num_filters_total = sum(num_filters)
        self.h_pool = concatenate(pooled_outputs, 3)
        self.h_pool_flat = K.reshape(self.h_pool, [-1, num_filters_total])

        #
        # Add highway
        # ----------------------------------------------------------------------------
        self.h_highway = highway(self.h_pool_flat,
                                 num_layers=1,
                                 activation='relu',
                                 gate_bias=0)

        #
        # Add dropout
        # ----------------------------------------------------------------------------
        self.h_drop = Dropout(self.dropout_keep_prob)(self.h_highway)

        #
        # Final (unnormalized) scores and predictions
        # ----------------------------------------------------------------------------
        self.scores = BatchNormalization()(self.h_drop)
        self.preds = Dense(num_classes,
                           activation='softmax',
                           kernel_regularizer=regularizers.l2(0.01),
                           activity_regularizer=regularizers.l1(0.01))(
                               self.h_drop)

        #
        # Train the model
        # ----------------------------------------------------------------------------
        model = Model(inputs=self.input_x, outputs=self.preds)
        model.compile(loss='binary_crossentropy',
                      optimizer='nadam',
                      metrics=['acc'])
Example #34
0
from keras.layers import Input, Dense
from keras.callbacks import ModelCheckpoint, TensorBoard
from keras import regularizers
from sklearn.model_selection import train_test_split 

input_dim = X.shape[1]
# how many neurons for compressed representation?

mse_limit = 10.0

for encoding_dim in 2, : #3, 4, 5:
	name = 'AE%d' % encoding_dim
	prefix = ('SIMPLE' if simplify_space else '') + transform + '-AE%d-' % encoding_dim
	
	input_layer = Input(shape=(input_dim, ))
	encoder = Dense(int(input_dim / 2), activation="tanh",activity_regularizer=regularizers.l1(10e-5))(input_layer)
	encoder = Dense(int(encoding_dim * 1.5), activation="tanh")(encoder)
	encoder = Dense(encoding_dim, activation="tanh")(encoder)
	decoder = Dense(int(encoding_dim * 1.5), activation='tanh')(encoder)
	decoder = Dense(int(input_dim / 2), activation='tanh')(decoder)
	decoder = Dense(input_dim, activation='tanh')(decoder)
	autoencoder = Model(inputs=input_layer, outputs=decoder)
	autoencoder.summary()

	nb_epoch = 100
	batch_size = 50
	autoencoder.compile(optimizer='adam', loss='mse' )

	t0 = time()
	history = autoencoder.fit(X, X, 
		epochs=nb_epoch, batch_size=batch_size,
def HOSubtractNN(x_train, x_val, x_test, y_train, y_val, y_test, verbose):
    print('--------- Human Observed Subtract Neural Network ---------')

    # Convert target labels to categorical data format
    y_train = np_utils.to_categorical(y_train, 2)
    y_val = np_utils.to_categorical(y_val, 2)
    y_test = np_utils.to_categorical(y_test, 2)

    # Neural Network Configuration
    input_nodes = x_train.shape[1]
    drop_out = 0.3
    layer_1_nodes = 256
    layer_2_nodes = 128
    layer_3_nodes = 32
    output_nodes = 2

    # Sequential Model of 3 hidden layers
    model = Sequential()

    model.add(
        Dense(layer_1_nodes,
              input_dim=input_nodes,
              kernel_regularizer=regularizers.l2(0),
              activity_regularizer=regularizers.l1(0),
              kernel_initializer=initializers.RandomUniform(seed=123)))
    model.add(Activation('relu'))
    #model.add(LeakyReLU(alpha=0.3))
    model.add(Dropout(drop_out))

    model.add(
        Dense(layer_2_nodes,
              input_dim=layer_1_nodes,
              kernel_regularizer=regularizers.l2(0),
              activity_regularizer=regularizers.l1(0),
              kernel_initializer=initializers.RandomUniform(seed=123)))
    model.add(Activation('relu'))
    #model.add(LeakyReLU(alpha=0.3))
    model.add(Dropout(drop_out))

    model.add(
        Dense(layer_3_nodes,
              input_dim=layer_2_nodes,
              kernel_regularizer=regularizers.l2(0),
              activity_regularizer=regularizers.l1(0),
              kernel_initializer=initializers.RandomUniform(seed=123)))
    model.add(Activation('relu'))
    #model.add(LeakyReLU(alpha=0.3))
    model.add(Dropout(drop_out))

    model.add(
        Dense(output_nodes,
              input_dim=layer_3_nodes,
              kernel_regularizer=regularizers.l2(0),
              activity_regularizer=regularizers.l1(0),
              kernel_initializer=initializers.RandomUniform(seed=551)))
    model.add(Activation('sigmoid'))

    model.summary()

    model.compile(optimizer=optimizers.Adadelta(lr=1.2,
                                                rho=0.95,
                                                epsilon=None,
                                                decay=0.0),
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    num_epochs = 5000
    model_batch_size = 128
    tb_batch_size = 32
    early_patience = 100

    tensorboard_cb = TensorBoard(log_dir='logs',
                                 batch_size=tb_batch_size,
                                 write_graph=True)
    earlystopping_cb = EarlyStopping(monitor='val_loss',
                                     verbose=1,
                                     patience=early_patience,
                                     mode='min')

    # Train model using Training Data
    history = model.fit(x_train,
                        y_train,
                        validation_data=(x_val, y_val),
                        epochs=num_epochs,
                        batch_size=model_batch_size,
                        shuffle=True,
                        callbacks=[tensorboard_cb, earlystopping_cb],
                        verbose=verbose)

    # Convert labels from categorical format to original vector format
    y_test = np.argmax(y_test, axis=1)

    # Predicted labels for Test Data
    pred = model.predict(x_test)
    pred = np.argmax(pred, axis=1)

    df = pd.DataFrame(history.history)
    print(df.tail(1))

    # Accuracy and Erms for Test Data
    accuracy, erms = GetErms(pred, y_test)
    print("Test Erms: " + str(erms) + " Test Acuracy: " + str(accuracy))

    # Precision and Recall for Test Data
    precision, recall = GetPrecisionRecall(pred, y_test)
    print("Precision: " + str(precision) + " Recall: " + str(recall))

    # PLot graph of Loss and Accuracy over epochs
    df.plot()
    plt.savefig('HO_Subtract_NN.jpg', grid=True, figsize=(7, 8))
Example #36
0
lamda2 = 0
lamda3 = 0
aa = 0.01
num_layers = range(1, 6)

for l1 in range(1, 5):
    for l2 in range(1, 5):
        for l3 in range(1, 5):
            Hlayer1 = 128 * l1
            Hlayer2 = 128 * l2
            Hlayer3 = 128 * l3

            model = Sequential([
                Dense(256,
                      input_dim=202,
                      kernel_regularizer=regularizers.l1(lamda1),
                      kernel_initializer='random_uniform'),
                Activation('relu'),
                Dense(Hlayer1,
                      kernel_regularizer=regularizers.l1_l2(0),
                      kernel_initializer='random_uniform'),
                Activation('relu'),
                Dense(Hlayer2,
                      kernel_regularizer=regularizers.l1_l2(0),
                      kernel_initializer='random_uniform'),
                Activation('relu'),
                Dense(Hlayer3,
                      kernel_regularizer=regularizers.l1_l2(0),
                      kernel_initializer='random_uniform'),
                Activation('relu'),
                Dense(51,
Example #37
0
def mlp(data_train,
        target_train,
        drug_id=drug_id,
        batch_size=15,
        epochs=100,
        isSave=False,
        save_in_path=""):

    inputdim = len(data_train[0])
    model = Sequential()
    model.add(
        Dense(inputdim // 2,
              input_dim=inputdim,
              init='uniform',
              kernel_regularizer=regularizers.l2(
                  0.0001)))  # 输入层,28*28=784,最好附加初始化,用identity
    model.add(Activation('relu'))  # 激活函数是tanh(后面变成了relu因为对mnist的处理结果会好一些)

    model.add(
        Dense(inputdim // 6,
              kernel_regularizer=regularizers.l2(0.0001),
              bias_regularizer=regularizers.l1(0.0001)))
    model.add(Activation('relu'))

    model.add(
        Dense(inputdim // 8,
              kernel_regularizer=regularizers.l2(0.0001),
              bias_regularizer=regularizers.l1(0.0001)))
    model.add(Activation('relu'))

    model.add(
        Dense(inputdim // 10,
              kernel_regularizer=regularizers.l2(0.0001),
              bias_regularizer=regularizers.l1(0.0001)))
    model.add(Activation('relu'))

    model.add(Dense(2))  # 输出结果和药物对应的话只是一维的
    model.add(Activation('sigmoid'))  # 最后一层linear,因为是实际的结果

    sgd = SGD(lr=0.01)
    model.compile(loss='binary_crossentropy',
                  optimizer=sgd,
                  metrics=["accuracy"])
    if isSave:
        modelname = save_in_path + 'model_mlp_fordrug' + str(drug_id) + '.h5'
        check_test = ModelCheckpoint(modelname,
                                     monitor='loss',
                                     verbose=0,
                                     save_best_only=True,
                                     save_weights_only=False,
                                     mode='min',
                                     period=1)
        model.fit(data_train,
                  target_train,
                  batch_size=batch_size,
                  epochs=epochs,
                  shuffle=True,
                  verbose=2,
                  callbacks=[check_test])
        # predicted = model.predict(data_test)
        # predicted = predicted.reshape(-1)
        # pearson = stats.pearsonr(predicted, target_test.reshape(-1))
        return model
    else:
        model.fit(data_train,
                  target_train,
                  batch_size=batch_size,
                  epochs=epochs,
                  shuffle=True,
                  verbose=2)
        # predicted = model.predict(data_test)
        # predicted = predicted.reshape(-1)
        # pearson = stats.pearsonr(predicted, target_test.reshape(-1))
        return model
Example #38
0
encoding_dim = 32
input_vector = Input(shape=(784, ))

# regular

encoded = Dense(encoding_dim, activation='relu')(input_vector)
decoded = Dense(784, activation='sigmoid')(encoded)
autoencoder_regular = Model(input_vector, decoded)

# sparse

encoded_sparse = Dense(
    encoding_dim,
    activation='relu',
    activity_regularizer=regularizers.l1(10e-3))(input_vector)
decoded_sparse = Dense(784, activation='sigmoid')(encoded_sparse)
autoencoder_sparse = Model(input_vector, decoded_sparse)

# deep

encoded_deep = Dense(128, activation='relu')(input_vector)
encoded_deep = Dense(64, activation='relu')(encoded_deep)
encoded_deep = Dense(32, activation='relu')(encoded_deep)

decoded_deep = Dense(64, activation='relu')(encoded_deep)
decoded_deep = Dense(128, activation='relu')(decoded_deep)
decoded_deep = Dense(784, activation='sigmoid')(decoded_deep)
autoencoder_deep = Model(input_vector, decoded_deep)

# convolutional
Example #39
0
#        a='drop 0.8'
#        c='y'
#    if j==9:
#        drop=0.9
#        a='drop 0.9'
#        c='#708090'
#    if j==10:
#        drop=1
#        a='drop 1.0'
#        c='#c0c0c0'

for j in range(9):  # test l1
    drop = 0.3

    if j == 0:
        ak = regularizers.l1(1)
        k = regularizers.l1(1)
        a = 'kernel l1 1.0'
        c = 'b'
    if j == 1:
        ak = regularizers.l1(0.1)
        k = regularizers.l1(0.1)
        a = 'kernel l1 0.1'
        c = 'c'
    if j == 2:
        ak = regularizers.l1(0.01)
        k = regularizers.l1(0.01)
        a = 'kernel l1 0.01'
        c = 'g'
    if j == 3:
        ak = regularizers.l1(0.03)
Example #40
0
    def __init__(self, 
                max_lenght,
                num_classes,
                vocab_size = {},
                rnn='lstm',
                rnn_units=100,
                merge_type = 'concat',
                dropout_before_rnn=0.5,
                dropout_after_rnn=0.5,
                embedding_size = {}):
            
        start_time = time.time()

        print('\n\n##########           CREATE A DEEPEST MODEL       #########\n')
        print('... max_lenght: {}\n... vocab_size: {}\n... classes: {}'.format(max_lenght, vocab_size, num_classes))
        self.vocab_size = vocab_size
        self.col_name = list(vocab_size.keys())
        self.max_lenght = max_lenght
        input_model = []
        embedding_layers = []
        hidden_input = []
        hidden_dropout  = []

        
        if not isinstance(embedding_size, dict):
            embbeding_default = embedding_size
            print('... embedding size dictionary is empty')
            print('... setting embedding size default: {}'.format(embbeding_default))
            embedding_size = dict(zip(self.col_name, np.full(len(self.col_name), embbeding_default)))
        
        assert set(vocab_size) == set(embedding_size), "ERRO: embedding size is different from vocab_size"

        assert len(embedding_size) > 0, "embedding size was not defined"

        print('\n\n#######         DATA TO MODEL CREATION       #######\n')
        print('... features to input: {}'.format(self.col_name))
        print('... embedding_size: {}'.format(embedding_size))
        print('... max_lenght: {}'.format(max_lenght))

        print('\n\n###########      Building Input and Embedding Layers      ###########') 
        for c in tqdm(self.col_name):
            print('... creating layer to column : {}'.format(c))
            print('... vocab_size to column {}: {}'.format(c, self.vocab_size[c]))
            i_model= Input(shape=(self.max_lenght,), 
                            name='Input_{}'.format(c)) 
            e_output_ = Embedding(input_dim = self.vocab_size[c], 
                                output_dim = embedding_size[c], 
                                name='Embedding_{}'.format(c), 
                                input_length=self.max_lenght)(i_model)

            input_model.append(i_model)  
            embedding_layers.append(e_output_)             
            
    
        print('... defining merge type on embedding as {}'.format(merge_type))
        # MERGE Layer
        if len(embedding_layers) == 1:
            hidden_input = embedding_layers[0]
        elif merge_type == 'add':
            hidden_input = Add()(embedding_layers)
        elif merge_type == 'avg':
            hidden_input = Average()(embedding_layers)
        else:
            hidden_input = Concatenate(axis=2)(embedding_layers) #Lucas fez assim
            #hidden_layers = Concatenate()(embedding_layers)#eu fazia assim 92,25 sparse_val_acc

         #DROPOUT before RNN
        #if self.dropout_before_rnn > 0:
        print('... creating a dropout Layer before RNN using {}'.format(dropout_before_rnn))
        hidden_dropout = Dropout(dropout_before_rnn)(hidden_input)
    
        print('\n\n###### Creating a recurrent neural network ####\n')
        # Recurrent Neural Network Layer
        # https://www.quora.com/What-is-the-meaning-of-%E2%80%9CThe-number-of-units-in-the-LSTM-cell
        if rnn == 'bilstm':
            print('... creating a BiLSTM')
            rnn_cell = Bidirectional(LSTM(units=rnn_units, recurrent_regularizer=l1(0.02)))(hidden_dropout)
            #input_shape=(embedding_size[cy_true]*len(embedding_layers),))(hidden_dropout)    
        else:
            print('... creating a LSTM')
            rnn_cell = LSTM(units=rnn_units, recurrent_regularizer=l1(0.02))(hidden_dropout)
            #input_shape=(embedding_size[c]*len(embedding_layers),))(hidden_dropout)

        print('... creating a dropout Layer after RNN using {}'.format(dropout_after_rnn))
        rnn_dropout = Dropout(dropout_after_rnn)(rnn_cell)
        
        #https://keras.io/initializers/#randomnormal
        output_model = Dense(num_classes, 
                            kernel_initializer=he_uniform(),
                            activation='softmax')(rnn_dropout)

        #– Encoding the labels as integers and using the sparse_categorical_crossentropy asloss function
        self.model = Model(inputs=input_model, outputs=output_model)
        print('... a deepest model was built')

        print('\n--------------------------------------\n')
        end_time = time.time()
        print('total Time: {}'.format(end_time - start_time))
set_session(tf.Session(config=config))
from keras.callbacks import ReduceLROnPlateau,ModelCheckpoint
from keras.utils import plot_model

# seed 1234 is used for reproducibility
np.random.seed(seed=1234)


loss = 'categorical_crossentropy'
lr = 0.01
momentum = 0.9
out_dir_name = 'attention'
activation = "relu"
optimizer = SGD(lr=lr, momentum=momentum, decay=0.0, nesterov=True)
dropout = 0.1
reg = l1(1.e-4)

batch_size = 128
epochs = 600
verbose = 1
shuffle = 1
train_x_mean = 0.5909#14884877    #the mean value of cross_subject_trainset
max_len = 300

feat_dim = 150


n_classes = 60


data_root = "/data2/attention_1/"
#将数据转化为数组
x_train, y_train = np.array(x_train), np.array(y_train)
#因为LSTM要求输入的数据格式为三维的,[training_number, time_steps, 1],因此对数据进行相应转化
x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], features_num))

#构建网络,使用的是序贯模型
model = Sequential()
#return_sequences=True返回的是全部输出,LSTM做第一层时,需要指定输入shape
model.add(LSTM(units=128,activation='softsign',dropout=0.001,recurrent_dropout=0.001, return_sequences=True,input_shape=[need_num, features_num]))   #activation='softsign',dropout=0.001,recurrent_dropout=0.001, stateful=False,return_sequences=True)(input)  #stateful=True,可以使得帧组之间产生关联。 记得要在fit时候,shuffle=True。
# model.add(BatchNormalization())

model.add(LSTM(units=64))
# model.add(BatchNormalization())
#kernel_regularizer=regularizers.l2(0.01),activity_regularizer=regularizers.l1(0.01)
model.add(Dense(64,kernel_regularizer=regularizers.l2(0.01),activity_regularizer=regularizers.l1(0.01)))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(32))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(24))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(12))
s_num))
#进行配置
adam=Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=True)
#AdaGrad
model.compile(optimizer=adam,loss='mean_squared_logarithmic_error')  #mean_absolute_error/mean_squared_error/mean_squared_logarithmic_error/
Example #43
0
model.add(Dropout(0.2))
model.add(Convolution2D(256, 3, 3, activation='relu'))
#model.add(BatchNormalization())

model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(Flatten())
#fully connected layer
model.add(Dense(600, activation='relu'))
#model.add(BatchNormalization())
model.add(Dropout(0.2))
model.add(Dense(200, activation='relu'))
#model.add(BatchNormalization())
model.add(
    Dense(7,
          W_regularizer=l1(0.02),
          activity_regularizer=regularizers.l1(0.03),
          activation='softmax'))

# optimizer:
model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

train_datagen = ImageDataGenerator(rescale=1. / 255)
test_datagen = ImageDataGenerator(rescale=1. / 255)

training_set = train_datagen.flow_from_directory('dataset/training',
                                                 target_size=(48, 48),
                                                 batch_size=128,
                                                 class_mode='categorical')
Example #44
0
    def kr_train_DNN_Seq_03(self, x_dim, features_train, features_test,
                            labels_train, labels_test, batch_size):
        # create model
        model = Sequential()
        model.add(
            Dense(5,
                  input_dim=x_dim,
                  init='uniform',
                  kernel_regularizer=regularizers.l2(0.01),
                  activity_regularizer=regularizers.l2(0.005),
                  activation='relu'))
        model.add(
            Dense(10,
                  init='uniform',
                  kernel_regularizer=regularizers.l2(0.015),
                  activity_regularizer=regularizers.l1(0.02),
                  activation='relu'))
        model.add(Dense(10, init='uniform', activation='relu'))
        model.add(Dense(1, init='uniform', activation='relu'))
        # Compile model
        model.compile(loss='mean_squared_logarithmic_error',
                      optimizer='adam',
                      metrics=['accuracy'])
        # checkpoint
        filepath = "weights-improvement-{epoch:02d}-{val_accuracy:.2f}.hdf5"
        checkpoint = ModelCheckpoint(filepath,
                                     monitor='val_accuracy',
                                     verbose=1,
                                     save_best_only=True,
                                     mode='max')
        callbacks_list = [checkpoint]
        # Model Summary
        model.summary()
        # Fit the model
        history = model.fit(features_train,
                            labels_train,
                            validation_split=0.2,
                            epochs=100,
                            batch_size=batch_size,
                            callbacks=callbacks_list,
                            verbose=2)
        model_yaml = model.to_yaml()
        with open("seq03_model.yaml", "w") as yaml_file:
            yaml_file.write(model_yaml)
        # serialize weights to HDF5
        model_json = model.to_json()
        with open("seq03_model.json", "w") as json_file:
            json_file.write(model_json)
        model.save_weights("seq03_model.h5")
        score = model.evaluate(features_test, labels_test, verbose=1)
        # round predictions
        accuracy = score[1]
        accuracy = accuracy * 100
        #output_file("line_plot.html")
        #line_graph = figure()
        #line_graph.line([history.history['acc']], [history.history['val_acc']])
        #show(line_graph)

        # Plot training & validation accuracy values
        #plt.plot(history.history['acc'])
        #plt.plot(history.history['val_acc'])
        #plt.title('Model accuracy')
        #plt.ylabel('Accuracy')
        #plt.xlabel('Epoch')
        #plt.legend(['Train', 'Test'], loc='upper left')
        #plt.show()

        # Plot training & validation loss values
        #plt.plot(history.history['loss'])
        #plt.plot(history.history['val_loss'])
        #plt.title('Model loss')
        #plt.ylabel('Loss')
        #plt.xlabel('Epoch')
        #plt.legend(['Train', 'Test'], loc='upper left')
        plt.show()
        return accuracy
Example #45
0
    def train(self,
              data: pd.DataFrame,
              percentage_train: float = .7,
              epochs: int = 100,
              n_layers: int = 2,
              neuron_per_layer: list = [],
              batch: int = 30,
              opt: str = "adam",
              act: str = "tanh"):

        self.encoder_layers.clear()
        self.decoder_layers.clear()
        self.data = data
        limit = round(len(self.data) * percentage_train)
        train = self.data[:limit]
        counter = 0
        if (n_layers == 1):
            multi_layer = False
        else:
            multi_layer = True

        data_scaled = self.scaler.fit_transform(data)
        if (neuron_per_layer == []):
            n = data_scaled.shape[1] - self.reduced_dimension
            aux = (n / n_layers)
            for i in range(1, n_layers):
                neuron_per_layer.append(data_scaled.shape[1] - round(aux * i))

        self.input = Input(shape=(data_scaled.shape[1], ))
        if (multi_layer):
            self.encoder_layers.append(
                Dense(neuron_per_layer[0],
                      activation=act,
                      activity_regularizer=regularizers.l1(10e-5))(self.input))
            for i in range(1, n_layers - 1):
                self.encoder_layers.append(
                    Dense(neuron_per_layer[i],
                          activation=act)(self.encoder_layers[i - 1]))
            self.encoder_layers.append(
                Dense(self.reduced_dimension,
                      activation=act)(self.encoder_layers[-1]))
            self.decoder_layers.append(
                Dense(neuron_per_layer[-1],
                      activation=act,
                      activity_regularizer=regularizers.l1(10e-5))(
                          self.encoder_layers[-1]))
            for i in range(n_layers - 3, -1, -1):
                self.decoder_layers.append(
                    Dense(neuron_per_layer[i],
                          activation=act)(self.decoder_layers[counter]))
                counter += 1
            self.decoder_layers.append(
                Dense(data_scaled.shape[1],
                      activation=act)(self.decoder_layers[counter]))
        else:
            self.encoder_layers.append(
                Dense(self.reduced_dimension,
                      activation=act,
                      activity_regularizer=regularizers.l1(10e-5))(self.input))
            self.decoder_layers.append(
                Dense(data_scaled.shape[1],
                      activation=act,
                      activity_regularizer=regularizers.l1(10e-5))(
                          self.encoder_layers[0]))
        autoencoder = Model(self.input, self.decoder_layers[-1])
        autoencoder.compile(optimizer=opt, loss='mse')
        X_train = data_scaled
        autoencoder.fit(x=X_train, y=X_train, epochs=epochs, batch_size=batch)
def res_atte(
           n_classes, 
           feat_dim,
           max_len,
           dropout=dropout,
           kernel_regularizer=l1(1.e-4),
           activation="relu"):

  config = [ 
             [(1,8,64)],
             [(1,8,64)],
             [(1,8,64)],
             [(2,8,128)],
             [(1,8,128)],
             [(1,8,128)],
             [(2,8,256)],
             [(1,8,256)],
             [(1,8,256)],
           ]
  initial_stride = 1
  initial_filter_dim = 8
  initial_num = 64
  res_s = []
  res_num = [2,5,8]
  
  input = Input(shape=(max_len,feat_dim))
  model = input
  print "xxxx"
  print model


  atten_map = Dense(300,activation="tanh")(model)
  print "atten_map:",atten_map
  atten_map = Dense(150)(atten_map)
  print "atten_map:",atten_map
  atten_map = Activation("softmax")(atten_map)
  model = multiply([model,atten_map])
  
  model = Conv1D(initial_num, 
                 initial_filter_dim,
                 strides=initial_stride,
                 padding="same",
                 kernel_initializer="he_normal",
                 kernel_regularizer=kernel_regularizer)(model)

  for depth in range(0,len(config)):
    res_s.append(model)
    for stride,filter_dim,num in config[depth]:
      model = BatchNormalization(axis=2)(model)
      model = Activation(activation)(model)
      model = Dropout(dropout)(model)

      model = Conv1D(num, 
                   filter_dim,
                   strides=stride,
                   padding="same",
                   kernel_initializer="he_normal",
                   kernel_regularizer=kernel_regularizer)(model)
      print model
      if depth in res_num:
        print len(res_s)
        #if depth == 2:
        #res = res_s[-3]
        #else:
        res = res_s[-3]
        res_shape = K.int_shape(res)
        model_shape = K.int_shape(model)
             
        if res_shape[2] != model_shape[2]:
          res = Conv1D(num, 
                       1,
                       strides=2,
                       padding="same",
                       kernel_initializer="he_normal",
                       kernel_regularizer=kernel_regularizer)(res)
        model_res = add([res,model])
        #print model
        model_shape = K.int_shape(model)
        atten_map = Dense(2*model_shape[-1],activation="relu")(model)
        print "atten_map:",atten_map
        atten_map = Dense(1)(atten_map)
        print "atten_map:",atten_map
        atten_map = Activation("softmax")(atten_map)

        model = multiply([model_res,atten_map])


  pool_window_shape = K.int_shape(model)
  gap = AveragePooling1D(pool_window_shape[1],
                           strides=1)(model)
  flatten = Flatten()(gap)
  dense = Dense(units=n_classes, 
                activation="softmax",
                kernel_initializer="he_normal")(flatten)
  model = Model(inputs=input, outputs=dense)
  return model
Example #47
0
def dense_model(Inputs, nclasses, l1Reg=0, dropoutRate=0.25):
    """
    Dense matrix, defaults similar to 2016 DeepCSV training
    """
    x = Dense(200, activation='relu', kernel_initializer='lecun_uniform', name='fc1_relu', W_regularizer=l1(l1Reg))(Inputs)
    x = Dropout(dropoutRate)(x)
    x = Dense(200, activation='relu', kernel_initializer='lecun_uniform', name='fc2_relu', W_regularizer=l1(l1Reg))(x)
    x = Dropout(dropoutRate)(x)
    x = Dense(200, activation='relu', kernel_initializer='lecun_uniform', name='fc3_relu', W_regularizer=l1(l1Reg))(x)
    x = Dropout(dropoutRate)(x)
    x = Dense(200, activation='relu', kernel_initializer='lecun_uniform', name='fc4_relu', W_regularizer=l1(l1Reg))(x)
    x = Dropout(dropoutRate)(x)
    x = Dense(200, activation='relu', kernel_initializer='lecun_uniform', name='fc5_relu', W_regularizer=l1(l1Reg))(x)
    predictions = Dense(nclasses, activation='softmax', kernel_initializer='lecun_uniform', name = 'output_softmax')(x)
    model = Model(inputs=Inputs, outputs=predictions)
    return model
Example #48
0
    sumXval = sumXval + XtestList[i].toarray()
Y_val = np_utils.to_categorical(Y_test - 1, 1000)

tensorboard = TensorBoard(log_dir='./exp5',
                          histogram_freq=1,
                          write_graph=True,
                          write_images=False)

nb_hidden = 1000
input1 = Input(shape=(1000, ))
input2 = Input(shape=(1000, ))
input3 = Input(shape=(1000, ))
concatL = Concatenate()([input1, input2, input3])
layer1 = Dense(nb_hidden,
               bias_initializer='zeros',
               kernel_regularizer=regularizers.l1(0.00002))(concatL)
layer1 = LeakyReLU(alpha=0.01)(layer1)
out = Dense(nb_classes,
            bias_initializer='zeros',
            kernel_regularizer=regularizers.l1(0.00002),
            kernel_initializer='identity',
            activation='softmax')(layer1)
model = Model([input1, input2, input3], out)
#model=load_model('genFit_ens450.h5');

model.compile(loss=nick_crossentropy,
              optimizer='adadelta',
              metrics=['accuracy', top3_acc, top5_acc])
G = myGeneratorTrain()
K = next(G)
X_i = K[0]
# Freeze all the layers

for layer in vgg_conv.layers[:]:
  layer.trainable = False
# Check the trainable status of the individual layers
for layer in vgg_conv.layers:
  print(layer, layer.trainable)

# Create the model
model = Sequential()
# Add the vgg convolutional base model
model.add(vgg_conv)
# Add new layers
model.add(Flatten())
model.add(Dense(1024, activation='relu',kernel_regularizer=l1(0.001), bias_regularizer=l1(0)))
model.add(Dropout(0.2))
model.add(Dense(256, activation='relu',kernel_regularizer=l1(0.001), bias_regularizer=l1(0)))
model.add(Dense(8, activation='softmax',kernel_regularizer=l1(0.001), bias_regularizer=l1(0)))
# Show a summary of the model. Check the number of trainable parameters
model.summary()

# Load the normalized images
train_datagen = ImageDataGenerator(rescale=1./255)
validation_datagen = ImageDataGenerator(rescale=1./255)

traindf=pd.read_csv('/content/drive/My Drive/Colab Notebooks/dataset/train.csv',dtype=str)

training_datagen = ImageDataGenerator(
      rescale = 1./255,
	    rotation_range=40,
Example #50
0
from keras.callbacks import TensorBoard, ReduceLROnPlateau, ModelCheckpoint

from keras import regularizers
from keras.optimizers import Adam

# 指定使用的GPU
os.environ["CUDA_VISIBLE_DEVICES"] = "8"

file_name = os.path.join(root_dir, 'model_data', 'model.h5')

# 超参数
num_classes = 2
batch_size = 64
epochs = 30
dropout_rate = 0.25
reg = regularizers.l1(1e-4)
test_split = 0.2
lr = 1e-4

# 数据增强超参数
horizontal_flip = True
vertical_flip = True
rotation_angle = 180
width_shift_range = 0.1
height_shift_range = 0.1


def build_model():
    base_model = InceptionV3(weights='imagenet', include_top=False)

    x = base_model.output
Example #51
0
class KerasRegularizersTest(test_combinations.TestCase,
                            parameterized.TestCase):
    def create_model(self,
                     kernel_regularizer=None,
                     bias_regularizer=None,
                     activity_regularizer=None):
        model = keras.models.Sequential()
        model.add(
            keras.layers.Dense(NUM_CLASSES,
                               kernel_regularizer=kernel_regularizer,
                               bias_regularizer=bias_regularizer,
                               activity_regularizer=activity_regularizer,
                               input_shape=(DATA_DIM, )))
        return model

    def regularizer_fn_tensor(x):
        return tf.constant(0.)

    def regularizer_fn_scalar(x):
        return 0.

    class RegularizerTensor(regularizers.Regularizer):
        def __call__(self, x):
            return tf.constant(0.)

    class RegularizerScalar(regularizers.Regularizer):
        def __call__(self, x):
            return 0.

    def get_data(self):
        (x_train, y_train), (x_test, y_test) = test_utils.get_test_data(
            train_samples=10,
            test_samples=10,
            input_shape=(DATA_DIM, ),
            num_classes=NUM_CLASSES)
        y_train = np_utils.to_categorical(y_train, NUM_CLASSES)
        y_test = np_utils.to_categorical(y_test, NUM_CLASSES)
        return (x_train, y_train), (x_test, y_test)

    def create_multi_input_model_from(self, layer1, layer2):
        input_1 = keras.layers.Input(shape=(DATA_DIM, ))
        input_2 = keras.layers.Input(shape=(DATA_DIM, ))
        out1 = layer1(input_1)
        out2 = layer2(input_2)
        out = keras.layers.Average()([out1, out2])
        model = keras.models.Model([input_1, input_2], out)
        model.add_loss(keras.backend.mean(out2))
        model.add_loss(tf.reduce_sum(input_1))
        return model

    @test_combinations.run_all_keras_modes
    @parameterized.named_parameters([
        ('l1', regularizers.l1()),
        ('l2', regularizers.l2()),
        ('l1_l2', regularizers.l1_l2()),
        ('l2_zero', keras.regularizers.l2(0.)),
        ('function_tensor', regularizer_fn_tensor),
        ('function_scalar', regularizer_fn_scalar),
        ('lambda_tensor', lambda x: tf.constant(0.)),
        ('lambda_scalar', lambda x: 0.),
        ('regularizer_base_class', regularizers.Regularizer()),
        ('regularizer_custom_class_tensor', RegularizerTensor()),
        ('regularizer_custom_class_scalar', RegularizerScalar()),
    ])
    def test_kernel_regularization(self, regularizer):
        (x_train, y_train), _ = self.get_data()
        model = self.create_model(kernel_regularizer=regularizer)
        model.compile(loss='categorical_crossentropy',
                      optimizer='sgd',
                      run_eagerly=test_utils.should_run_eagerly())
        self.assertEqual(len(model.losses), 1)
        model.fit(x_train, y_train, batch_size=10, epochs=1, verbose=0)

    @test_combinations.run_all_keras_modes
    @parameterized.named_parameters([
        ('l1', regularizers.l1()),
        ('l2', regularizers.l2()),
        ('l1_l2', regularizers.l1_l2()),
        ('l2_zero', keras.regularizers.l2(0.)),
        ('function_tensor', regularizer_fn_tensor),
        ('function_scalar', regularizer_fn_scalar),
        ('lambda_tensor', lambda x: tf.constant(0.)),
        ('lambda_scalar', lambda x: 0.),
        ('regularizer_base_class', regularizers.Regularizer()),
        ('regularizer_custom_class_tensor', RegularizerTensor()),
        ('regularizer_custom_class_scalar', RegularizerScalar()),
    ])
    def test_bias_regularization(self, regularizer):
        (x_train, y_train), _ = self.get_data()
        model = self.create_model(bias_regularizer=regularizer)
        model.compile(loss='categorical_crossentropy',
                      optimizer='sgd',
                      run_eagerly=test_utils.should_run_eagerly())
        self.assertEqual(len(model.losses), 1)
        model.fit(x_train, y_train, batch_size=10, epochs=1, verbose=0)

    @test_combinations.run_all_keras_modes
    @parameterized.named_parameters([
        ('l1', regularizers.l1()),
        ('l2', regularizers.l2()),
        ('l1_l2', regularizers.l1_l2()),
        ('l2_zero', keras.regularizers.l2(0.)),
        ('function_tensor', regularizer_fn_tensor),
        ('function_scalar', regularizer_fn_scalar),
        ('lambda_tensor', lambda x: tf.constant(0.)),
        ('lambda_scalar', lambda x: 0.),
        ('regularizer_base_class', regularizers.Regularizer()),
        ('regularizer_custom_class_tensor', RegularizerTensor()),
        ('regularizer_custom_class_scalar', RegularizerScalar()),
    ])
    def test_activity_regularization(self, regularizer):
        (x_train, y_train), _ = self.get_data()
        model = self.create_model(activity_regularizer=regularizer)
        model.compile(loss='categorical_crossentropy',
                      optimizer='sgd',
                      run_eagerly=test_utils.should_run_eagerly())
        self.assertEqual(len(model.losses), 1 if tf.executing_eagerly() else 1)
        model.fit(x_train, y_train, batch_size=10, epochs=1, verbose=0)

    @test_combinations.run_all_keras_modes
    @test_combinations.run_with_all_model_types
    def test_zero_regularization(self):
        # Verifies that training with zero regularization works.
        x, y = np.ones((10, 10)), np.ones((10, 3))
        model = test_utils.get_model_from_layers([
            keras.layers.Dense(3, kernel_regularizer=keras.regularizers.l2(0))
        ],
                                                 input_shape=(10, ))
        model.compile('sgd',
                      'mse',
                      run_eagerly=test_utils.should_run_eagerly())
        model.fit(x, y, batch_size=5, epochs=1)

    def test_custom_regularizer_saving(self):
        def my_regularizer(weights):
            return tf.reduce_sum(tf.abs(weights))

        inputs = keras.Input((10, ))
        outputs = keras.layers.Dense(1,
                                     kernel_regularizer=my_regularizer)(inputs)
        model = keras.Model(inputs, outputs)
        model2 = model.from_config(
            model.get_config(),
            custom_objects={'my_regularizer': my_regularizer})
        self.assertEqual(model2.layers[1].kernel_regularizer, my_regularizer)

    @test_combinations.run_all_keras_modes
    @parameterized.named_parameters([
        ('l1', regularizers.l1()),
        ('l2', regularizers.l2()),
        ('l1_l2', regularizers.l1_l2()),
    ])
    def test_regularization_shared_layer(self, regularizer):
        dense_layer = keras.layers.Dense(NUM_CLASSES,
                                         kernel_regularizer=regularizer,
                                         activity_regularizer=regularizer)
        model = self.create_multi_input_model_from(dense_layer, dense_layer)
        model.compile(loss='categorical_crossentropy',
                      optimizer='sgd',
                      run_eagerly=test_utils.should_run_eagerly())
        self.assertLen(model.losses, 5)

    @test_combinations.run_all_keras_modes
    @parameterized.named_parameters([
        ('l1', regularizers.l1()),
        ('l2', regularizers.l2()),
        ('l1_l2', regularizers.l1_l2()),
    ])
    def test_regularization_shared_model(self, regularizer):
        dense_layer = keras.layers.Dense(NUM_CLASSES,
                                         kernel_regularizer=regularizer,
                                         activity_regularizer=regularizer)

        input_tensor = keras.layers.Input(shape=(DATA_DIM, ))
        dummy_model = keras.models.Model(input_tensor,
                                         dense_layer(input_tensor))

        model = self.create_multi_input_model_from(dummy_model, dummy_model)
        model.compile(loss='categorical_crossentropy',
                      optimizer='sgd',
                      run_eagerly=test_utils.should_run_eagerly())
        self.assertLen(model.losses, 6)

    @test_combinations.run_all_keras_modes
    @parameterized.named_parameters([
        ('l1', regularizers.l1()),
        ('l2', regularizers.l2()),
        ('l1_l2', regularizers.l1_l2()),
    ])
    def test_regularization_shared_layer_in_different_models(
            self, regularizer):
        shared_dense = keras.layers.Dense(NUM_CLASSES,
                                          kernel_regularizer=regularizer,
                                          activity_regularizer=regularizer)
        models = []
        for _ in range(2):
            input_tensor = keras.layers.Input(shape=(DATA_DIM, ))
            unshared_dense = keras.layers.Dense(NUM_CLASSES,
                                                kernel_regularizer=regularizer)
            out = unshared_dense(shared_dense(input_tensor))
            models.append(keras.models.Model(input_tensor, out))

        model = self.create_multi_input_model_from(layer1=models[0],
                                                   layer2=models[1])
        model.compile(loss='categorical_crossentropy',
                      optimizer='sgd',
                      run_eagerly=test_utils.should_run_eagerly())

        # We expect to see 9 losses on the model:
        # - 2 from the 2 add_loss calls on the outer model.
        # - 3 from the weight regularizers on the shared_dense layer, unshared_dense
        # in inner model 1, unshared_dense in inner model 2.
        # - 4 from activity regularizers on the shared_dense layer.
        self.assertLen(model.losses, 9)

    def test_deserialization_error(self):
        with self.assertRaisesRegex(ValueError,
                                    'Could not interpret regularizer'):
            keras.regularizers.get(0)

    @parameterized.named_parameters([
        ('l1', regularizers.l1(l1=None), 0.01),
        ('l2', regularizers.l2(l2=None), 0.01),
        ('l1_l2', regularizers.l1_l2(l1=None, l2=None), 0.),
    ])
    def test_default_value_when_init_with_none(self, regularizer,
                                               expected_value):
        expected_value = np.asarray(expected_value)
        if hasattr(regularizer, 'l1'):
            self.assertAllClose(regularizer.l1, expected_value)
        if hasattr(regularizer, 'l2'):
            self.assertAllClose(regularizer.l2, expected_value)

    @test_utils.run_v2_only
    def test_orthogonal_regularizer(self):
        # Test correctness.
        factor = 0.1
        reg_rows = regularizers.OrthogonalRegularizer(factor=factor,
                                                      mode='rows')
        reg_cols = regularizers.OrthogonalRegularizer(factor=factor,
                                                      mode='columns')

        # Test with square matrix
        inputs = tf.constant([[1, 1, 1, 1], [2, 0, 0, 0], [0, 0, 3, 1]],
                             dtype='float32')
        normalized_rows = tf.math.l2_normalize(inputs, axis=1)
        normalized_cols = tf.math.l2_normalize(inputs, axis=0)
        rows_pairs = [
            tf.reduce_sum(normalized_rows[0] * normalized_rows[1]),
            tf.reduce_sum(normalized_rows[0] * normalized_rows[2]),
            tf.reduce_sum(normalized_rows[1] * normalized_rows[2]),
        ]
        col_pairs = [
            tf.reduce_sum(normalized_cols[:, 0] * normalized_cols[:, 1]),
            tf.reduce_sum(normalized_cols[:, 0] * normalized_cols[:, 2]),
            tf.reduce_sum(normalized_cols[:, 0] * normalized_cols[:, 3]),
            tf.reduce_sum(normalized_cols[:, 1] * normalized_cols[:, 2]),
            tf.reduce_sum(normalized_cols[:, 1] * normalized_cols[:, 3]),
            tf.reduce_sum(normalized_cols[:, 2] * normalized_cols[:, 3]),
        ]
        num_row_pairs = 3
        num_col_pairs = 6
        # Expected: factor * sum(pairwise_dot_products_of_rows) / num_row_pairs
        self.assertAllClose(reg_rows(inputs),
                            factor * sum(rows_pairs) / num_row_pairs)
        # Expected: factor * sum(pairwise_dot_products_of_columns) / num_col_pairs
        self.assertAllClose(reg_cols(inputs),
                            factor * sum(col_pairs) / num_col_pairs)

        # Test incorrect usage.
        with self.assertRaisesRegex(ValueError, 'must have rank 2'):
            reg_rows(tf.constant([1, 1], dtype='float32'))

        # Test serialization
        self.assertDictEqual(reg_cols.get_config(), {
            'factor': factor,
            'mode': 'columns'
        })

        # Test usage in model.
        model_inputs = keras.Input((3, ))
        model_outputs = keras.layers.Dense(
            4, kernel_regularizer=reg_rows)(model_inputs)
        model = keras.Model(model_inputs, model_outputs)
        model.compile(optimizer='rmsprop', loss='mse')
        model.fit(np.random.random((16, 3)),
                  np.random.random((16, 4)),
                  epochs=1)

        # Test serialization and deserialiation as part of model.
        inputs = tf.constant([[1, 1, 1], [2, 0, 0], [0, 0, 3]],
                             dtype='float32')
        outputs = model(inputs)
        config = model.get_config()
        weights = model.get_weights()
        model = keras.Model.from_config(config)
        model.set_weights(weights)
        self.assertAllClose(model(inputs), outputs, atol=1e-5)
Example #52
0
    def _compare_models(architecture, loss, weight_decay=0.0, number_of_samples=3, learning_rate=0.01, epochs=1):
        np.random.seed(42)

        x = np.random.rand(number_of_samples, architecture[0]['input'])
        if 'MSE' == loss:
            y = np.random.rand(number_of_samples, 1)
            keras_loss = 'MSE'
            keras_metrics = None
        elif 'cross-entropy' == loss:
            number_of_classes = architecture[-1]['output']
            y = np.eye(number_of_classes)[np.random.choice(number_of_classes, size=number_of_samples)]
            keras_loss = losses.categorical_crossentropy
            keras_metrics = [metrics.categorical_accuracy]
        else:
            assert False, loss

        actual = mydnn(architecture,
                       loss,
                       weight_decay)
        sgd = optimizers.SGD(lr=learning_rate)
        desired = Sequential()

        for index, layer in enumerate(architecture):
            units = layer['output']
            activation = layer['nonlinear']

            if 'sot-max' == activation:
                activation = 'softmax'

            def kernel_initializer(shape, dtype=None):
                w = actual._architecture[index]._w.get_value()
                assert w.shape == shape

                return w

            def bias_initializer(shape, dtype=None):
                b = actual._architecture[index]._b.get_value()
                assert b.shape == shape

                return b

            if 'l1' == layer['regularization']:
                kernel_regularizer = regularizers.l1(weight_decay)
            elif 'l2' == layer['regularization']:
                kernel_regularizer = regularizers.l2(weight_decay)
            else:
                assert False

            desired.add(layers.Dense(units,
                                     activation=activation,
                                     kernel_initializer=kernel_initializer,
                                     bias_initializer=bias_initializer,
                                     kernel_regularizer=kernel_regularizer,
                                     input_shape=(layer['input'],)))

        desired.compile(sgd, keras_loss, metrics=keras_metrics)

        desired.fit(x, y, batch_size=x.shape[0], epochs=1)
        actual.fit(x, y, epochs, x.shape[0], learning_rate)

        weights_and_biases_desired = desired.get_weights()
        weights_and_biases_actual = list()

        for index in range(len(architecture)):
            weights_and_biases_actual.append(actual._architecture[index]._w.get_value())
            weights_and_biases_actual.append(actual._architecture[index]._b.get_value())

        for weight_actual, weight_desired in zip(weights_and_biases_actual, weights_and_biases_desired):
            np.testing.assert_allclose(weight_actual, weight_desired, atol=1e-5)

        y_predicted_actual = actual.predict(x, batch_size=number_of_samples)
        y_predicted_desired = desired.predict(x, batch_size=number_of_samples)

        np.testing.assert_allclose(y_predicted_actual, y_predicted_desired)

        if 'MSE' == loss:
            loss_actual = actual.evaluate(x, y, batch_size=number_of_samples)
            loss_desired = desired.evaluate(x, y, batch_size=number_of_samples)

            np.testing.assert_allclose(loss_actual, loss_desired)
        else:
            loss_actual, accuracy_actual = actual.evaluate(x, y, batch_size=number_of_samples)
            loss_desired, accuracy_desired = desired.evaluate(x, y, batch_size=number_of_samples)

            np.testing.assert_allclose(loss_actual, loss_desired)
            np.testing.assert_allclose(accuracy_actual, accuracy_desired)
Example #53
0
    def build_model(self):
        """Build a critic (value) network that maps (state, action) pairs -> Q-values."""
        # Define input layers
        """
        它在某些方面比行动者模型简单,但是需要注意几点。首先,行动者模型旨在将状态映射到动作,
        而评论者模型需要将(状态、动作)对映射到它们的 Q 值。这一点体现在了输入层中。"""
        states = layers.Input(shape=(self.state_size, ), name='states')
        actions = layers.Input(shape=(self.action_size, ), name='actions')

        # Add hidden layer(s) for state pathway
        """It took me so, use_bias=True, bias_constraint=regularizers.l2(0.02)"""
        net_states = states
        net_states = layers.Dense(
            units=32,
            kernel_regularizer=regularizers.l2(0.001),
            activity_regularizer=regularizers.l1(0.001))(net_states)
        net_states = layers.LeakyReLU(alpha=0.3)(net_states)
        net_states = layers.BatchNormalization()(net_states)
        net_states = layers.Dropout(0.4)(net_states)

        net_states = layers.Dense(
            units=64,
            kernel_regularizer=regularizers.l2(0.001),
            activity_regularizer=regularizers.l1(0.001))(net_states)
        net_states = layers.LeakyReLU(alpha=0.3)(net_states)
        net_states = layers.BatchNormalization()(net_states)
        net_states = layers.Dropout(0.4)(net_states)

        net_actions = actions
        net_actions = layers.Dense(
            units=32,
            kernel_regularizer=regularizers.l2(0.001),
            activity_regularizer=regularizers.l1(0.001))(net_actions)
        net_actions = layers.LeakyReLU(alpha=0.3)(net_actions)
        net_actions = layers.BatchNormalization()(net_actions)
        net_actions = layers.Dropout(0.4)(net_actions)

        net_actions = layers.Dense(
            units=64,
            kernel_regularizer=regularizers.l2(0.001),
            activity_regularizer=regularizers.l1(0.001))(net_actions)
        net_actions = layers.LeakyReLU(alpha=0.3)(net_actions)
        net_actions = layers.BatchNormalization()(net_actions)
        net_actions = layers.Dropout(0.4)(net_actions)

        # Try different layer sizes, activations, add batch normalization, regularizers, etc.

        # Combine state and action pathways
        """这两个层级首先可以通过单独的“路径”(迷你子网络)处理,但是最终需要结合到一起
        例如,可以通过使用 Keras 中的 Add 层级类型来实现请参阅合并层级):"""
        net = layers.Add()([net_states, net_actions])

        net = layers.Dense(units=256,
                           kernel_regularizer=regularizers.l2(0.001),
                           activity_regularizer=regularizers.l1(0.001))(net)
        net = layers.LeakyReLU(alpha=0.3)(net)
        net = layers.BatchNormalization()(net)
        net = layers.Dropout(0.4)(net)

        net = layers.Dense(units=16,
                           kernel_regularizer=regularizers.l2(0.001),
                           activity_regularizer=regularizers.l1(0.001))(net)
        net = layers.LeakyReLU(alpha=0.3)(net)

        # Add more layers to the combined network if needed

        # Add final output layer to prduce action values (Q values)
        Q_values = layers.Dense(units=1, name='q_values')(net)

        # Create Keras model
        self.model = models.Model(inputs=[states, actions], outputs=Q_values)

        # Define optimizer and compile model for training with built-in loss function
        optimizer = optimizers.Adam(lr=1e-4)
        self.model.compile(optimizer=optimizer, loss='mse')

        # Compute action gradients (derivative of Q values w.r.t. to actions)
        """该模型的最终输出是任何给定(状态、动作)对的 Q 值。但是,我们还需要计算此 Q 值相对于相应动作向量的梯度,以用于训练行动者模型。
        这一步需要明确执行,并且需要定义一个单独的函数来访问这些梯度:"""
        action_gradients = K.gradients(Q_values, actions)

        # Define an additional function to fetch action gradients (to be used by actor model)
        self.get_action_gradients = K.function(
            inputs=[*self.model.input, K.learning_phase()],
            outputs=action_gradients)
Example #54
0
# ## Set up a neural network with relu-encoder

# In[72]:

nb_epoch = 100
batch_size = 128
input_dim = df_train_0_x_rescaled.shape[1]  #num of predictor variables,
encoding_dim = 16
hidden_dim = int(encoding_dim / 2)
learning_rate = 1e-1

input_layer = Input(shape=(input_dim, ))
encoder = Dense(
    encoding_dim,
    activation="tanh",
    activity_regularizer=regularizers.l1(learning_rate))(input_layer)
encoder = Dense(hidden_dim, activation="relu")(encoder)
decoder = Dense(hidden_dim, activation='tanh')(encoder)
decoder = Dense(input_dim, activation='relu')(decoder)
autoencoder = Model(inputs=input_layer, outputs=decoder)

# This requires pydot and graphviz installed

# In[73]:

from IPython.display import SVG
from keras.utils.vis_utils import model_to_dot

SVG(
    model_to_dot(autoencoder, show_shapes=True,
                 show_layer_names=True).create(prog='dot', format='svg'))
Example #55
0
def create_model(x_train, y_train, x_val, y_val, x_test, y_test):

    if sys.argv[1] == 'german':
        input_n = 24
    elif sys.argv[1] == 'australian':
        input_n = 15

    batch_size = 32
    epochs = 500
    inits = [
        'Zeros', 'Ones', 'RandomNormal', 'RandomUniform', 'TruncatedNormal',
        'Orthogonal', 'lecun_uniform', 'lecun_normal', 'he_uniform',
        'he_normal', 'glorot_uniform', 'glorot_normal'
    ]
    acts = [
        'tanh', 'softsign', 'sigmoid', 'hard_sigmoid', 'relu', 'softplus',
        'LeakyReLU', 'PReLU', 'elu', 'selu'
    ]
    init = inits[int({{quniform(0, 11, 1)}})]
    act = acts[1]

    neurons = int({{quniform(9, 180, 9)}})
    layers = {{choice([1, 2, 4, 8])}}
    norm = {{choice(['no', 'l1', 'l2'])}}
    dropout = {{choice([0, 1])}}
    earlystop = {{choice([0, 1])}}
    k1 = None
    k2 = None
    p = None

    if norm == 'no':
        reg = None
    elif norm == 'l1':
        k1 = {{loguniform(-9.2, -2.3)}}
        reg = regularizers.l1(k1)
    elif norm == 'l2':
        k2 = {{loguniform(-9.2, -2.3)}}
        reg = regularizers.l2(k2)

    X_input = Input((input_n, ))
    X = X_input

    for _ in range(layers):
        X = Dense(
            neurons,
            kernel_initializer=init,
            kernel_regularizer=reg,
        )(X)

        if act == 'LeakyReLU':
            X = LeakyReLU()(X)
        elif act == 'PReLU':
            X = PReLU()(X)
        else:
            X = Activation(act)(X)

        if dropout == 1:
            p = {{uniform(0, 1)}}
            X = Dropout(p)(X)

    X = Dense(1, kernel_initializer=init, kernel_regularizer=reg)(X)
    X_outputs = Activation('sigmoid')(X)

    model = Model(inputs=X_input, outputs=X_outputs)
    model.compile(
        loss='binary_crossentropy',
        optimizer='adam',
        metrics=['accuracy'],
    )

    patience = int({{quniform(1, 500, 1)}})
    es = EarlyStopping(
        monitor='val_loss',
        patience=patience,
        verbose=0,
        mode='auto',
    )
    if earlystop == 1:
        model.fit(
            x_train,
            y_train,
            batch_size=batch_size,
            verbose=0,
            epochs=epochs,
            validation_data=(x_val, y_val),
            callbacks=[es],
        )
    else:
        model.fit(
            x_train,
            y_train,
            batch_size=batch_size,
            verbose=0,
            epochs=epochs,
            validation_data=(x_val, y_val),
        )

    loss_t, score_t = model.evaluate(x_train, y_train, verbose=0)
    loss_v, score_v = model.evaluate(x_val, y_val, verbose=0)
    loss_te, score_te = model.evaluate(x_test, y_test, verbose=0)

    print(init + '\t' + act + '\t' + str(neurons) + '\t' + str(layers) + '\t' +
          str(norm) + '\t' + str(dropout) + '\t' + str(earlystop) +
          '%-24s%-24s%-24s%s' % (str(k1), str(k2), str(p), str(patience)) +
          '  ' + str(score_v) + '  ' + str(loss_v) + '  ' + str(score_te) +
          '  ' + str(loss_te))
    return {'loss': loss_v, 'status': STATUS_OK, 'model': model}
Example #56
0
 def i3d_flattened(self, num_classes = 60):
     i3d = Model(inputs = self.model.input, outputs = self.model.get_layer(index=-4).output)
     x = conv3d_bn(i3d.output, num_classes, 1, 1, 1, padding='same', use_bias=True, use_activation_fn=False, use_bn=False, name='Conv3d_6a_1x1')
     num_frames_remaining = int(x.shape[1])
     x = Flatten()(x)
     predictions = Dense(num_classes, activation = 'softmax', kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l1(0.01))(x)
     new_model  = Model(inputs = i3d.input, outputs = predictions)
     
     #for layer in i3d.layers:
     #    layer.trainable = False
     
     return new_model
Example #57
0
from keras.layers import BatchNormalization
from keras.layers import GaussianNoise, GaussianDropout, AlphaDropout
from keras.layers import Input
from keras.layers import TimeDistributed, Bidirectional
from keras import regularizers

fillerMap = {
    'constant': 'Constant',
    'uniform': 'RandomUniform',
    'gaussian': 'RandomNormal',
    'xavier': 'glorot_normal',
    'msra': 'he_normal'
}

regularizerMap = {
    'l1': regularizers.l1(),
    'l2': regularizers.l2(),
    'l1_l2': regularizers.l1_l2(),
    'L1L2': regularizers.l1_l2(),
    'None': None
}

constraintMap = {
    'max_norm': 'max_norm',
    'non_neg': 'non_neg',
    'unit_norm': 'unit_norm',
    'MaxNorm': 'max_norm',
    'NonNeg': 'non_neg',
    'UnitNorm': 'unit_norm',
    'None': None
}
Example #58
0
    def train(self, dataset, testset, tagger, epochs=3):
        """
        @param dataset : a list of dependency trees
        """
        N = len(dataset)
        sequences = [(dtree.tokens, self.oracle_derivation(dtree))
                     for dtree in dataset]

        X_S, X_B, Y = [], [], []

        def map_tokens(S, B, tokens):
            S = [
                tagger.x_codes[tokens[s][0]] if tokens[s][0] in tagger.x_codes
                else tagger.x_codes["__UNK__"] for s in S
            ]
            B = [
                tagger.x_codes[tokens[b][0]] if tokens[b][0] in tagger.x_codes
                else tagger.x_codes["__UNK__"] for b in B
            ]
            return S, B

        ##### DARK SIDE #####
        for tokens, ref_derivation in sequences:
            current_config = ref_derivation[0][1]
            for action, config in ref_derivation[
                    1:]:  #do not learn the "None" dummy action
                S, B, _, _ = current_config
                current_config = config
                S, B = map_tokens(S, B, tokens)
                X_S.append(S)
                X_B.append(B)
                Y.append(action)
        XS_encoded, XB_encoded = pad_sequences(
            X_S, maxlen=tagger.mL), pad_sequences(X_B, maxlen=tagger.mL)
        self.x_dict = tagger.x_codes
        self.mL = tagger.mL
        self.y_size = len(set(Y))
        self.y_list = list(set(Y))
        self.y_dict = {y: i for i, y in enumerate(self.y_list)}
        self.reverse_y_dict = dict(enumerate(self.y_list))
        Y_encoded = np.zeros(shape=(len(Y), self.y_size))
        for i, y in enumerate(Y):
            Y_encoded[i, self.y_dict[y]] = 1.
        ###$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$###
        # print(XS_encoded)
        tXS, tXB, tY = self.prep(testset)

        # exit()
        ipt_stack = Input(shape=(tagger.mL, ))
        ipt_stack_pos = Input(shape=(self.yML, ))
        ipt_buffer = Input(shape=(tagger.mL, ))
        e_stack = tagger.model.get_layer("embedding_1")(ipt_stack)
        e_buffer = tagger.model.get_layer("embedding_1")(ipt_buffer)
        l_s = tagger.model.get_layer("bidirectional_1")(e_stack)
        l_b = tagger.model.get_layer("bidirectional_1")(e_buffer)
        l1 = LSTM(122, return_sequences=True)

        l1 = concatenate([l1(l_s), l1(l_b)], axis=1)
        # l2 = Flatten())
        l3 = LSTM(122)(l1)
        o = Dense(self.y_size, activation="softmax")(l3)
        self.nn_parser = Model([ipt_stack, ipt_buffer], o)
        self.nn_parser.summary()
        sgd = RMSprop(lr=.01)
        self.nn_parser.compile(optimizer=sgd,
                               loss='categorical_crossentropy',
                               metrics=['accuracy'])
        self.nn_parser.fit([XS_encoded, XB_encoded],
                           Y_encoded,
                           epochs=epochs,
                           verbose=1,
                           validation_split=.2)
        print(self.nn_parser.evaluate([tXS, tXB], tY, batch_size=64))
        return self
Example #59
0
                                                    test_size=0.2,
                                                    random_state=42)

X_train = X_train.values
X_test = X_test.values

X_train.shape

input_dim = X_train.shape[1]
encoding_dim = 162

input_layer = Input(shape=(input_dim, ))

encoder = Dense(encoding_dim,
                activation="tanh",
                activity_regularizer=regularizers.l1(10e-5))(input_layer)
encoder = Dense(int(encoding_dim / 2), activation="relu")(encoder)

decoder = Dense(int(encoding_dim / 2), activation='tanh')(encoder)
decoder = Dense(input_dim, activation='relu')(decoder)

autoencoder = Model(inputs=input_layer, outputs=decoder)

nb_epoch = 30
batch_size = 32

autoencoder.compile(loss='categorical_crossentropy',
                    optimizer='rmsprop',
                    metrics=['accuracy'])

checkpointer = ModelCheckpoint(filepath="model.h5",
Example #60
0
ty_track = np.zeros((dmp.timesteps, dmp.n_dmps))
tdy_track = np.zeros((dmp.timesteps, dmp.n_dmps))
tddy_track = np.zeros((dmp.timesteps, dmp.n_dmps))


goals = [[-1, 0] for i in range(10)]#[[np.cos(theta), np.sin(theta)] for theta in np.linspace(0, 2*np.pi, 20)[:-1]]

for goal in goals:
    dmp.goal = goal
    dmp.reset_state()
    for t in range(dmp.timesteps):
        ty_track[t], tdy_track[t], tddy_track[t] = dmp.step(external_force=avoid_obstacles(dmp.y, dmp.dy, goal, t))

model = Sequential()
model.add(Dense(10, activation='relu', bias_regularizer=regularizers.l1(0.01), input_dim=3))
model.add(Dense(2, activation='tanh'))



inpt = np.vstack((obst_vec[:,0], obst_vec[:,1], cur_vel))
inpt = np.swapaxes(inpt, 0, 1)
inpt = preprocessing.scale(inpt)

model.compile(optimizer='adagrad',
              loss='mse', metrics=[metrics.mse])

label = c_t