def main(nb_units, depth, nb_epoch, filter_size, project_factor, nb_dense): h5_fname = "/home/leon/data/generated_tags_variable_depth_map_normed.hdf5" batch_size = 32 output_dir = "models/holy/mask_generator_var_with_depth_n{}_d{}_e{}/" \ .format(nb_units, depth, nb_epoch) use_l1_regularizer = False os.makedirs(output_dir) gen = generator(h5_fname, batch_size) nb_input = next(gen)[0][0].shape[1] x = Input(shape=(nb_input,)) mask, depth_map = mask_generator_all_conv(x, nb_units=nb_units, depth=depth, filter_size=filter_size) if use_l1_regularizer: layers = collect_layers(x, [mask, depth_map]) for l in layers: if hasattr(l, 'W_regularizer'): l.W_regularizer = l1(0.001) if hasattr(l, 'b_regularizer'): l.b_regularizer = l1(0.001) g = Model(x, [mask, depth_map]) optimizer = Adam() mask_more_weight = 5 total_pixels = mask_more_weight*64**2 + 16**2 weight_depth_map = 16**2 / total_pixels weight_mask = mask_more_weight*64**2 / total_pixels g.compile(optimizer, [weight_mse(weight_mask), weight_mse(weight_depth_map)]) scheduler = AutomaticLearningRateScheduler( optimizer, 'loss', epoch_patience=4, min_improvment=0.0002) history = HistoryPerBatch() save = SaveModels({'{epoch:^03}_mask_generator.hdf5': g}, output_dir=output_dir) g.fit_generator(gen, samples_per_epoch=200*batch_size, nb_epoch=nb_epoch, verbose=1, callbacks=[scheduler, save]) ins = next(generator(h5_fname, batch_size)) predict_masks, predict_depth_map = g.predict(ins[0]) masks, depth_map = ins[1] def clip(x): return np.clip(x, 0, 1) zip_visualise_tiles(clip(masks), clip(predict_masks), show=False) plt.savefig(output_dir + "predict_tags.png") zip_visualise_tiles(clip(depth_map), clip(predict_depth_map), show=False) plt.savefig(output_dir + "predict_depth_map.png") g.save_weights(output_dir + "mask_generator.hdf5", overwrite=True) with open(output_dir + 'mask_generator.json', 'w+') as f: f.write(g.to_json()) with open(output_dir + 'history.json', 'w+') as f: json.dump(history.history, f)
def __init__(self): X1 = T.tensor4() X2 = T.tensor4() X = [X1, X2] Y = [T.ivector()] model = Model() #conv1 model.add(Conv(filter_shape = (32, 3, 3, 3), regularizers = {'W': l1(0.0001)}, w_shared = True, n_inputs = 2)) model.add(Conv(filter_shape = (32, 32, 2, 2), regularizers = {'W': l1(0.0001)}, w_shared = True, n_inputs = 2)) model.add(Pooling(pool_size = (2,2))) model.add(Activation(mode = 'tanh')) #conv2 model.add(Conv(filter_shape = (32, 32, 3, 3), regularizers = {'W': l1(0.0001)}, w_shared = True, n_inputs = 2)) model.add(Pooling(pool_size = (2,2))) model.add(Activation(mode = 'tanh')) #abs_diff model.add(Abs_diff()) #conv3 model.add(Conv(filter_shape = (32, 32, 3, 3), regularizers = {'W': l1(0.0001)}, w_shared = True)) model.add(Pooling(pool_size = (2,2))) model.add(Activation(mode = 'tanh')) model.add(Flatten()) self.f = theano.function(X, model.f(X, is_train = True)) model.add(Fully((2880, 512))) model.add(Activation(mode = 'tanh')) model.add(Dropout(0.5)) model.add(Fully((512, 2))) model.add(Activation(mode = 'softmax')) model.build(CostFunc.nll, RMSprop(), X, Y) self.model = model
def __init__(self): left = Sequential() left.add(Dense(784, 50)) left.add(Activation('relu')) model = Sequential() model.add(Merge([left, left], mode='sum')) model.add(Dense(50, 10)) model.add(Activation('softmax')) pdb.set_trace() model = Sequential() left = Sequential() num_kernel = 32 l1_penalty = 0.0001 b_mode = 'full' left.add(Convolution2D(num_kernel, 3, 2, 2, W_regularizer=l1(l1_penalty), border_mode=b_mode)) left.add(Convolution2D(num_kernel, num_kernel, 2, 2, W_regularizer=l1(l1_penalty), border_mode=b_mode)) left.add(LeakyReLU(0.1)) #left.add(Activation('relu')) left.add(MaxPooling2D(poolsize=(2, 2))) #left.add(Convolution2D(num_kernel, 3, 2, 2, W_regularizer=l1(l1_penalty), border_mode=b_mode)) #left.add(Convolution2D(num_kernel, num_kernel, 2, 2, W_regularizer=l1(l1_penalty), border_mode=b_mode)) #left.add(LeakyReLU(0.1)) ##left.add(Activation('relu')) #left.add(MaxPooling2D(poolsize=(2, 2))) model.add(Merge([left, left], mode='sum')) pdb.set_trace() self.f = theano.function(model.get_input(), model.get_output())
def skip_layer(x, nb_layers=16): x1 = Dense(nb_layers, kernel_regularizer=l1(0.01))(x) x1 = Activation('relu')(x1) x2 = Dense(nb_layers, kernel_regularizer=l1(0.01))(x1) x2 = Activation('relu')(x2) x3 = Add()([x1, x2]) return x3
def __init__(self, seq_length, use_RNN=False, num_tasks=1, num_filters=(15, 15, 15), conv_width=(15, 15, 15), pool_width=35, GRU_size=35, TDD_size=15, L1=0, dropout=0.0, num_epochs=100, verbose=1): self.saved_params = locals() self.seq_length = seq_length self.input_shape = (1, 4, self.seq_length) self.num_tasks = num_tasks self.num_epochs = num_epochs self.verbose = verbose self.model = Sequential() assert len(num_filters) == len(conv_width) for i, (nb_filter, nb_col) in enumerate(zip(num_filters, conv_width)): conv_height = 4 if i == 0 else 1 self.model.add(Convolution2D( nb_filter=nb_filter, nb_row=conv_height, nb_col=nb_col, activation='linear', init='he_normal', input_shape=self.input_shape, W_regularizer=l1(L1), b_regularizer=l1(L1))) self.model.add(Activation('relu')) self.model.add(Dropout(dropout)) self.model.add(MaxPooling2D(pool_size=(1, pool_width))) if use_RNN: num_max_pool_outputs = self.model.layers[-1].output_shape[-1] self.model.add(Reshape((num_filters[-1], num_max_pool_outputs))) self.model.add(Permute((2, 1))) self.model.add(GRU(GRU_size, return_sequences=True)) self.model.add(TimeDistributedDense(TDD_size, activation='relu')) self.model.add(Flatten()) self.model.add(Dense(output_dim=self.num_tasks)) self.model.add(Activation('sigmoid')) self.model.compile(optimizer='adam', loss='binary_crossentropy') self.train_losses = None self.valid_losses = None
def test_dense(): layer_test(layers.Dense, kwargs={'units': 3}, input_shape=(3, 2)) layer_test(layers.Dense, kwargs={'units': 3}, input_shape=(3, 4, 2)) layer_test(layers.Dense, kwargs={'units': 3}, input_shape=(None, None, 2)) layer_test(layers.Dense, kwargs={'units': 3}, input_shape=(3, 4, 5, 2)) layer_test(layers.Dense, kwargs={'units': 3, 'kernel_regularizer': regularizers.l2(0.01), 'bias_regularizer': regularizers.l1(0.01), 'activity_regularizer': regularizers.L1L2(l1=0.01, l2=0.01), 'kernel_constraint': constraints.MaxNorm(1), 'bias_constraint': constraints.max_norm(1)}, input_shape=(3, 2)) layer = layers.Dense(3, kernel_regularizer=regularizers.l1(0.01), bias_regularizer='l1') layer.build((None, 4)) assert len(layer.losses) == 2
def __init__(self, X_shape_1, n_h_nodes, alpha): in_model = Input(shape=(X_shape_1,)) hidden_l = Dense(n_h_nodes, activation='relu', W_regularizer=l1(alpha))(in_model) out_model = Dense(1, activation='linear', W_regularizer=l1(alpha))(hidden_l) super().__init__(input = in_model, output=out_model) self.compile(optimizer='adadelta', loss='mse', metrics=['accuracy'])
def test_regularization_shared_layer(): dense_layer = Dense(num_classes, kernel_regularizer=regularizers.l1(), activity_regularizer=regularizers.l1()) model = create_multi_input_model_from(dense_layer, dense_layer) model.compile(loss='categorical_crossentropy', optimizer='sgd') assert len(model.losses) == 6
def buildModel(weight_decay1, weight_decay2, nodes_per_layer, activation_functions=['sigmoid','sigmoid'], solver='Adadelta', regulization_method='l2'): # Sequential to get a linear layer structure as in feedforward networks model = Sequential() if(regulization_method is 'l1'): # input to hidden layer model.add(Dense(input_dim=X_train.shape[1], # input dimension output_dim=nodes_per_layer, # output dimension init='uniform', # initialization of the weights activation=activation_functions[0], # activation function, W_regularizer=l1(weight_decay1), # weight decay on the first level conncetion weight b_regularizer=l1(weight_decay1) # weight decay for bias unit )) # hidden to output layer model.add(Dense(input_dim=nodes_per_layer, # input dimension must match the previous output_dim output_dim=2, # output dimension must match the target dimension init='uniform', # initialization of the weights activation=activation_functions[1], # activation function, W_regularizer=l1(weight_decay2), # weight decay b_regularizer=l1(weight_decay2) # weight decay for bias unit )) else: # input to hidden layer model.add(Dense(input_dim=X_train.shape[1], # input dimension output_dim=nodes_per_layer, # output dimension init='uniform', # initialization of the weights activation=activation_functions[0], # activation function, W_regularizer=l2(weight_decay1), # weight decay on the first level conncetion weight b_regularizer=l2(weight_decay1) # weight decay for bias unit )) # hidden to output layer model.add(Dense(input_dim=nodes_per_layer, # input dimension must match the previous output_dim output_dim=2, # output dimension must match the target dimension init='uniform', # initialization of the weights activation=activation_functions[1], # activation function, W_regularizer=l2(weight_decay2), # weight decay b_regularizer=l2(weight_decay2) # weight decay for bias unit )) # setup solver if(solver is 'Adam'): bp = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08) if(solver is 'Adadelta'): bp = Adadelta(lr=1.0, rho=0.95, epsilon=1e-08) if (solver is 'Adagrad'): bp = Adagrad(lr=0.01, epsilon=1e-08) else: bp = SGD(lr=0.01) # learning rate recommended to tune, but was found to have no influenceas long it is between 0.01 and 0.1 # compile model model.compile(loss='binary_crossentropy', optimizer=bp, class_mode="binary") return model
def test_regularizer(layer_class): layer = layer_class(units, return_sequences=False, weights=None, batch_input_shape=(num_samples, timesteps, embedding_dim), kernel_regularizer=regularizers.l1(0.01), recurrent_regularizer=regularizers.l1(0.01), activity_regularizer='l1', bias_regularizer='l2') layer.build((None, None, 2)) assert len(layer.losses) == 3 layer(K.variable(np.ones((2, 3, 2)))) assert len(layer.losses) == 4
def test_regularization_shared_model(): dense_layer = Dense(num_classes, kernel_regularizer=regularizers.l1(), activity_regularizer=regularizers.l1()) input_tensor = Input(shape=(data_dim,)) dummy_model = Model(input_tensor, dense_layer(input_tensor)) model = create_multi_input_model_from(dummy_model, dummy_model) model.compile(loss='categorical_crossentropy', optimizer='sgd') assert len(model.losses) == 6
def test_regularization_shared_layer_in_different_models(): shared_dense = Dense(num_classes, kernel_regularizer=regularizers.l1(), activity_regularizer=regularizers.l1()) models = [] for _ in range(2): input_tensor = Input(shape=(data_dim,)) unshared_dense = Dense(num_classes, kernel_regularizer=regularizers.l1()) out = unshared_dense(shared_dense(input_tensor)) models.append(Model(input_tensor, out)) model = create_multi_input_model_from(*models) model.compile(loss='categorical_crossentropy', optimizer='sgd') assert len(model.losses) == 8
def main(nb_units, nb_epoch): h5_fname = "/home/leon/data/generated_tags_variable_depth_map.hdf5" batch_size = 32 output_dir = "models/holy/mask_generator_var_with_depth_n{}_d3/".format(nb_units) use_l1_regularizer = False assert not os.path.exists(output_dir) gen = generator(h5_fname, batch_size) nb_input = next(gen)[0][0].shape[1] x = Input(shape=(nb_input,)) out = mask_generator(x, nb_units=nb_units, dense_factor=7, nb_dense_layers=3, nb_output_channels=2) if use_l1_regularizer: layers = collect_layers(x, out) for l in layers: if hasattr(l, 'W_regularizer'): l.W_regularizer = l1(0.001) if hasattr(l, 'b_regularizer'): l.b_regularizer = l1(0.001) g = Model(x, out) optimizer = Adam() g.compile(optimizer, mse) scheduler = AutomaticLearningRateScheduler( optimizer, 'loss', epoch_patience=4, min_improvment=0.0002) history = HistoryPerBatch() g.fit_generator(gen, samples_per_epoch=200*batch_size, nb_epoch=nb_epoch, verbose=1, callbacks=[scheduler]) ins = next(gen) out = g.predict(ins[0]) def clip(x): return np.clip(x, 0, 1) os.makedirs(output_dir, exist_ok=True) zip_visualise_tiles(clip(ins[1][:, :1]), clip(out[:, :1]), show=False) plt.savefig(output_dir + "predict_tags.png") zip_visualise_tiles(clip(ins[1][:, 1:]), clip(out[:, 1:]), show=False) plt.savefig(output_dir + "predict_depth_map.png") g.save_weights(output_dir + "mask_generator.hdf5", overwrite=True) with open(output_dir + 'mask_generator.json', 'w+') as f: f.write(g.to_json()) with open(output_dir + 'history.json', 'w+') as f: json.dump(history.history, f)
def __init__(self, timesteps, attention_size, bias=True, W_regularizer=regularizers.l1(0.01), W_constraint=None, U_regularizer=regularizers.l1(0.01), U_constraint=None, **kwargs): self.supports_masking = True self.init = initializers.get('glorot_uniform') self.bias = bias self.timesteps = timesteps self.attention_size = attention_size self.W_regularizer = regularizers.get(W_regularizer) self.W_constraint = constraints.get(W_constraint) self.U_regularizer = regularizers.get(U_regularizer) self.U_constraint = constraints.get(U_constraint) super(Attention, self).__init__(**kwargs)
def test_activity_regularization(): x_train, y_train = get_data() for reg in [regularizers.l1(), regularizers.l2()]: model = create_model(activity_regularizer=reg) model.compile(loss='categorical_crossentropy', optimizer='sgd') assert len(model.losses) == 1 model.train_on_batch(x_train, y_train)
def test_dense(): from keras import regularizers from keras import constraints layer_test(core.Dense, kwargs={'output_dim': 3}, input_shape=(3, 2)) layer_test(core.Dense, kwargs={'output_dim': 3}, input_shape=(3, 4, 2)) layer_test(core.Dense, kwargs={'output_dim': 3}, input_shape=(None, None, 2)) layer_test(core.Dense, kwargs={'output_dim': 3}, input_shape=(3, 4, 5, 2)) layer_test(core.Dense, kwargs={'output_dim': 3, 'W_regularizer': regularizers.l2(0.01), 'b_regularizer': regularizers.l1(0.01), 'activity_regularizer': regularizers.activity_l2(0.01), 'W_constraint': constraints.MaxNorm(1), 'b_constraint': constraints.MaxNorm(1)}, input_shape=(3, 2))
def create_model(input_shape, n_classes): model = Sequential() model.add(Conv2D(32, (3, 3), padding='same', activation='relu', input_shape=input_shape)) model.add(Conv2D(32, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Conv2D(64, (3, 3), padding='same', activation='relu')) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) # model.add(Conv2D(32, (3, 3), padding='same', activation='relu')) # model.add(Conv2D(32, (3, 3), activation='relu')) # model.add(MaxPooling2D(pool_size=(2, 2))) # model.add(Dropout(0.25)) model.add(Conv2D(64, (3, 3), padding='same', activation='relu')) model.add(Conv2D(64, (3, 3), activation='relu', kernel_regularizer=regularizers.l1(0.001))) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(256, activation='relu', activity_regularizer=regularizers.l2(0.001))) model.add(Dropout(0.25)) model.add(Dense(n_classes, activation='softmax')) return model
def regularizer(): if args.l2_reg: return l2(args.l2_reg) elif args.l1_reg: return l1(args.l1_reg) else: return None
def train(self, X_train, Y_train, X_test, iter_): ''' output: predicted class: 0, 1, 2 ''' inputs = Input(shape=(12,)) x1 = Dense(96, activation='relu', W_regularizer=l1(0.01))(inputs) x2 = Dense(96, activation='relu', W_regularizer=l1(0.01))(x1) #drop = Dropout(0.2)(x) prediction = Dense(3, activation='relu', W_regularizer=l1(0.01))(x2) model = Model(input=inputs, output=prediction) model.compile(optimizer='adagrad', loss='poisson') model.fit(X_train, Y_train, nb_epoch=iter_, batch_size=100) pred = model.predict(X_test) return (np.argmax(pred, axis=1), np.max(pred, axis=1))
def test_W_reg(): for reg in [regularizers.identity(), regularizers.l1(), regularizers.l2(), regularizers.l1l2()]: model = create_model(weight_reg=reg) model.compile(loss='categorical_crossentropy', optimizer='rmsprop') model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0) model.evaluate(X_test[test_ids, :], Y_test[test_ids, :], verbose=0)
def test_activity_regularization(): (x_train, y_train), (x_test, y_test) = get_data() for reg in [regularizers.l1(), regularizers.l2()]: model = create_model(activity_regularizer=reg) model.compile(loss='categorical_crossentropy', optimizer='sgd') assert len(model.losses) == 1 model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=0)
def __init__(self, seq_length=None, keras_model=None, use_RNN=False, num_tasks=1, num_filters=(15, 15, 15), conv_width=(15, 15, 15), pool_width=35, GRU_size=35, TDD_size=15, L1=0, dropout=0.0, num_epochs=100, verbose=1): from keras.models import Sequential from keras.layers.core import ( Activation, Dense, Dropout, Flatten, Permute, Reshape, TimeDistributedDense ) from keras.layers.convolutional import Convolution2D, MaxPooling2D from keras.layers.recurrent import GRU from keras.regularizers import l1 self.num_tasks = num_tasks self.num_epochs = num_epochs self.verbose = verbose self.train_metrics = [] self.valid_metrics = [] if keras_model is not None and seq_length is None: self.model = keras_model self.num_tasks = keras_model.layers[-1].output_shape[-1] elif seq_length is not None and keras_model is None: self.model = Sequential() assert len(num_filters) == len(conv_width) for i, (nb_filter, nb_col) in enumerate(zip(num_filters, conv_width)): conv_height = 4 if i == 0 else 1 self.model.add(Convolution2D( nb_filter=nb_filter, nb_row=conv_height, nb_col=nb_col, activation='linear', init='he_normal', input_shape=(1, 4, seq_length), W_regularizer=l1(L1), b_regularizer=l1(L1))) self.model.add(Activation('relu')) self.model.add(Dropout(dropout)) self.model.add(MaxPooling2D(pool_size=(1, pool_width))) if use_RNN: num_max_pool_outputs = self.model.layers[-1].output_shape[-1] self.model.add(Reshape((num_filters[-1], num_max_pool_outputs))) self.model.add(Permute((2, 1))) self.model.add(GRU(GRU_size, return_sequences=True)) self.model.add(TimeDistributedDense(TDD_size, activation='relu')) self.model.add(Flatten()) self.model.add(Dense(output_dim=self.num_tasks)) self.model.add(Activation('sigmoid')) self.model.compile(optimizer='adam', loss='binary_crossentropy') else: raise ValueError("Exactly one of seq_length or keras_model must be specified!")
def get_regularizer(lambda_l1=None, lambda_l2=None): regularizer = None if lambda_l1 is None and lambda_l2 is not None: regularizer = l2(l=lambda_l2) elif lambda_l1 is not None and lambda_l2 is None: regularizer = l1(l=lambda_l1) elif lambda_l1 is not None and lambda_l2 is not None: regularizer = l1l2(l1=lambda_l1, l2=lambda_l2) return regularizer
def test_regularizer(layer_class): layer = layer_class(units, return_sequences=False, weights=None, input_shape=(timesteps, embedding_dim), kernel_regularizer=regularizers.l1(0.01), recurrent_regularizer=regularizers.l1(0.01), bias_regularizer='l2') layer.build((None, None, embedding_dim)) assert len(layer.losses) == 3 assert len(layer.cell.losses) == 3 layer = layer_class(units, return_sequences=False, weights=None, input_shape=(timesteps, embedding_dim), activity_regularizer='l2') assert layer.activity_regularizer x = K.variable(np.ones((num_samples, timesteps, embedding_dim))) layer(x) assert len(layer.cell.get_losses_for(x)) == 0 assert len(layer.get_losses_for(x)) == 1
def build_model(in_dim, out_dim=1, n_hidden=100, l1_norm=0.0, n_deep=5, drop=0.1, learning_rate=0.1): model = Sequential() # Input layer model.add(Dense( input_dim=in_dim, output_dim=n_hidden, init='glorot_uniform', activation='tanh', W_regularizer=l1(l1_norm))) # do X layers for layer in range(n_deep-1): model.add(Dropout(drop)) model.add(Dense( output_dim=np.round(n_hidden/2**(layer+1)), init='glorot_uniform', activation='tanh', W_regularizer=l1(l1_norm))) # Output layer if out_dim == 1: activation = 'tanh' else: activation = 'softmax' model.add(Dense(out_dim, init='glorot_uniform', activation=activation)) # Optimization algorithms opt = Adadelta(lr=learning_rate) if out_dim == 1: model.compile(loss='binary_crossentropy', optimizer=opt, class_mode='binary') else: model.compile(loss='categorical_crossentropy', optimizer=opt, class_mode='categorical') return model
def model_generator(latent_dim, input_shape, hidden_dim=512, reg=lambda: l1(1e-7)): return Sequential([ Dense(hidden_dim, name="generator_h1", input_dim=latent_dim, W_regularizer=reg()), LeakyReLU(0.2), Dense(hidden_dim, name="generator_h2", W_regularizer=reg()), LeakyReLU(0.2), Dense(np.prod(input_shape), name="generator_x_flat", W_regularizer=reg()), Activation('sigmoid'), Reshape(input_shape, name="generator_x")], name="generator")
def MLP_regression(layer1=None, layer2=None, layer3=None, input_shape=(64,)): model = Sequential() if layer1: model.add(Dense(layer1, input_dim=input_shape[0], init='glorot_uniform',\ activation='sigmoid', W_regularizer=l1(0.00),\ )) # model.add(Dropout(0.5)) if layer2: model.add(Dense(layer2, init='glorot_uniform',\ activation='sigmoid', W_regularizer=l1(0.01),\ )) if layer3: model.add(Dense(layer3, init='glorot_uniform',\ activation='sigmoid', W_regularizer=l1(0.01),\ )) #model.add(Dropout(0.5)) model.add(Dense(1, init='glorot_uniform', activation='sigmoid')) sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True) model.compile(class_mode='binary', loss='binary_crossentrob', optimizer=RMSprop()) return model
def make_3l_nn(): model = Sequential() model.add(Dense(1024, input_dim=x_train.shape[1], W_regularizer=l1(0.001), init="normal")) model.add(Activation("relu")) model.add(Dropout(0.5)) model.add(Dense(256)) model.add(Activation("relu")) model.add(Dropout(0.5)) model.add(Dense(6)) model.add(Activation("sigmoid")) return model
def test_W_reg(): (X_train, Y_train), (X_test, Y_test), test_ids = get_data() for reg in [regularizers.l1(), regularizers.l2(), regularizers.l1l2()]: model = create_model(weight_reg=reg) model.compile(loss='categorical_crossentropy', optimizer='rmsprop') assert len(model.losses) == 1 model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0) model.evaluate(X_test[test_ids, :], Y_test[test_ids, :], verbose=0)
def NNet(TrainX, TrainY, TestX): ## the network remained to be XJBplay model = Sequential() numNode = 20 model.add(Dense(numNode, input_dim=len(TrainX[0]), W_regularizer=l1(0.1))) model.add(BatchNormalization()) model.add(Activation('tanh')) model.add(Dropout(0.5)) model.add(Dense(numNode, W_regularizer=l2(1))) model.add(BatchNormalization()) model.add(Activation('tanh')) model.add(Dropout(0.2)) model.add(Dense(numNode, W_regularizer=l1(0.1))) model.add(BatchNormalization()) model.add(PReLU()) model.add(Dropout(0.5)) model.add(Dense(numNode, W_regularizer=l2(1))) model.add(BatchNormalization()) model.add(Activation('tanh')) model.add(Dropout(0.2)) model.add(Dense(output_dim=2)) model.add(Activation("softmax")) model.compile(loss='sparse_categorical_crossentropy', optimizer="adam", metrics=["accuracy"]) model.fit(np.array(TrainX), TrainY, batch_size=int(len(TrainX)*.9), nb_epoch = 1000, shuffle=True, verbose=0, validation_split=0.2) PredY = model.predict_classes(np.array(TestX), batch_size=int(len(TrainX)*.9),verbose=0,) return PredY
from keras.regularizers import l1 from keras.layers import Activation from matplotlib import pyplot # generate 2d classification dataset X, y = make_circles(n_samples=100, noise=0.1, random_state=1) # split into train and test n_train = 30 trainX, testX = X[:n_train, :], X[n_train:, :] trainy, testy = y[:n_train], y[n_train:] # define model model = Sequential() model.add( Dense(500, input_dim=2, activation='linear', activity_regularizer=l1(0.0001))) model.add(Activation('relu')) model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) # fit model history = model.fit(trainX, trainy, validation_data=(testX, testy), epochs=4000, verbose=0) # evaluate the model _, train_acc = model.evaluate(trainX, trainy, verbose=0) _, test_acc = model.evaluate(testX, testy, verbose=0) print('Train: %.3f, Test: %.3f' % (train_acc, test_acc))
X_full = np.transpose(X, (0, 2, 1)) X_train = X_train[..., np.newaxis] X_cv = X_cv[..., np.newaxis] X_test = X_test[..., np.newaxis] X_full = X_full[..., np.newaxis] model = Sequential() model.add( Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=l1(0.01), input_shape=(501, 40, 1))), model.add( Conv2D(filters=32, kernel_size=(3, 3), padding='same', kernel_regularizer=l1(0.01), activation='relu')) model.add(MaxPool2D(2, 2)) model.add( Conv2D(filters=32, kernel_size=(3, 3), padding='same',
def __init__(self, max_sequence_length, num_classes, vocab_size, embedding_size, filter_sizes, num_filters, l2_reg_lambda=0.0): # # Placeholders for input, output and dropout # ---------------------------------------------------------------------------- self.input_x = Input(shape=(max_sequence_length, ), dtype='int32') self.input_y = Input(shape=(num_classes, ), dtype='float32') self.dropout_keep_prob = Input(shape=1, dtype='float32') # Keeping track of l2 regularization loss (optional) #l2_loss = tf.constant(0.0) # # Embedding layer # ---------------------------------------------------------------------------- embedding_layer = Embedding(vocab_size, embedding_size, embeddings_initializer='uniform', input_length=max_sequence_length, trainable=False) self.embedded_chars = embedding_layer()(self.input) self.embedded_chars_expanded = K.expand_dims(self.embedded_chars, -1) # # Create a convolution + maxpool layer for each filter size # ---------------------------------------------------------------------------- pooled_outputs = [] for filter_size, num_filter in zip(filter_sizes, num_filters): # Convolution Layer conv = Conv2D(filters=num_filter, kernel_size=filter_size, padding='valid', activation="relu", strides=1)(self.embedded_chars_expanded) # Max-pooling over the outputs conv = MaxPooling2D(pool_size=2)(conv) pooled_outputs.append(conv) # # Combine all the pooled features # ---------------------------------------------------------------------------- num_filters_total = sum(num_filters) self.h_pool = concatenate(pooled_outputs, 3) self.h_pool_flat = K.reshape(self.h_pool, [-1, num_filters_total]) # # Add highway # ---------------------------------------------------------------------------- self.h_highway = highway(self.h_pool_flat, num_layers=1, activation='relu', gate_bias=0) # # Add dropout # ---------------------------------------------------------------------------- self.h_drop = Dropout(self.dropout_keep_prob)(self.h_highway) # # Final (unnormalized) scores and predictions # ---------------------------------------------------------------------------- self.scores = BatchNormalization()(self.h_drop) self.preds = Dense(num_classes, activation='softmax', kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l1(0.01))( self.h_drop) # # Train the model # ---------------------------------------------------------------------------- model = Model(inputs=self.input_x, outputs=self.preds) model.compile(loss='binary_crossentropy', optimizer='nadam', metrics=['acc'])
from keras.layers import Input, Dense from keras.callbacks import ModelCheckpoint, TensorBoard from keras import regularizers from sklearn.model_selection import train_test_split input_dim = X.shape[1] # how many neurons for compressed representation? mse_limit = 10.0 for encoding_dim in 2, : #3, 4, 5: name = 'AE%d' % encoding_dim prefix = ('SIMPLE' if simplify_space else '') + transform + '-AE%d-' % encoding_dim input_layer = Input(shape=(input_dim, )) encoder = Dense(int(input_dim / 2), activation="tanh",activity_regularizer=regularizers.l1(10e-5))(input_layer) encoder = Dense(int(encoding_dim * 1.5), activation="tanh")(encoder) encoder = Dense(encoding_dim, activation="tanh")(encoder) decoder = Dense(int(encoding_dim * 1.5), activation='tanh')(encoder) decoder = Dense(int(input_dim / 2), activation='tanh')(decoder) decoder = Dense(input_dim, activation='tanh')(decoder) autoencoder = Model(inputs=input_layer, outputs=decoder) autoencoder.summary() nb_epoch = 100 batch_size = 50 autoencoder.compile(optimizer='adam', loss='mse' ) t0 = time() history = autoencoder.fit(X, X, epochs=nb_epoch, batch_size=batch_size,
def HOSubtractNN(x_train, x_val, x_test, y_train, y_val, y_test, verbose): print('--------- Human Observed Subtract Neural Network ---------') # Convert target labels to categorical data format y_train = np_utils.to_categorical(y_train, 2) y_val = np_utils.to_categorical(y_val, 2) y_test = np_utils.to_categorical(y_test, 2) # Neural Network Configuration input_nodes = x_train.shape[1] drop_out = 0.3 layer_1_nodes = 256 layer_2_nodes = 128 layer_3_nodes = 32 output_nodes = 2 # Sequential Model of 3 hidden layers model = Sequential() model.add( Dense(layer_1_nodes, input_dim=input_nodes, kernel_regularizer=regularizers.l2(0), activity_regularizer=regularizers.l1(0), kernel_initializer=initializers.RandomUniform(seed=123))) model.add(Activation('relu')) #model.add(LeakyReLU(alpha=0.3)) model.add(Dropout(drop_out)) model.add( Dense(layer_2_nodes, input_dim=layer_1_nodes, kernel_regularizer=regularizers.l2(0), activity_regularizer=regularizers.l1(0), kernel_initializer=initializers.RandomUniform(seed=123))) model.add(Activation('relu')) #model.add(LeakyReLU(alpha=0.3)) model.add(Dropout(drop_out)) model.add( Dense(layer_3_nodes, input_dim=layer_2_nodes, kernel_regularizer=regularizers.l2(0), activity_regularizer=regularizers.l1(0), kernel_initializer=initializers.RandomUniform(seed=123))) model.add(Activation('relu')) #model.add(LeakyReLU(alpha=0.3)) model.add(Dropout(drop_out)) model.add( Dense(output_nodes, input_dim=layer_3_nodes, kernel_regularizer=regularizers.l2(0), activity_regularizer=regularizers.l1(0), kernel_initializer=initializers.RandomUniform(seed=551))) model.add(Activation('sigmoid')) model.summary() model.compile(optimizer=optimizers.Adadelta(lr=1.2, rho=0.95, epsilon=None, decay=0.0), loss='categorical_crossentropy', metrics=['accuracy']) num_epochs = 5000 model_batch_size = 128 tb_batch_size = 32 early_patience = 100 tensorboard_cb = TensorBoard(log_dir='logs', batch_size=tb_batch_size, write_graph=True) earlystopping_cb = EarlyStopping(monitor='val_loss', verbose=1, patience=early_patience, mode='min') # Train model using Training Data history = model.fit(x_train, y_train, validation_data=(x_val, y_val), epochs=num_epochs, batch_size=model_batch_size, shuffle=True, callbacks=[tensorboard_cb, earlystopping_cb], verbose=verbose) # Convert labels from categorical format to original vector format y_test = np.argmax(y_test, axis=1) # Predicted labels for Test Data pred = model.predict(x_test) pred = np.argmax(pred, axis=1) df = pd.DataFrame(history.history) print(df.tail(1)) # Accuracy and Erms for Test Data accuracy, erms = GetErms(pred, y_test) print("Test Erms: " + str(erms) + " Test Acuracy: " + str(accuracy)) # Precision and Recall for Test Data precision, recall = GetPrecisionRecall(pred, y_test) print("Precision: " + str(precision) + " Recall: " + str(recall)) # PLot graph of Loss and Accuracy over epochs df.plot() plt.savefig('HO_Subtract_NN.jpg', grid=True, figsize=(7, 8))
lamda2 = 0 lamda3 = 0 aa = 0.01 num_layers = range(1, 6) for l1 in range(1, 5): for l2 in range(1, 5): for l3 in range(1, 5): Hlayer1 = 128 * l1 Hlayer2 = 128 * l2 Hlayer3 = 128 * l3 model = Sequential([ Dense(256, input_dim=202, kernel_regularizer=regularizers.l1(lamda1), kernel_initializer='random_uniform'), Activation('relu'), Dense(Hlayer1, kernel_regularizer=regularizers.l1_l2(0), kernel_initializer='random_uniform'), Activation('relu'), Dense(Hlayer2, kernel_regularizer=regularizers.l1_l2(0), kernel_initializer='random_uniform'), Activation('relu'), Dense(Hlayer3, kernel_regularizer=regularizers.l1_l2(0), kernel_initializer='random_uniform'), Activation('relu'), Dense(51,
def mlp(data_train, target_train, drug_id=drug_id, batch_size=15, epochs=100, isSave=False, save_in_path=""): inputdim = len(data_train[0]) model = Sequential() model.add( Dense(inputdim // 2, input_dim=inputdim, init='uniform', kernel_regularizer=regularizers.l2( 0.0001))) # 输入层,28*28=784,最好附加初始化,用identity model.add(Activation('relu')) # 激活函数是tanh(后面变成了relu因为对mnist的处理结果会好一些) model.add( Dense(inputdim // 6, kernel_regularizer=regularizers.l2(0.0001), bias_regularizer=regularizers.l1(0.0001))) model.add(Activation('relu')) model.add( Dense(inputdim // 8, kernel_regularizer=regularizers.l2(0.0001), bias_regularizer=regularizers.l1(0.0001))) model.add(Activation('relu')) model.add( Dense(inputdim // 10, kernel_regularizer=regularizers.l2(0.0001), bias_regularizer=regularizers.l1(0.0001))) model.add(Activation('relu')) model.add(Dense(2)) # 输出结果和药物对应的话只是一维的 model.add(Activation('sigmoid')) # 最后一层linear,因为是实际的结果 sgd = SGD(lr=0.01) model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=["accuracy"]) if isSave: modelname = save_in_path + 'model_mlp_fordrug' + str(drug_id) + '.h5' check_test = ModelCheckpoint(modelname, monitor='loss', verbose=0, save_best_only=True, save_weights_only=False, mode='min', period=1) model.fit(data_train, target_train, batch_size=batch_size, epochs=epochs, shuffle=True, verbose=2, callbacks=[check_test]) # predicted = model.predict(data_test) # predicted = predicted.reshape(-1) # pearson = stats.pearsonr(predicted, target_test.reshape(-1)) return model else: model.fit(data_train, target_train, batch_size=batch_size, epochs=epochs, shuffle=True, verbose=2) # predicted = model.predict(data_test) # predicted = predicted.reshape(-1) # pearson = stats.pearsonr(predicted, target_test.reshape(-1)) return model
encoding_dim = 32 input_vector = Input(shape=(784, )) # regular encoded = Dense(encoding_dim, activation='relu')(input_vector) decoded = Dense(784, activation='sigmoid')(encoded) autoencoder_regular = Model(input_vector, decoded) # sparse encoded_sparse = Dense( encoding_dim, activation='relu', activity_regularizer=regularizers.l1(10e-3))(input_vector) decoded_sparse = Dense(784, activation='sigmoid')(encoded_sparse) autoencoder_sparse = Model(input_vector, decoded_sparse) # deep encoded_deep = Dense(128, activation='relu')(input_vector) encoded_deep = Dense(64, activation='relu')(encoded_deep) encoded_deep = Dense(32, activation='relu')(encoded_deep) decoded_deep = Dense(64, activation='relu')(encoded_deep) decoded_deep = Dense(128, activation='relu')(decoded_deep) decoded_deep = Dense(784, activation='sigmoid')(decoded_deep) autoencoder_deep = Model(input_vector, decoded_deep) # convolutional
# a='drop 0.8' # c='y' # if j==9: # drop=0.9 # a='drop 0.9' # c='#708090' # if j==10: # drop=1 # a='drop 1.0' # c='#c0c0c0' for j in range(9): # test l1 drop = 0.3 if j == 0: ak = regularizers.l1(1) k = regularizers.l1(1) a = 'kernel l1 1.0' c = 'b' if j == 1: ak = regularizers.l1(0.1) k = regularizers.l1(0.1) a = 'kernel l1 0.1' c = 'c' if j == 2: ak = regularizers.l1(0.01) k = regularizers.l1(0.01) a = 'kernel l1 0.01' c = 'g' if j == 3: ak = regularizers.l1(0.03)
def __init__(self, max_lenght, num_classes, vocab_size = {}, rnn='lstm', rnn_units=100, merge_type = 'concat', dropout_before_rnn=0.5, dropout_after_rnn=0.5, embedding_size = {}): start_time = time.time() print('\n\n########## CREATE A DEEPEST MODEL #########\n') print('... max_lenght: {}\n... vocab_size: {}\n... classes: {}'.format(max_lenght, vocab_size, num_classes)) self.vocab_size = vocab_size self.col_name = list(vocab_size.keys()) self.max_lenght = max_lenght input_model = [] embedding_layers = [] hidden_input = [] hidden_dropout = [] if not isinstance(embedding_size, dict): embbeding_default = embedding_size print('... embedding size dictionary is empty') print('... setting embedding size default: {}'.format(embbeding_default)) embedding_size = dict(zip(self.col_name, np.full(len(self.col_name), embbeding_default))) assert set(vocab_size) == set(embedding_size), "ERRO: embedding size is different from vocab_size" assert len(embedding_size) > 0, "embedding size was not defined" print('\n\n####### DATA TO MODEL CREATION #######\n') print('... features to input: {}'.format(self.col_name)) print('... embedding_size: {}'.format(embedding_size)) print('... max_lenght: {}'.format(max_lenght)) print('\n\n########### Building Input and Embedding Layers ###########') for c in tqdm(self.col_name): print('... creating layer to column : {}'.format(c)) print('... vocab_size to column {}: {}'.format(c, self.vocab_size[c])) i_model= Input(shape=(self.max_lenght,), name='Input_{}'.format(c)) e_output_ = Embedding(input_dim = self.vocab_size[c], output_dim = embedding_size[c], name='Embedding_{}'.format(c), input_length=self.max_lenght)(i_model) input_model.append(i_model) embedding_layers.append(e_output_) print('... defining merge type on embedding as {}'.format(merge_type)) # MERGE Layer if len(embedding_layers) == 1: hidden_input = embedding_layers[0] elif merge_type == 'add': hidden_input = Add()(embedding_layers) elif merge_type == 'avg': hidden_input = Average()(embedding_layers) else: hidden_input = Concatenate(axis=2)(embedding_layers) #Lucas fez assim #hidden_layers = Concatenate()(embedding_layers)#eu fazia assim 92,25 sparse_val_acc #DROPOUT before RNN #if self.dropout_before_rnn > 0: print('... creating a dropout Layer before RNN using {}'.format(dropout_before_rnn)) hidden_dropout = Dropout(dropout_before_rnn)(hidden_input) print('\n\n###### Creating a recurrent neural network ####\n') # Recurrent Neural Network Layer # https://www.quora.com/What-is-the-meaning-of-%E2%80%9CThe-number-of-units-in-the-LSTM-cell if rnn == 'bilstm': print('... creating a BiLSTM') rnn_cell = Bidirectional(LSTM(units=rnn_units, recurrent_regularizer=l1(0.02)))(hidden_dropout) #input_shape=(embedding_size[cy_true]*len(embedding_layers),))(hidden_dropout) else: print('... creating a LSTM') rnn_cell = LSTM(units=rnn_units, recurrent_regularizer=l1(0.02))(hidden_dropout) #input_shape=(embedding_size[c]*len(embedding_layers),))(hidden_dropout) print('... creating a dropout Layer after RNN using {}'.format(dropout_after_rnn)) rnn_dropout = Dropout(dropout_after_rnn)(rnn_cell) #https://keras.io/initializers/#randomnormal output_model = Dense(num_classes, kernel_initializer=he_uniform(), activation='softmax')(rnn_dropout) #– Encoding the labels as integers and using the sparse_categorical_crossentropy asloss function self.model = Model(inputs=input_model, outputs=output_model) print('... a deepest model was built') print('\n--------------------------------------\n') end_time = time.time() print('total Time: {}'.format(end_time - start_time))
set_session(tf.Session(config=config)) from keras.callbacks import ReduceLROnPlateau,ModelCheckpoint from keras.utils import plot_model # seed 1234 is used for reproducibility np.random.seed(seed=1234) loss = 'categorical_crossentropy' lr = 0.01 momentum = 0.9 out_dir_name = 'attention' activation = "relu" optimizer = SGD(lr=lr, momentum=momentum, decay=0.0, nesterov=True) dropout = 0.1 reg = l1(1.e-4) batch_size = 128 epochs = 600 verbose = 1 shuffle = 1 train_x_mean = 0.5909#14884877 #the mean value of cross_subject_trainset max_len = 300 feat_dim = 150 n_classes = 60 data_root = "/data2/attention_1/"
#将数据转化为数组 x_train, y_train = np.array(x_train), np.array(y_train) #因为LSTM要求输入的数据格式为三维的,[training_number, time_steps, 1],因此对数据进行相应转化 x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], features_num)) #构建网络,使用的是序贯模型 model = Sequential() #return_sequences=True返回的是全部输出,LSTM做第一层时,需要指定输入shape model.add(LSTM(units=128,activation='softsign',dropout=0.001,recurrent_dropout=0.001, return_sequences=True,input_shape=[need_num, features_num])) #activation='softsign',dropout=0.001,recurrent_dropout=0.001, stateful=False,return_sequences=True)(input) #stateful=True,可以使得帧组之间产生关联。 记得要在fit时候,shuffle=True。 # model.add(BatchNormalization()) model.add(LSTM(units=64)) # model.add(BatchNormalization()) #kernel_regularizer=regularizers.l2(0.01),activity_regularizer=regularizers.l1(0.01) model.add(Dense(64,kernel_regularizer=regularizers.l2(0.01),activity_regularizer=regularizers.l1(0.01))) model.add(Activation('relu')) model.add(Dropout(0.2)) model.add(Dense(32)) model.add(Activation('relu')) model.add(Dropout(0.2)) model.add(Dense(24)) model.add(Activation('relu')) model.add(Dropout(0.2)) model.add(Dense(12)) s_num)) #进行配置 adam=Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=True) #AdaGrad model.compile(optimizer=adam,loss='mean_squared_logarithmic_error') #mean_absolute_error/mean_squared_error/mean_squared_logarithmic_error/
model.add(Dropout(0.2)) model.add(Convolution2D(256, 3, 3, activation='relu')) #model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.2)) model.add(Flatten()) #fully connected layer model.add(Dense(600, activation='relu')) #model.add(BatchNormalization()) model.add(Dropout(0.2)) model.add(Dense(200, activation='relu')) #model.add(BatchNormalization()) model.add( Dense(7, W_regularizer=l1(0.02), activity_regularizer=regularizers.l1(0.03), activation='softmax')) # optimizer: model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) train_datagen = ImageDataGenerator(rescale=1. / 255) test_datagen = ImageDataGenerator(rescale=1. / 255) training_set = train_datagen.flow_from_directory('dataset/training', target_size=(48, 48), batch_size=128, class_mode='categorical')
def kr_train_DNN_Seq_03(self, x_dim, features_train, features_test, labels_train, labels_test, batch_size): # create model model = Sequential() model.add( Dense(5, input_dim=x_dim, init='uniform', kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l2(0.005), activation='relu')) model.add( Dense(10, init='uniform', kernel_regularizer=regularizers.l2(0.015), activity_regularizer=regularizers.l1(0.02), activation='relu')) model.add(Dense(10, init='uniform', activation='relu')) model.add(Dense(1, init='uniform', activation='relu')) # Compile model model.compile(loss='mean_squared_logarithmic_error', optimizer='adam', metrics=['accuracy']) # checkpoint filepath = "weights-improvement-{epoch:02d}-{val_accuracy:.2f}.hdf5" checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max') callbacks_list = [checkpoint] # Model Summary model.summary() # Fit the model history = model.fit(features_train, labels_train, validation_split=0.2, epochs=100, batch_size=batch_size, callbacks=callbacks_list, verbose=2) model_yaml = model.to_yaml() with open("seq03_model.yaml", "w") as yaml_file: yaml_file.write(model_yaml) # serialize weights to HDF5 model_json = model.to_json() with open("seq03_model.json", "w") as json_file: json_file.write(model_json) model.save_weights("seq03_model.h5") score = model.evaluate(features_test, labels_test, verbose=1) # round predictions accuracy = score[1] accuracy = accuracy * 100 #output_file("line_plot.html") #line_graph = figure() #line_graph.line([history.history['acc']], [history.history['val_acc']]) #show(line_graph) # Plot training & validation accuracy values #plt.plot(history.history['acc']) #plt.plot(history.history['val_acc']) #plt.title('Model accuracy') #plt.ylabel('Accuracy') #plt.xlabel('Epoch') #plt.legend(['Train', 'Test'], loc='upper left') #plt.show() # Plot training & validation loss values #plt.plot(history.history['loss']) #plt.plot(history.history['val_loss']) #plt.title('Model loss') #plt.ylabel('Loss') #plt.xlabel('Epoch') #plt.legend(['Train', 'Test'], loc='upper left') plt.show() return accuracy
def train(self, data: pd.DataFrame, percentage_train: float = .7, epochs: int = 100, n_layers: int = 2, neuron_per_layer: list = [], batch: int = 30, opt: str = "adam", act: str = "tanh"): self.encoder_layers.clear() self.decoder_layers.clear() self.data = data limit = round(len(self.data) * percentage_train) train = self.data[:limit] counter = 0 if (n_layers == 1): multi_layer = False else: multi_layer = True data_scaled = self.scaler.fit_transform(data) if (neuron_per_layer == []): n = data_scaled.shape[1] - self.reduced_dimension aux = (n / n_layers) for i in range(1, n_layers): neuron_per_layer.append(data_scaled.shape[1] - round(aux * i)) self.input = Input(shape=(data_scaled.shape[1], )) if (multi_layer): self.encoder_layers.append( Dense(neuron_per_layer[0], activation=act, activity_regularizer=regularizers.l1(10e-5))(self.input)) for i in range(1, n_layers - 1): self.encoder_layers.append( Dense(neuron_per_layer[i], activation=act)(self.encoder_layers[i - 1])) self.encoder_layers.append( Dense(self.reduced_dimension, activation=act)(self.encoder_layers[-1])) self.decoder_layers.append( Dense(neuron_per_layer[-1], activation=act, activity_regularizer=regularizers.l1(10e-5))( self.encoder_layers[-1])) for i in range(n_layers - 3, -1, -1): self.decoder_layers.append( Dense(neuron_per_layer[i], activation=act)(self.decoder_layers[counter])) counter += 1 self.decoder_layers.append( Dense(data_scaled.shape[1], activation=act)(self.decoder_layers[counter])) else: self.encoder_layers.append( Dense(self.reduced_dimension, activation=act, activity_regularizer=regularizers.l1(10e-5))(self.input)) self.decoder_layers.append( Dense(data_scaled.shape[1], activation=act, activity_regularizer=regularizers.l1(10e-5))( self.encoder_layers[0])) autoencoder = Model(self.input, self.decoder_layers[-1]) autoencoder.compile(optimizer=opt, loss='mse') X_train = data_scaled autoencoder.fit(x=X_train, y=X_train, epochs=epochs, batch_size=batch)
def res_atte( n_classes, feat_dim, max_len, dropout=dropout, kernel_regularizer=l1(1.e-4), activation="relu"): config = [ [(1,8,64)], [(1,8,64)], [(1,8,64)], [(2,8,128)], [(1,8,128)], [(1,8,128)], [(2,8,256)], [(1,8,256)], [(1,8,256)], ] initial_stride = 1 initial_filter_dim = 8 initial_num = 64 res_s = [] res_num = [2,5,8] input = Input(shape=(max_len,feat_dim)) model = input print "xxxx" print model atten_map = Dense(300,activation="tanh")(model) print "atten_map:",atten_map atten_map = Dense(150)(atten_map) print "atten_map:",atten_map atten_map = Activation("softmax")(atten_map) model = multiply([model,atten_map]) model = Conv1D(initial_num, initial_filter_dim, strides=initial_stride, padding="same", kernel_initializer="he_normal", kernel_regularizer=kernel_regularizer)(model) for depth in range(0,len(config)): res_s.append(model) for stride,filter_dim,num in config[depth]: model = BatchNormalization(axis=2)(model) model = Activation(activation)(model) model = Dropout(dropout)(model) model = Conv1D(num, filter_dim, strides=stride, padding="same", kernel_initializer="he_normal", kernel_regularizer=kernel_regularizer)(model) print model if depth in res_num: print len(res_s) #if depth == 2: #res = res_s[-3] #else: res = res_s[-3] res_shape = K.int_shape(res) model_shape = K.int_shape(model) if res_shape[2] != model_shape[2]: res = Conv1D(num, 1, strides=2, padding="same", kernel_initializer="he_normal", kernel_regularizer=kernel_regularizer)(res) model_res = add([res,model]) #print model model_shape = K.int_shape(model) atten_map = Dense(2*model_shape[-1],activation="relu")(model) print "atten_map:",atten_map atten_map = Dense(1)(atten_map) print "atten_map:",atten_map atten_map = Activation("softmax")(atten_map) model = multiply([model_res,atten_map]) pool_window_shape = K.int_shape(model) gap = AveragePooling1D(pool_window_shape[1], strides=1)(model) flatten = Flatten()(gap) dense = Dense(units=n_classes, activation="softmax", kernel_initializer="he_normal")(flatten) model = Model(inputs=input, outputs=dense) return model
def dense_model(Inputs, nclasses, l1Reg=0, dropoutRate=0.25): """ Dense matrix, defaults similar to 2016 DeepCSV training """ x = Dense(200, activation='relu', kernel_initializer='lecun_uniform', name='fc1_relu', W_regularizer=l1(l1Reg))(Inputs) x = Dropout(dropoutRate)(x) x = Dense(200, activation='relu', kernel_initializer='lecun_uniform', name='fc2_relu', W_regularizer=l1(l1Reg))(x) x = Dropout(dropoutRate)(x) x = Dense(200, activation='relu', kernel_initializer='lecun_uniform', name='fc3_relu', W_regularizer=l1(l1Reg))(x) x = Dropout(dropoutRate)(x) x = Dense(200, activation='relu', kernel_initializer='lecun_uniform', name='fc4_relu', W_regularizer=l1(l1Reg))(x) x = Dropout(dropoutRate)(x) x = Dense(200, activation='relu', kernel_initializer='lecun_uniform', name='fc5_relu', W_regularizer=l1(l1Reg))(x) predictions = Dense(nclasses, activation='softmax', kernel_initializer='lecun_uniform', name = 'output_softmax')(x) model = Model(inputs=Inputs, outputs=predictions) return model
sumXval = sumXval + XtestList[i].toarray() Y_val = np_utils.to_categorical(Y_test - 1, 1000) tensorboard = TensorBoard(log_dir='./exp5', histogram_freq=1, write_graph=True, write_images=False) nb_hidden = 1000 input1 = Input(shape=(1000, )) input2 = Input(shape=(1000, )) input3 = Input(shape=(1000, )) concatL = Concatenate()([input1, input2, input3]) layer1 = Dense(nb_hidden, bias_initializer='zeros', kernel_regularizer=regularizers.l1(0.00002))(concatL) layer1 = LeakyReLU(alpha=0.01)(layer1) out = Dense(nb_classes, bias_initializer='zeros', kernel_regularizer=regularizers.l1(0.00002), kernel_initializer='identity', activation='softmax')(layer1) model = Model([input1, input2, input3], out) #model=load_model('genFit_ens450.h5'); model.compile(loss=nick_crossentropy, optimizer='adadelta', metrics=['accuracy', top3_acc, top5_acc]) G = myGeneratorTrain() K = next(G) X_i = K[0]
# Freeze all the layers for layer in vgg_conv.layers[:]: layer.trainable = False # Check the trainable status of the individual layers for layer in vgg_conv.layers: print(layer, layer.trainable) # Create the model model = Sequential() # Add the vgg convolutional base model model.add(vgg_conv) # Add new layers model.add(Flatten()) model.add(Dense(1024, activation='relu',kernel_regularizer=l1(0.001), bias_regularizer=l1(0))) model.add(Dropout(0.2)) model.add(Dense(256, activation='relu',kernel_regularizer=l1(0.001), bias_regularizer=l1(0))) model.add(Dense(8, activation='softmax',kernel_regularizer=l1(0.001), bias_regularizer=l1(0))) # Show a summary of the model. Check the number of trainable parameters model.summary() # Load the normalized images train_datagen = ImageDataGenerator(rescale=1./255) validation_datagen = ImageDataGenerator(rescale=1./255) traindf=pd.read_csv('/content/drive/My Drive/Colab Notebooks/dataset/train.csv',dtype=str) training_datagen = ImageDataGenerator( rescale = 1./255, rotation_range=40,
from keras.callbacks import TensorBoard, ReduceLROnPlateau, ModelCheckpoint from keras import regularizers from keras.optimizers import Adam # 指定使用的GPU os.environ["CUDA_VISIBLE_DEVICES"] = "8" file_name = os.path.join(root_dir, 'model_data', 'model.h5') # 超参数 num_classes = 2 batch_size = 64 epochs = 30 dropout_rate = 0.25 reg = regularizers.l1(1e-4) test_split = 0.2 lr = 1e-4 # 数据增强超参数 horizontal_flip = True vertical_flip = True rotation_angle = 180 width_shift_range = 0.1 height_shift_range = 0.1 def build_model(): base_model = InceptionV3(weights='imagenet', include_top=False) x = base_model.output
class KerasRegularizersTest(test_combinations.TestCase, parameterized.TestCase): def create_model(self, kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None): model = keras.models.Sequential() model.add( keras.layers.Dense(NUM_CLASSES, kernel_regularizer=kernel_regularizer, bias_regularizer=bias_regularizer, activity_regularizer=activity_regularizer, input_shape=(DATA_DIM, ))) return model def regularizer_fn_tensor(x): return tf.constant(0.) def regularizer_fn_scalar(x): return 0. class RegularizerTensor(regularizers.Regularizer): def __call__(self, x): return tf.constant(0.) class RegularizerScalar(regularizers.Regularizer): def __call__(self, x): return 0. def get_data(self): (x_train, y_train), (x_test, y_test) = test_utils.get_test_data( train_samples=10, test_samples=10, input_shape=(DATA_DIM, ), num_classes=NUM_CLASSES) y_train = np_utils.to_categorical(y_train, NUM_CLASSES) y_test = np_utils.to_categorical(y_test, NUM_CLASSES) return (x_train, y_train), (x_test, y_test) def create_multi_input_model_from(self, layer1, layer2): input_1 = keras.layers.Input(shape=(DATA_DIM, )) input_2 = keras.layers.Input(shape=(DATA_DIM, )) out1 = layer1(input_1) out2 = layer2(input_2) out = keras.layers.Average()([out1, out2]) model = keras.models.Model([input_1, input_2], out) model.add_loss(keras.backend.mean(out2)) model.add_loss(tf.reduce_sum(input_1)) return model @test_combinations.run_all_keras_modes @parameterized.named_parameters([ ('l1', regularizers.l1()), ('l2', regularizers.l2()), ('l1_l2', regularizers.l1_l2()), ('l2_zero', keras.regularizers.l2(0.)), ('function_tensor', regularizer_fn_tensor), ('function_scalar', regularizer_fn_scalar), ('lambda_tensor', lambda x: tf.constant(0.)), ('lambda_scalar', lambda x: 0.), ('regularizer_base_class', regularizers.Regularizer()), ('regularizer_custom_class_tensor', RegularizerTensor()), ('regularizer_custom_class_scalar', RegularizerScalar()), ]) def test_kernel_regularization(self, regularizer): (x_train, y_train), _ = self.get_data() model = self.create_model(kernel_regularizer=regularizer) model.compile(loss='categorical_crossentropy', optimizer='sgd', run_eagerly=test_utils.should_run_eagerly()) self.assertEqual(len(model.losses), 1) model.fit(x_train, y_train, batch_size=10, epochs=1, verbose=0) @test_combinations.run_all_keras_modes @parameterized.named_parameters([ ('l1', regularizers.l1()), ('l2', regularizers.l2()), ('l1_l2', regularizers.l1_l2()), ('l2_zero', keras.regularizers.l2(0.)), ('function_tensor', regularizer_fn_tensor), ('function_scalar', regularizer_fn_scalar), ('lambda_tensor', lambda x: tf.constant(0.)), ('lambda_scalar', lambda x: 0.), ('regularizer_base_class', regularizers.Regularizer()), ('regularizer_custom_class_tensor', RegularizerTensor()), ('regularizer_custom_class_scalar', RegularizerScalar()), ]) def test_bias_regularization(self, regularizer): (x_train, y_train), _ = self.get_data() model = self.create_model(bias_regularizer=regularizer) model.compile(loss='categorical_crossentropy', optimizer='sgd', run_eagerly=test_utils.should_run_eagerly()) self.assertEqual(len(model.losses), 1) model.fit(x_train, y_train, batch_size=10, epochs=1, verbose=0) @test_combinations.run_all_keras_modes @parameterized.named_parameters([ ('l1', regularizers.l1()), ('l2', regularizers.l2()), ('l1_l2', regularizers.l1_l2()), ('l2_zero', keras.regularizers.l2(0.)), ('function_tensor', regularizer_fn_tensor), ('function_scalar', regularizer_fn_scalar), ('lambda_tensor', lambda x: tf.constant(0.)), ('lambda_scalar', lambda x: 0.), ('regularizer_base_class', regularizers.Regularizer()), ('regularizer_custom_class_tensor', RegularizerTensor()), ('regularizer_custom_class_scalar', RegularizerScalar()), ]) def test_activity_regularization(self, regularizer): (x_train, y_train), _ = self.get_data() model = self.create_model(activity_regularizer=regularizer) model.compile(loss='categorical_crossentropy', optimizer='sgd', run_eagerly=test_utils.should_run_eagerly()) self.assertEqual(len(model.losses), 1 if tf.executing_eagerly() else 1) model.fit(x_train, y_train, batch_size=10, epochs=1, verbose=0) @test_combinations.run_all_keras_modes @test_combinations.run_with_all_model_types def test_zero_regularization(self): # Verifies that training with zero regularization works. x, y = np.ones((10, 10)), np.ones((10, 3)) model = test_utils.get_model_from_layers([ keras.layers.Dense(3, kernel_regularizer=keras.regularizers.l2(0)) ], input_shape=(10, )) model.compile('sgd', 'mse', run_eagerly=test_utils.should_run_eagerly()) model.fit(x, y, batch_size=5, epochs=1) def test_custom_regularizer_saving(self): def my_regularizer(weights): return tf.reduce_sum(tf.abs(weights)) inputs = keras.Input((10, )) outputs = keras.layers.Dense(1, kernel_regularizer=my_regularizer)(inputs) model = keras.Model(inputs, outputs) model2 = model.from_config( model.get_config(), custom_objects={'my_regularizer': my_regularizer}) self.assertEqual(model2.layers[1].kernel_regularizer, my_regularizer) @test_combinations.run_all_keras_modes @parameterized.named_parameters([ ('l1', regularizers.l1()), ('l2', regularizers.l2()), ('l1_l2', regularizers.l1_l2()), ]) def test_regularization_shared_layer(self, regularizer): dense_layer = keras.layers.Dense(NUM_CLASSES, kernel_regularizer=regularizer, activity_regularizer=regularizer) model = self.create_multi_input_model_from(dense_layer, dense_layer) model.compile(loss='categorical_crossentropy', optimizer='sgd', run_eagerly=test_utils.should_run_eagerly()) self.assertLen(model.losses, 5) @test_combinations.run_all_keras_modes @parameterized.named_parameters([ ('l1', regularizers.l1()), ('l2', regularizers.l2()), ('l1_l2', regularizers.l1_l2()), ]) def test_regularization_shared_model(self, regularizer): dense_layer = keras.layers.Dense(NUM_CLASSES, kernel_regularizer=regularizer, activity_regularizer=regularizer) input_tensor = keras.layers.Input(shape=(DATA_DIM, )) dummy_model = keras.models.Model(input_tensor, dense_layer(input_tensor)) model = self.create_multi_input_model_from(dummy_model, dummy_model) model.compile(loss='categorical_crossentropy', optimizer='sgd', run_eagerly=test_utils.should_run_eagerly()) self.assertLen(model.losses, 6) @test_combinations.run_all_keras_modes @parameterized.named_parameters([ ('l1', regularizers.l1()), ('l2', regularizers.l2()), ('l1_l2', regularizers.l1_l2()), ]) def test_regularization_shared_layer_in_different_models( self, regularizer): shared_dense = keras.layers.Dense(NUM_CLASSES, kernel_regularizer=regularizer, activity_regularizer=regularizer) models = [] for _ in range(2): input_tensor = keras.layers.Input(shape=(DATA_DIM, )) unshared_dense = keras.layers.Dense(NUM_CLASSES, kernel_regularizer=regularizer) out = unshared_dense(shared_dense(input_tensor)) models.append(keras.models.Model(input_tensor, out)) model = self.create_multi_input_model_from(layer1=models[0], layer2=models[1]) model.compile(loss='categorical_crossentropy', optimizer='sgd', run_eagerly=test_utils.should_run_eagerly()) # We expect to see 9 losses on the model: # - 2 from the 2 add_loss calls on the outer model. # - 3 from the weight regularizers on the shared_dense layer, unshared_dense # in inner model 1, unshared_dense in inner model 2. # - 4 from activity regularizers on the shared_dense layer. self.assertLen(model.losses, 9) def test_deserialization_error(self): with self.assertRaisesRegex(ValueError, 'Could not interpret regularizer'): keras.regularizers.get(0) @parameterized.named_parameters([ ('l1', regularizers.l1(l1=None), 0.01), ('l2', regularizers.l2(l2=None), 0.01), ('l1_l2', regularizers.l1_l2(l1=None, l2=None), 0.), ]) def test_default_value_when_init_with_none(self, regularizer, expected_value): expected_value = np.asarray(expected_value) if hasattr(regularizer, 'l1'): self.assertAllClose(regularizer.l1, expected_value) if hasattr(regularizer, 'l2'): self.assertAllClose(regularizer.l2, expected_value) @test_utils.run_v2_only def test_orthogonal_regularizer(self): # Test correctness. factor = 0.1 reg_rows = regularizers.OrthogonalRegularizer(factor=factor, mode='rows') reg_cols = regularizers.OrthogonalRegularizer(factor=factor, mode='columns') # Test with square matrix inputs = tf.constant([[1, 1, 1, 1], [2, 0, 0, 0], [0, 0, 3, 1]], dtype='float32') normalized_rows = tf.math.l2_normalize(inputs, axis=1) normalized_cols = tf.math.l2_normalize(inputs, axis=0) rows_pairs = [ tf.reduce_sum(normalized_rows[0] * normalized_rows[1]), tf.reduce_sum(normalized_rows[0] * normalized_rows[2]), tf.reduce_sum(normalized_rows[1] * normalized_rows[2]), ] col_pairs = [ tf.reduce_sum(normalized_cols[:, 0] * normalized_cols[:, 1]), tf.reduce_sum(normalized_cols[:, 0] * normalized_cols[:, 2]), tf.reduce_sum(normalized_cols[:, 0] * normalized_cols[:, 3]), tf.reduce_sum(normalized_cols[:, 1] * normalized_cols[:, 2]), tf.reduce_sum(normalized_cols[:, 1] * normalized_cols[:, 3]), tf.reduce_sum(normalized_cols[:, 2] * normalized_cols[:, 3]), ] num_row_pairs = 3 num_col_pairs = 6 # Expected: factor * sum(pairwise_dot_products_of_rows) / num_row_pairs self.assertAllClose(reg_rows(inputs), factor * sum(rows_pairs) / num_row_pairs) # Expected: factor * sum(pairwise_dot_products_of_columns) / num_col_pairs self.assertAllClose(reg_cols(inputs), factor * sum(col_pairs) / num_col_pairs) # Test incorrect usage. with self.assertRaisesRegex(ValueError, 'must have rank 2'): reg_rows(tf.constant([1, 1], dtype='float32')) # Test serialization self.assertDictEqual(reg_cols.get_config(), { 'factor': factor, 'mode': 'columns' }) # Test usage in model. model_inputs = keras.Input((3, )) model_outputs = keras.layers.Dense( 4, kernel_regularizer=reg_rows)(model_inputs) model = keras.Model(model_inputs, model_outputs) model.compile(optimizer='rmsprop', loss='mse') model.fit(np.random.random((16, 3)), np.random.random((16, 4)), epochs=1) # Test serialization and deserialiation as part of model. inputs = tf.constant([[1, 1, 1], [2, 0, 0], [0, 0, 3]], dtype='float32') outputs = model(inputs) config = model.get_config() weights = model.get_weights() model = keras.Model.from_config(config) model.set_weights(weights) self.assertAllClose(model(inputs), outputs, atol=1e-5)
def _compare_models(architecture, loss, weight_decay=0.0, number_of_samples=3, learning_rate=0.01, epochs=1): np.random.seed(42) x = np.random.rand(number_of_samples, architecture[0]['input']) if 'MSE' == loss: y = np.random.rand(number_of_samples, 1) keras_loss = 'MSE' keras_metrics = None elif 'cross-entropy' == loss: number_of_classes = architecture[-1]['output'] y = np.eye(number_of_classes)[np.random.choice(number_of_classes, size=number_of_samples)] keras_loss = losses.categorical_crossentropy keras_metrics = [metrics.categorical_accuracy] else: assert False, loss actual = mydnn(architecture, loss, weight_decay) sgd = optimizers.SGD(lr=learning_rate) desired = Sequential() for index, layer in enumerate(architecture): units = layer['output'] activation = layer['nonlinear'] if 'sot-max' == activation: activation = 'softmax' def kernel_initializer(shape, dtype=None): w = actual._architecture[index]._w.get_value() assert w.shape == shape return w def bias_initializer(shape, dtype=None): b = actual._architecture[index]._b.get_value() assert b.shape == shape return b if 'l1' == layer['regularization']: kernel_regularizer = regularizers.l1(weight_decay) elif 'l2' == layer['regularization']: kernel_regularizer = regularizers.l2(weight_decay) else: assert False desired.add(layers.Dense(units, activation=activation, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, kernel_regularizer=kernel_regularizer, input_shape=(layer['input'],))) desired.compile(sgd, keras_loss, metrics=keras_metrics) desired.fit(x, y, batch_size=x.shape[0], epochs=1) actual.fit(x, y, epochs, x.shape[0], learning_rate) weights_and_biases_desired = desired.get_weights() weights_and_biases_actual = list() for index in range(len(architecture)): weights_and_biases_actual.append(actual._architecture[index]._w.get_value()) weights_and_biases_actual.append(actual._architecture[index]._b.get_value()) for weight_actual, weight_desired in zip(weights_and_biases_actual, weights_and_biases_desired): np.testing.assert_allclose(weight_actual, weight_desired, atol=1e-5) y_predicted_actual = actual.predict(x, batch_size=number_of_samples) y_predicted_desired = desired.predict(x, batch_size=number_of_samples) np.testing.assert_allclose(y_predicted_actual, y_predicted_desired) if 'MSE' == loss: loss_actual = actual.evaluate(x, y, batch_size=number_of_samples) loss_desired = desired.evaluate(x, y, batch_size=number_of_samples) np.testing.assert_allclose(loss_actual, loss_desired) else: loss_actual, accuracy_actual = actual.evaluate(x, y, batch_size=number_of_samples) loss_desired, accuracy_desired = desired.evaluate(x, y, batch_size=number_of_samples) np.testing.assert_allclose(loss_actual, loss_desired) np.testing.assert_allclose(accuracy_actual, accuracy_desired)
def build_model(self): """Build a critic (value) network that maps (state, action) pairs -> Q-values.""" # Define input layers """ 它在某些方面比行动者模型简单,但是需要注意几点。首先,行动者模型旨在将状态映射到动作, 而评论者模型需要将(状态、动作)对映射到它们的 Q 值。这一点体现在了输入层中。""" states = layers.Input(shape=(self.state_size, ), name='states') actions = layers.Input(shape=(self.action_size, ), name='actions') # Add hidden layer(s) for state pathway """It took me so, use_bias=True, bias_constraint=regularizers.l2(0.02)""" net_states = states net_states = layers.Dense( units=32, kernel_regularizer=regularizers.l2(0.001), activity_regularizer=regularizers.l1(0.001))(net_states) net_states = layers.LeakyReLU(alpha=0.3)(net_states) net_states = layers.BatchNormalization()(net_states) net_states = layers.Dropout(0.4)(net_states) net_states = layers.Dense( units=64, kernel_regularizer=regularizers.l2(0.001), activity_regularizer=regularizers.l1(0.001))(net_states) net_states = layers.LeakyReLU(alpha=0.3)(net_states) net_states = layers.BatchNormalization()(net_states) net_states = layers.Dropout(0.4)(net_states) net_actions = actions net_actions = layers.Dense( units=32, kernel_regularizer=regularizers.l2(0.001), activity_regularizer=regularizers.l1(0.001))(net_actions) net_actions = layers.LeakyReLU(alpha=0.3)(net_actions) net_actions = layers.BatchNormalization()(net_actions) net_actions = layers.Dropout(0.4)(net_actions) net_actions = layers.Dense( units=64, kernel_regularizer=regularizers.l2(0.001), activity_regularizer=regularizers.l1(0.001))(net_actions) net_actions = layers.LeakyReLU(alpha=0.3)(net_actions) net_actions = layers.BatchNormalization()(net_actions) net_actions = layers.Dropout(0.4)(net_actions) # Try different layer sizes, activations, add batch normalization, regularizers, etc. # Combine state and action pathways """这两个层级首先可以通过单独的“路径”(迷你子网络)处理,但是最终需要结合到一起 例如,可以通过使用 Keras 中的 Add 层级类型来实现请参阅合并层级):""" net = layers.Add()([net_states, net_actions]) net = layers.Dense(units=256, kernel_regularizer=regularizers.l2(0.001), activity_regularizer=regularizers.l1(0.001))(net) net = layers.LeakyReLU(alpha=0.3)(net) net = layers.BatchNormalization()(net) net = layers.Dropout(0.4)(net) net = layers.Dense(units=16, kernel_regularizer=regularizers.l2(0.001), activity_regularizer=regularizers.l1(0.001))(net) net = layers.LeakyReLU(alpha=0.3)(net) # Add more layers to the combined network if needed # Add final output layer to prduce action values (Q values) Q_values = layers.Dense(units=1, name='q_values')(net) # Create Keras model self.model = models.Model(inputs=[states, actions], outputs=Q_values) # Define optimizer and compile model for training with built-in loss function optimizer = optimizers.Adam(lr=1e-4) self.model.compile(optimizer=optimizer, loss='mse') # Compute action gradients (derivative of Q values w.r.t. to actions) """该模型的最终输出是任何给定(状态、动作)对的 Q 值。但是,我们还需要计算此 Q 值相对于相应动作向量的梯度,以用于训练行动者模型。 这一步需要明确执行,并且需要定义一个单独的函数来访问这些梯度:""" action_gradients = K.gradients(Q_values, actions) # Define an additional function to fetch action gradients (to be used by actor model) self.get_action_gradients = K.function( inputs=[*self.model.input, K.learning_phase()], outputs=action_gradients)
# ## Set up a neural network with relu-encoder # In[72]: nb_epoch = 100 batch_size = 128 input_dim = df_train_0_x_rescaled.shape[1] #num of predictor variables, encoding_dim = 16 hidden_dim = int(encoding_dim / 2) learning_rate = 1e-1 input_layer = Input(shape=(input_dim, )) encoder = Dense( encoding_dim, activation="tanh", activity_regularizer=regularizers.l1(learning_rate))(input_layer) encoder = Dense(hidden_dim, activation="relu")(encoder) decoder = Dense(hidden_dim, activation='tanh')(encoder) decoder = Dense(input_dim, activation='relu')(decoder) autoencoder = Model(inputs=input_layer, outputs=decoder) # This requires pydot and graphviz installed # In[73]: from IPython.display import SVG from keras.utils.vis_utils import model_to_dot SVG( model_to_dot(autoencoder, show_shapes=True, show_layer_names=True).create(prog='dot', format='svg'))
def create_model(x_train, y_train, x_val, y_val, x_test, y_test): if sys.argv[1] == 'german': input_n = 24 elif sys.argv[1] == 'australian': input_n = 15 batch_size = 32 epochs = 500 inits = [ 'Zeros', 'Ones', 'RandomNormal', 'RandomUniform', 'TruncatedNormal', 'Orthogonal', 'lecun_uniform', 'lecun_normal', 'he_uniform', 'he_normal', 'glorot_uniform', 'glorot_normal' ] acts = [ 'tanh', 'softsign', 'sigmoid', 'hard_sigmoid', 'relu', 'softplus', 'LeakyReLU', 'PReLU', 'elu', 'selu' ] init = inits[int({{quniform(0, 11, 1)}})] act = acts[1] neurons = int({{quniform(9, 180, 9)}}) layers = {{choice([1, 2, 4, 8])}} norm = {{choice(['no', 'l1', 'l2'])}} dropout = {{choice([0, 1])}} earlystop = {{choice([0, 1])}} k1 = None k2 = None p = None if norm == 'no': reg = None elif norm == 'l1': k1 = {{loguniform(-9.2, -2.3)}} reg = regularizers.l1(k1) elif norm == 'l2': k2 = {{loguniform(-9.2, -2.3)}} reg = regularizers.l2(k2) X_input = Input((input_n, )) X = X_input for _ in range(layers): X = Dense( neurons, kernel_initializer=init, kernel_regularizer=reg, )(X) if act == 'LeakyReLU': X = LeakyReLU()(X) elif act == 'PReLU': X = PReLU()(X) else: X = Activation(act)(X) if dropout == 1: p = {{uniform(0, 1)}} X = Dropout(p)(X) X = Dense(1, kernel_initializer=init, kernel_regularizer=reg)(X) X_outputs = Activation('sigmoid')(X) model = Model(inputs=X_input, outputs=X_outputs) model.compile( loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'], ) patience = int({{quniform(1, 500, 1)}}) es = EarlyStopping( monitor='val_loss', patience=patience, verbose=0, mode='auto', ) if earlystop == 1: model.fit( x_train, y_train, batch_size=batch_size, verbose=0, epochs=epochs, validation_data=(x_val, y_val), callbacks=[es], ) else: model.fit( x_train, y_train, batch_size=batch_size, verbose=0, epochs=epochs, validation_data=(x_val, y_val), ) loss_t, score_t = model.evaluate(x_train, y_train, verbose=0) loss_v, score_v = model.evaluate(x_val, y_val, verbose=0) loss_te, score_te = model.evaluate(x_test, y_test, verbose=0) print(init + '\t' + act + '\t' + str(neurons) + '\t' + str(layers) + '\t' + str(norm) + '\t' + str(dropout) + '\t' + str(earlystop) + '%-24s%-24s%-24s%s' % (str(k1), str(k2), str(p), str(patience)) + ' ' + str(score_v) + ' ' + str(loss_v) + ' ' + str(score_te) + ' ' + str(loss_te)) return {'loss': loss_v, 'status': STATUS_OK, 'model': model}
def i3d_flattened(self, num_classes = 60): i3d = Model(inputs = self.model.input, outputs = self.model.get_layer(index=-4).output) x = conv3d_bn(i3d.output, num_classes, 1, 1, 1, padding='same', use_bias=True, use_activation_fn=False, use_bn=False, name='Conv3d_6a_1x1') num_frames_remaining = int(x.shape[1]) x = Flatten()(x) predictions = Dense(num_classes, activation = 'softmax', kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l1(0.01))(x) new_model = Model(inputs = i3d.input, outputs = predictions) #for layer in i3d.layers: # layer.trainable = False return new_model
from keras.layers import BatchNormalization from keras.layers import GaussianNoise, GaussianDropout, AlphaDropout from keras.layers import Input from keras.layers import TimeDistributed, Bidirectional from keras import regularizers fillerMap = { 'constant': 'Constant', 'uniform': 'RandomUniform', 'gaussian': 'RandomNormal', 'xavier': 'glorot_normal', 'msra': 'he_normal' } regularizerMap = { 'l1': regularizers.l1(), 'l2': regularizers.l2(), 'l1_l2': regularizers.l1_l2(), 'L1L2': regularizers.l1_l2(), 'None': None } constraintMap = { 'max_norm': 'max_norm', 'non_neg': 'non_neg', 'unit_norm': 'unit_norm', 'MaxNorm': 'max_norm', 'NonNeg': 'non_neg', 'UnitNorm': 'unit_norm', 'None': None }
def train(self, dataset, testset, tagger, epochs=3): """ @param dataset : a list of dependency trees """ N = len(dataset) sequences = [(dtree.tokens, self.oracle_derivation(dtree)) for dtree in dataset] X_S, X_B, Y = [], [], [] def map_tokens(S, B, tokens): S = [ tagger.x_codes[tokens[s][0]] if tokens[s][0] in tagger.x_codes else tagger.x_codes["__UNK__"] for s in S ] B = [ tagger.x_codes[tokens[b][0]] if tokens[b][0] in tagger.x_codes else tagger.x_codes["__UNK__"] for b in B ] return S, B ##### DARK SIDE ##### for tokens, ref_derivation in sequences: current_config = ref_derivation[0][1] for action, config in ref_derivation[ 1:]: #do not learn the "None" dummy action S, B, _, _ = current_config current_config = config S, B = map_tokens(S, B, tokens) X_S.append(S) X_B.append(B) Y.append(action) XS_encoded, XB_encoded = pad_sequences( X_S, maxlen=tagger.mL), pad_sequences(X_B, maxlen=tagger.mL) self.x_dict = tagger.x_codes self.mL = tagger.mL self.y_size = len(set(Y)) self.y_list = list(set(Y)) self.y_dict = {y: i for i, y in enumerate(self.y_list)} self.reverse_y_dict = dict(enumerate(self.y_list)) Y_encoded = np.zeros(shape=(len(Y), self.y_size)) for i, y in enumerate(Y): Y_encoded[i, self.y_dict[y]] = 1. ###$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$### # print(XS_encoded) tXS, tXB, tY = self.prep(testset) # exit() ipt_stack = Input(shape=(tagger.mL, )) ipt_stack_pos = Input(shape=(self.yML, )) ipt_buffer = Input(shape=(tagger.mL, )) e_stack = tagger.model.get_layer("embedding_1")(ipt_stack) e_buffer = tagger.model.get_layer("embedding_1")(ipt_buffer) l_s = tagger.model.get_layer("bidirectional_1")(e_stack) l_b = tagger.model.get_layer("bidirectional_1")(e_buffer) l1 = LSTM(122, return_sequences=True) l1 = concatenate([l1(l_s), l1(l_b)], axis=1) # l2 = Flatten()) l3 = LSTM(122)(l1) o = Dense(self.y_size, activation="softmax")(l3) self.nn_parser = Model([ipt_stack, ipt_buffer], o) self.nn_parser.summary() sgd = RMSprop(lr=.01) self.nn_parser.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy']) self.nn_parser.fit([XS_encoded, XB_encoded], Y_encoded, epochs=epochs, verbose=1, validation_split=.2) print(self.nn_parser.evaluate([tXS, tXB], tY, batch_size=64)) return self
test_size=0.2, random_state=42) X_train = X_train.values X_test = X_test.values X_train.shape input_dim = X_train.shape[1] encoding_dim = 162 input_layer = Input(shape=(input_dim, )) encoder = Dense(encoding_dim, activation="tanh", activity_regularizer=regularizers.l1(10e-5))(input_layer) encoder = Dense(int(encoding_dim / 2), activation="relu")(encoder) decoder = Dense(int(encoding_dim / 2), activation='tanh')(encoder) decoder = Dense(input_dim, activation='relu')(decoder) autoencoder = Model(inputs=input_layer, outputs=decoder) nb_epoch = 30 batch_size = 32 autoencoder.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy']) checkpointer = ModelCheckpoint(filepath="model.h5",
ty_track = np.zeros((dmp.timesteps, dmp.n_dmps)) tdy_track = np.zeros((dmp.timesteps, dmp.n_dmps)) tddy_track = np.zeros((dmp.timesteps, dmp.n_dmps)) goals = [[-1, 0] for i in range(10)]#[[np.cos(theta), np.sin(theta)] for theta in np.linspace(0, 2*np.pi, 20)[:-1]] for goal in goals: dmp.goal = goal dmp.reset_state() for t in range(dmp.timesteps): ty_track[t], tdy_track[t], tddy_track[t] = dmp.step(external_force=avoid_obstacles(dmp.y, dmp.dy, goal, t)) model = Sequential() model.add(Dense(10, activation='relu', bias_regularizer=regularizers.l1(0.01), input_dim=3)) model.add(Dense(2, activation='tanh')) inpt = np.vstack((obst_vec[:,0], obst_vec[:,1], cur_vel)) inpt = np.swapaxes(inpt, 0, 1) inpt = preprocessing.scale(inpt) model.compile(optimizer='adagrad', loss='mse', metrics=[metrics.mse]) label = c_t