def conv_to_rnn(inputs, n_out, *args, l2_reg=0.01):
sigma = 0.05
"""Convolution on each stimulus, then pass sequence to an RNN"""
# Applies this conv layer to each stimulus in the sequence individually
y = TimeDistributed(Conv2D(8,
15,
data_format="channels_first",
kernel_regularizer=l2(1e-3)),
input_shape=(40, 1, 50, 50))(inputs)
y = Activation('relu')(GaussianNoise(sigma)(y))
y = TimeDistributed(
Conv2D(8,
11,
data_format="channels_first",
kernel_regularizer=l2(1e-3)))(y)
y = Activation('relu')(GaussianNoise(sigma)(y))
# Flatten feature maps to pass to LSTM
y = TimeDistributed(Flatten())(y)
y = SimpleRNN(50,
activation='relu',
kernel_initializer='random_normal',
recurrent_initializer='random_normal')(y)
y = Dense(n_out, init='normal')(y)
outputs = Activation('softplus')(y)
return Model(inputs, outputs, name="CONV_TO_RNN")
def get_seq2seq_model_one_hot(input_size, output_size, MAX_LEN_OUTPUT):
seq2seq_model = Sequential()
seq2seq_model.add(GaussianNoise(
0.15,
input_shape=(None, input_size))) # El modelo original no tenia GN
seq2seq_model.add(
BatchNormalization()) # El modelo original no tenia BN
seq2seq_model.add(
LSTM(750, return_sequences=False, activation="relu")
) # max_word_index+2 for one_hot, dim_embeddings for embedding
seq2seq_model.add(RepeatVector(MAX_LEN_OUTPUT))
seq2seq_model.add(GaussianNoise(0.15))
seq2seq_model.add(
BatchNormalization()) # El modelo original no tenia BN
seq2seq_model.add(LSTM(950, return_sequences=True, activation="relu"))
seq2seq_model.add(
BatchNormalization()) # El modelo original no tenia BN
seq2seq_model.add(
TimeDistributed(Dense(output_size, activation="softmax"))
) # max_word_index+2 for one_hot, dim_embeddings for embedding
seq2seq_model.compile(optimizer='adadelta',
sample_weight_mode="temporal",
loss='categorical_crossentropy',
metrics=['accuracy'])
return seq2seq_model
def encode(x, use_noise, relu_max):
print 'encoder input shape:', x._keras_shape
assert x._keras_shape[1:] == (28, 28, 1)
# 28, 28, 1
y = Conv2D(20, 5, 5, activation='relu', border_mode='same', subsample=(2,2))(x)
y = BN(mode=2, axis=3)(y)
# 14, 14, 20
y = Conv2D(40, 3, 3, activation='relu', border_mode='same', subsample=(2,2))(y)
y = BN(mode=2, axis=3)(y)
# 7, 7, 40
print 'pre_fc shape:', y._keras_shape
latent_dim = 80
y = Conv2D(latent_dim, 7, 7, activation='linear',
border_mode='same', subsample=(7,7))(y)
# 1, 1, latent_dim
if use_noise and not relu_max:
print 'add noise and pretend relu_max will be:', RELU_MAX
y = GaussianNoise(0.2 * RELU_MAX)(y)
y = Activation(utils.relu_n(relu_max))(y)
if relu_max:
print 'relu max:', relu_max
y = Activation(utils.scale_down(relu_max))(y)
# y in [0, 1]
if use_noise:
y = GaussianNoise(0.2)(y)
y = Activation('relu')(y)
y = Reshape((latent_dim,))(y)
# 80
return y
def discriminator_google_mnistM(img_dim,wd):
disc_input = Input(shape=img_dim, name="discriminator_input")
x = Conv2D(64, (3, 3), strides=(1, 1), name="conv1",border_mode="same",weight_norm=False, kernel_initializer=RandomNormal(stddev=0.02),kernel_regularizer=l2(wd))(disc_input)
x=BatchNormGAN(axis=1)(x)
x = Dropout(0.1)(x)
x = LeakyReLU(0.2)(x)
x = GaussianNoise( sigma=0.2 )(x)
x = Conv2D(128, (3, 3), strides=(2, 2), name="conv2",border_mode="same",weight_norm=False, kernel_initializer=RandomNormal(stddev=0.02),kernel_regularizer=l2(wd))(x)
x = Dropout(0.2)(x)
# x = LeakyReLU(0.2)(x)
x = GaussianNoise( sigma=0.2 )(x)
x = Conv2D(256, (3, 3), strides=(2, 2), name="conv3",border_mode="same",weight_norm=False, kernel_initializer=RandomNormal(stddev=0.02),kernel_regularizer=l2(wd))(x)
x=BatchNormGAN(axis=1)(x)
x = Dropout(0.2)(x)
x = LeakyReLU(0.2)(x)
x = GaussianNoise( sigma=0.2 )(x)
x = Conv2D(512, (3, 3), strides=(2, 2), name="conv4",border_mode="same",weight_norm=False, kernel_initializer=RandomNormal(stddev=0.02),kernel_regularizer=l2(wd))(x)
x = Dropout(0.2)(x)
# x = LeakyReLU(0.2)(x)
x = GaussianNoise( sigma=0.2 )(x)
x = Flatten()(x)
x = Dense(1, init=RandomNormal(stddev=0.02),activation='sigmoid', name='fc',W_regularizer=l2(wd))(x)
discriminator_model = Model(input=[disc_input], output=x, name="discriminator_google")
visualize_model(discriminator_model)
return discriminator_model
def build_model(input_dim,
h0_dim=20,
h1_dim=20,
output_dim=1,
rec_layer_type=ReducedLSTMA,
rec_layer_init='zero',
layer_type=TimeDistributedDense,
lr=.001,
base_name='rlstm',
add_input_noise=True,
add_target_noise=True):
model = Sequential()
if add_input_noise:
model.add(GaussianNoise(.1, input_shape=(None, input_dim)))
model.add(
LSTM(h0_dim,
input_dim=input_dim,
init='uniform_small',
activation='tanh'))
model.add(Dropout(.4))
model.add(LSTM(h1_dim, init='uniform_small', activation='tanh'))
model.add(Dropout(.4))
model.add(
rec_layer_type(output_dim, init=rec_layer_init, return_sequences=True))
if add_target_noise:
model.add(GaussianNoise(5.))
model.compile(loss="mse", optimizer=RMSprop(lr=lr))
model.base_name = base_name
yaml_string = model.to_yaml()
# print(yaml_string)
with open(model_savedir + model.base_name + '.yaml', 'w') as f:
f.write(yaml_string)
return model
def nips_cnn(inputs, n_out):
"""NIPS 2016 CNN Model"""
# injected noise strength
sigma = 0.1
# first layer
y = Conv2D(16,
15,
data_format="channels_first",
kernel_regularizer=l2(1e-3))(inputs)
y = Activation('relu')(GaussianNoise(sigma)(y))
# second layer
y = Conv2D(8, 9, data_format="channels_first",
kernel_regularizer=l2(1e-3))(y)
y = Activation('relu')(GaussianNoise(sigma)(y))
y = Flatten()(y)
y = Dense(n_out,
init='normal',
kernel_regularizer=l2(1e-3),
activity_regularizer=l1(1e-3))(y)
outputs = Activation('softplus')(y)
return Model(inputs, outputs, name='NIPS_CNN')
def build_model():
model = Sequential()
# 第一个卷积层，4个卷积核
model.add(
Convolution2D(8,
5,
5,
border_mode='valid',
dim_ordering='th',
input_shape=(1, 20, 20)))
model.add(ZeroPadding2D((1, 1)))
model.add(GaussianNoise(0.001))
model.add(Activation('tanh'))
model.add(MaxPooling2D(pool_size=(2, 2), dim_ordering='th'))
model.add(Activation('tanh'))
# 第二个卷积层，8个卷积核，
# model.add(GaussianNoise(0.001))
# model.add(UpSampling2D(size=(2, 2), dim_ordering='th'))
model.add(
AtrousConvolution2D(16, 3, 3, border_mode='valid', dim_ordering='th'))
# model.add(ZeroPadding2D((1, 1)))
model.add(Activation('tanh'))
# model.add(MaxPooling2D(pool_size=(2, 2), dim_ordering='th'))
# model.add(Activation('tanh'))
# 全连接层，先将前一层输出的二维特征图flatten为一维的
model.add(Flatten())
model.add(Dense(20))
model.add(Activation('tanh'))
# LSTM 层
model.add(Reshape((20, 1)))
model.add(
LSTM(input_dim=1,
output_dim=32,
activation='tanh',
inner_activation='tanh',
return_sequences=True))
model.add(GaussianNoise(0.01))
model.add(
LSTM(64,
activation='tanh',
inner_activation='tanh',
return_sequences=False))
model.add(Dropout(0.2)) # Dropout overfitting
model.add(Dense(1))
model.add(Activation('linear'))
start = time.time()
# 使用SGD + momentum
# model.compile里的参数loss就是损失函数(目标函数)
# sgd = SGD(lr=0.08, decay=1e-6, momentum=0.9, nesterov=True)
# model.compile(loss="mse", optimizer=sgd)
model.compile(loss="mse", optimizer="Nadam") # Nadam # rmsprop
print "Compilation Time : ", time.time() - start
return model
def createModel(dropout_fraction=0.0, batch_normalization=False):
input = Input(shape=(3, 256, 256))
x = Convolution2D(32,
5,
5,
activation='relu',
W_regularizer=l2(0.01),
border_mode='same',
name='conv1')(input)
#x = Dropout(dropout_fraction)(x)
x = MaxPooling2D((4, 4), strides=(4, 4), name='pool1')(x)
x = BatchNormalization()(x)
x = GaussianNoise(0.05)(x) #Roque
x = Convolution2D(32,
5,
5,
activation='relu',
W_regularizer=l2(0.01),
border_mode='same',
name='conv2')(x)
#x = Dropout(dropout_fraction)(x)
x = MaxPooling2D((4, 4), strides=(4, 4), name='pool2')(x)
x = BatchNormalization()(x)
x = GaussianNoise(0.05)(x) #Roque
x = MaxPooling2D((4, 4), strides=(4, 4),
name='pool3')(x) #Roque (Before (2,2))
#Classification block
x = Flatten(name='flatten')(x)
#x = Dropout(dropout_fraction)(x)
x = Dense(512, activation='relu', name='fc1')(x)
#x = Dropout(dropout_fraction)(x)
x = Dense(8, activation='softmax', name='predictions')(x)
model = Model(input=input, output=x)
return model
def create_mergegrouped(base_params):
# np.random.seed(seed)
feature_len = len(X[0, 0, :])
season_len = len(X[0, :])
node_stretch = base_params['layer_scale']
noise_std = base_params['noise']
conv_size = base_params['conv_size']
# sized based on number of features times scale factor
l1_size = int(feature_len * node_stretch)
l2_size = int(feature_len * node_stretch)
left_model = Sequential()
right_model = Sequential()
left_model.add(
GaussianNoise(noise_std, input_shape=(season_len, feature_len)))
right_model.add(
GaussianNoise(noise_std, input_shape=(season_len, feature_len)))
left_model.add(
Convolution1D(conv_size,
1,
input_shape=(season_len, feature_len),
input_length=feature_len))
right_model.add(
Convolution1D(conv_size,
1,
input_shape=(season_len, feature_len),
input_length=feature_len))
left_model.add(Flatten())
right_model.add(Flatten())
left_model.add(Dense(l1_size, init='normal', activation='sigmoid'))
left_model.add(Dropout(.15))
right_model.add(Dense(l1_size, init='normal', activation='tanh'))
right_model.add(Dropout(.15))
final_model = Sequential()
merged = Merge([left_model, right_model], mode='ave')
final_model.add(merged)
final_model.add(Dense(l2_size, init='normal', activation='sigmoid'))
final_model.add(Dropout(.1))
final_model.add(Dense(12, init='normal', activation='softmax', bias=True))
final_model.compile(
loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy', 'precision', 'recall', 'fmeasure'])
return final_model
def discriminator_2048x7x7(img_dim, wd, n_classes, disc_type):
disc_input = Input(shape=img_dim, name="discriminator_input")
x = Conv2D(128, (7, 7),
strides=(1, 1),
name="conv1",
border_mode="same",
kernel_initializer=RandomNormal(stddev=0.02),
kernel_regularizer=l2(wd))(disc_input)
x = BatchNormGAN(axis=1)(x)
x = Dropout(0.1)(x)
x = LeakyReLU(0.2)(x)
x = GaussianNoise(sigma=0.2)(x)
aux = x
x = Conv2D(1, (3, 3),
strides=(1, 1),
name="finale_conv",
border_mode="same",
kernel_initializer=RandomNormal(stddev=0.02),
kernel_regularizer=l2(wd))(x)
aux = Flatten()(aux)
aux = Dense(n_classes,
activation='softmax',
name='auxiliary',
W_regularizer=l2(wd))(aux)
x = GlobalAveragePooling2D()(x)
discriminator_model_domain = Model(input=[disc_input],
output=[x],
name="discriminator_domain")
discriminator_model_class = Model(input=[disc_input],
output=[aux],
name="discriminator_class")
visualize_model(discriminator_model_domain)
visualize_model(discriminator_model_class)
return discriminator_model_domain, discriminator_model_class
def get_model(input_shape,
embedding_layer,
classes=6,
units=1024,
dtype=tf.float32):
sentence_indices = Input(shape=input_shape, dtype=dtype)
embeddings = embedding_layer(sentence_indices)
noised_embeddings = GaussianNoise(0.2)(embeddings)
dropped_embeddings = Dropout(rate=0.2)(noised_embeddings)
# recurrent_regularizer=l1_l2(0.01,0.01)
# activity_regularizer=l1_l2(0.01, 0.01)
# kernel_regularizer=l1_l2(0.01, 0.01)
# bias_regularizer=l1_l2(0.01, 0.01)
x = Bidirectional(LSTM(units=units,
return_sequences=False))(dropped_embeddings)
# x = Attention(input_shape[0])(x)
x = Dropout(rate=0.3)(x)
# x = Bidirectional(LSTM(units=units))(x)
# x = Dropout(rate=0.5)(x)
x = Dense(units=classes, activation="softmax")(x)
return Model(inputs=sentence_indices, outputs=x)
def create_model_3_noise():
inputs = Input((32, 32, 32, 1))
noise = GaussianNoise(sigma=0.05)(inputs)
conv1 = Convolution3D(32, 3, 3, 3, activation='relu', border_mode='same')(noise)
conv1 = SpatialDropout3D(0.1)(conv1)
conv1 = Convolution3D(64, 3, 3, 3, activation='relu', border_mode='same')(conv1)
pool1 = MaxPooling3D(pool_size=(2,2, 2))(conv1)
conv2 = Convolution3D(128, 3, 3, 3, activation='relu', border_mode='same')(pool1)
conv2 = SpatialDropout3D(0.1)(conv2)
conv2 = Convolution3D(128, 3, 3, 3, activation='relu', border_mode='same')(conv2)
pool2 = MaxPooling3D(pool_size=(2,2, 2))(conv2)
x = Flatten()(pool2)
x = Dense(64, init='normal')(x)
x = Dropout(0.5)(x)
predictions = Dense(1, init='normal', activation='sigmoid')(x)
model = Model(input=inputs, output=predictions)
model.summary()
optimizer = Adam(lr=0.000001)
model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['binary_accuracy','precision','recall','mean_squared_error','accuracy'])
return model
def build_rlstm2(input_dim,
h0_dim,
h1_dim,
output_dim=1,
lstm_init='zero',
lr=.001,
base_name='rlstm',
add_input_noise=True,
add_target_noise=False):
model = Sequential()
if add_input_noise:
model.add(GaussianNoise(.1, input_shape=(None, input_dim)))
model.add(
RLSTM(input_dim,
h0_dim,
h1_dim,
output_dim,
init=lstm_init,
W_h0_regularizer=l2(0.0005),
W_h1_regularizer=l2(0.0005),
return_sequences=True))
# if add_target_noise:
# model.add(GaussianNoise(5.))
model.compile(loss="mse", optimizer=RMSprop(lr=lr))
model.base_name = base_name
yaml_string = model.to_yaml()
# print(yaml_string)
with open(model_savedir + model.base_name + '.yaml', 'w') as f:
f.write(yaml_string)
return model
def train_classifier(data, num_classes, train_labels, train_features):
"""
Function to train the neural network
:param data: the config dictionary
:param num_classes: the distinct number of labels (binary/multi-class)
:param train_labels: the labels of the training instances
:param train_features: the features of the training instances
:return: the MLP classifier
"""
# Neural Network model using keras
data = data["keras"]
print("Creating Sequential model in keras")
# Create new model with the specified params in the config file
data_dimension = train_features.shape[1]
epochs = data["epochs"]
batch_size=data["batch_size"]
model = Sequential([
Dense(data["hidden_layers_size"], input_shape=(data_dimension,), name="first_dense"),
Activation('relu', name="first_activation"),
Dense(data["hidden_layers_size"], activation='relu', name="second_dense"),
Dropout(data["dropout"]),
Dense(data["hidden_layers_size"], activation='relu', name="third_dense"),
GaussianNoise(data["gaussian_noise"]),
Dense(num_classes, name="Dense_to_numclasses"),
Activation('softmax', name="classification_activation"),
])
# Train the model and return it
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
# Convert labels to categorical one-hot encoding
one_hot_labels = keras.utils.to_categorical(train_labels, num_classes=num_classes)
model.fit(train_features, one_hot_labels, epochs=epochs, batch_size=batch_size)
return model
def denoisingAutoencoder(self, noise, deep_size):
if self.dropout_noise is None:
self.model.add(
GaussianNoise(noise, input_shape=(self.input_size, )))
else:
self.model.add(
Dropout(self.dropout_noise[0],
input_shape=(self.input_size, )))
if deep_size is not None:
self.model.add(
Dense(output_dim=deep_size,
input_dim=self.hidden_layer_size,
init=self.layer_init,
activation=self.hidden_activation,
W_regularizer=l2(self.reg)))
self.model.add(
Dense(output_dim=self.hidden_layer_size,
input_dim=self.input_size,
init=self.layer_init,
activation=self.hidden_activation,
W_regularizer=l2(self.reg)))
self.model.add(
Dense(output_dim=self.output_size,
init=self.layer_init,
activation=self.output_activation,
W_regularizer=l2(self.reg)))
self.model.compile(loss=self.loss, optimizer=self.optimizer)
return self.model
def encode(x, relu_max):
print 'encoder input shape:', x._keras_shape
assert x._keras_shape[1:] == (96, 96, 3)
# 96, 96, 3
y = Conv2D(64,
3,
3,
activation='relu',
border_mode='same',
subsample=(2, 2))(x)
y = BN(mode=2, axis=3)(y)
# 48, 48, 64
y = Conv2D(128,
3,
3,
activation='relu',
border_mode='same',
subsample=(2, 2))(y)
y = BN(mode=2, axis=3)(y)
# 24, 24, 128
y = Conv2D(256,
3,
3,
activation='relu',
border_mode='same',
subsample=(2, 2))(y)
y = BN(mode=2, axis=3)(y)
# 12, 12, 256
y = Conv2D(512,
3,
3,
activation='relu',
border_mode='same',
subsample=(2, 2))(y)
y = BN(mode=2, axis=3)(y)
# 6, 6, 512
assert y._keras_shape[1:] == (6, 6, 512), \
'%s vs %s' % (y._keras_shape[1:], [6, 6, 512])
y = Conv2D(LATENT_DIM,
6,
6,
activation='linear',
border_mode='same',
subsample=(6, 6))(y)
# 1, 1, LATENT_DIM
if not relu_max:
print 'add noise and pretend relu_max will be:', RELU_MAX
y = GaussianNoise(0.2 * RELU_MAX)(y)
y = Activation(utils.relu_n(relu_max))(y)
if relu_max:
print 'relu_max:', relu_max
y = Activation(utils.scale_down(relu_max))(y)
# y in [0, 1]
y = Reshape((LATENT_DIM, ))(y) # or Reshape([-1])(y) ?
# LATENT_DIM
return y
def mk_model():
i = Input(shape=(C, H, W))
stack = [
GaussianNoise(0.2),
Conv2D(8, 3, 3, border_mode='valid', activation='relu'),
Conv2D(8, 3, 3, border_mode='valid', activation='relu'),
MPool2D(pool_size=(2, 2)),
Conv2D(16, 3, 3, border_mode='valid', activation='relu'),
Conv2D(16, 3, 3, border_mode='valid', activation='relu'),
MPool2D(pool_size=(2, 2)),
Flatten(),
Dropout(0.5),
Dense(32, activation='relu'),
# Dropout(0.5),
# Dense(32, activation='relu'),
Dense(NDIM, activation='softmax'),
]
y = i
for layer in stack:
y = layer(y)
m = Model(input=i, output=y)
m.compile(optimizer='adam', loss='categorical_crossentropy')
return m, stack[0]
def get_RNN_model(in_shape,
td_num=512,
ltsm_out_dim=256,
nb_hidden=100,
drop1=0.5,
drop2=0.5):
model = Sequential()
model.add(GaussianNoise(0.05, input_shape=in_shape))
model.add(TimeDistributedDense(td_num))
model.add(LSTM(ltsm_out_dim, return_sequences=True))
reg = l2(0.05)
# model.add(TimeDistributedDense(td_num, W_regularizer=l2(0.03)))
#reg.set_param(model.layers[3].get_params()[0][0])
#model.layers[3].regularizers = [reg]
model.add(Dropout(drop1))
model.add(LSTM(ltsm_out_dim))
# reg = l2(0.05)
# reg.set_param(model.layers[3].get_params()[0][0])
# model.layers[3].regularizers = [reg]
model.add(Dropout(drop1))
# model.regularizers = [l2(0.05)]
#model.add(Activation('relu'))
model.add(Flatten())
model.add(Dense(nb_hidden, W_regularizer=l2(0.05)))
model.add(Activation('relu'))
model.add(Dropout(drop2))
model.add(Dense(1))
model.add(Activation('linear'))
model.compile(loss='mse', optimizer='rmsprop')
return model
def build_discriminator():
# build a relatively standard conv net, with LeakyReLUs as suggested in
# the reference paper
cnn = Sequential()
cnn.add(GaussianNoise(0.05, input_shape=(3, 32, 32))) # Add this layer to prevent D from overfitting!
cnn.add(Conv2D(16, kernel_size=3, strides=2, padding='same',
kernel_initializer='glorot_normal', bias_initializer='Zeros'))
cnn.add(LeakyReLU(alpha=0.2))
cnn.add(Dropout(0.5))
cnn.add(Conv2D(32, kernel_size=3, strides=1, padding='same',
kernel_initializer='glorot_normal', bias_initializer='Zeros'))
cnn.add(BatchNormalization())
cnn.add(LeakyReLU(alpha=0.2))
cnn.add(Dropout(0.5))
cnn.add(Conv2D(64, kernel_size=3, strides=2, padding='same',
kernel_initializer='glorot_normal', bias_initializer='Zeros'))
cnn.add(BatchNormalization())
cnn.add(LeakyReLU(alpha=0.2))
cnn.add(Dropout(0.5))
cnn.add(Conv2D(128, kernel_size=3, strides=1, padding='same',
kernel_initializer='glorot_normal', bias_initializer='Zeros'))
cnn.add(BatchNormalization())
cnn.add(LeakyReLU(alpha=0.2))
cnn.add(Dropout(0.5))
cnn.add(Conv2D(256, kernel_size=3, strides=2, padding='same',
kernel_initializer='glorot_normal', bias_initializer='Zeros'))
cnn.add(BatchNormalization())
cnn.add(LeakyReLU(alpha=0.2))
cnn.add(Dropout(0.5))
cnn.add(Conv2D(512, kernel_size=3, strides=1, padding='same',
kernel_initializer='glorot_normal', bias_initializer='Zeros'))
cnn.add(BatchNormalization())
cnn.add(LeakyReLU(alpha=0.2))
cnn.add(Dropout(0.5))
cnn.add(Flatten())
cnn.add(MinibatchDiscrimination(50, 30))
image = Input(shape=(3, 32, 32))
features = cnn(image)
# first output (name=generation) is whether or not the discriminator
# thinks the image that is being shown is fake, and the second output
# (name=auxiliary) is the class that the discriminator thinks the image
# belongs to.
fake = Dense(1, activation='sigmoid', name='generation',
kernel_initializer='glorot_normal', bias_initializer='Zeros')(features)
aux = Dense(class_num, activation='softmax', name='auxiliary',
kernel_initializer='glorot_normal', bias_initializer='Zeros')(features)
return Model(image, [fake, aux])
def create_G(input_dim=(100, ), output_dim=(9, 20)):
G = Sequential()
G.add(Dense(input_shape=input_dim, \
units= 128, \
kernel_initializer=initializers.random_normal(stddev=0.02)))
G.add(BatchNormalization())
#G.add(Conv2DTranspose(32, 5, strides=(2,1), activation=Activation('relu'), padding='same',kernel_initializer='glorot_uniform'))
# G.add(GaussianDropout(0.25)) #https://arxiv.org/pdf/1611.07004v1.pdf
# G.add(GaussianNoise(0.05))
G.add(Activation('relu'))
G.add(Dense(128))
G.add(BatchNormalization())
G.add(GaussianDropout(0.25)) #https://arxiv.org/pdf/1611.07004v1.pdf
G.add(GaussianNoise(0.05))
G.add(Activation('relu'))
G.add(
Dense(np.prod(output_dim),
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l2(0.01)))
G.add(BatchNormalization())
# G.add(GaussianDropout(0.25)) #https://arxiv.org/pdf/1611.07004v1.pdf
# G.add(GaussianNoise(0.05))
G.add(Activation('sigmoid'))
G.add(Reshape(output_dim))
return (G)
def build_model5(input_shape,
nb_output_bin,
kernel_sizes,
checkpoints_dir=None):
# ref: Deep Speaker Feature Learning for Text-independent Speaker Verification
shape = list(input_shape)
dilation_depth = 4
stack = 1
glog.info('Shape:: %s', shape)
out = start = Input(shape=shape, name='start')
out = GaussianNoise(0.025)(out)
delay_kernel_size = kernel_sizes.pop(0)
for j in range(stack):
for i in range(dilation_depth):
out = Conv2D(128,
delay_kernel_size,
dilation_rate=(2**i, 1),
name='relu_dilation_%s_%s' % (j, 2**i),
padding='same',
activation='relu')(out)
out = Dense(512, name='bottleneck')(out)
out = Conv2D(256, kernel_sizes.pop(0))(out)
# shape = (6, 24) or (6, 33)
out = MaxPooling2D(kernel_sizes.pop(0))(out)
# shape = (3, 12)
out = Conv2D(256, kernel_sizes.pop(0))(out)
# shape = (2, 8)
out = MaxPooling2D(kernel_sizes.pop(0))(out)
# out = Conv2D(128, kernel_sizes.pop(0))(out)
# out = Dropout(0.5)(out)
# out = Conv2D(128, kernel_sizes.pop(0))(out)
# out = Dropout(0.5)(out)
print(out.shape)
# shape = (3, 5)
out = Reshape(kernel_sizes.pop(0))(out)
# D vector
out = Dense(400, name='d_vector')(out)
out = GlobalAveragePooling1D()(out)
out = Dense(nb_output_bin, activation='softmax')(out)
model = Model(inputs=start, outputs=out)
field_width, field_ms = compute_receptive_field(dilation_depth, 1)
glog.info('model with width=%s and width_ms=%s' % (field_width, field_ms))
model.summary()
if checkpoints_dir:
_load_checkoint(model, checkpoints_dir)
else:
model.epoch_num = 0
return model
def copy_cnn(inputs, n_out, *args, l2_reg=0.01):
"""Standard CNN with no batch norm"""
sigma = 0.05
print(inputs.shape)
y = Conv2D(8,
15,
data_format="channels_first",
kernel_regularizer=l2(1e-3))(inputs)
y = Activation('relu')(GaussianNoise(sigma)(y))
y = Conv2D(8,
11,
data_format="channels_first",
kernel_regularizer=l2(1e-3))(y)
y = Activation('relu')(GaussianNoise(sigma)(y))
y = Dense(n_out, use_bias=False)(Flatten()(y))
outputs = Activation('softplus')(y)
return Model(inputs, outputs, name='COPY_CNN')
def get_unet_mod(patch_size = (None,None),learning_rate = 1e-5,\
learning_decay = 1e-6,gn_std = 0.025, drop_out = 0.25):
''' Get U-Net model with gaussian noise and dropout'''
gaussian_noise_std = gn_std
dropout = drop_out
input_img = Input((patch_size[0], patch_size[1], 3))
input_with_noise = GaussianNoise(gaussian_noise_std)(input_img)
conv1 = Conv2D(32, (3, 3), activation='relu',
padding='same')(input_with_noise)
conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(pool1)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(128, (3, 3), activation='relu', padding='same')(pool2)
conv3 = Conv2D(128, (3, 3), activation='relu', padding='same')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = Conv2D(256, (3, 3), activation='relu', padding='same')(pool3)
conv4 = Conv2D(256, (3, 3), activation='relu', padding='same')(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)
pool4 = Dropout(dropout)(pool4)
conv5 = Conv2D(512, (3, 3), activation='relu', padding='same')(pool4)
conv5 = Conv2D(512, (3, 3), activation='relu', padding='same')(conv5)
up6 = concatenate([UpSampling2D(size=(2, 2))(conv5), conv4], axis=-1)
up6 = Dropout(dropout)(up6)
conv6 = Conv2D(256, (3, 3), activation='relu', padding='same')(up6)
conv6 = Conv2D(256, (3, 3), activation='relu', padding='same')(conv6)
up7 = concatenate([UpSampling2D(size=(2, 2))(conv6), conv3], axis=-1)
up7 = Dropout(dropout)(up7)
conv7 = Conv2D(128, (3, 3), activation='relu', padding='same')(up7)
conv7 = Conv2D(128, (3, 3), activation='relu', padding='same')(conv7)
up8 = concatenate([UpSampling2D(size=(2, 2))(conv7), conv2], axis=-1)
up8 = Dropout(dropout)(up8)
conv8 = Conv2D(64, (3, 3), activation='relu', padding='same')(up8)
conv8 = Conv2D(64, (3, 3), activation='relu', padding='same')(conv8)
up9 = concatenate([UpSampling2D(size=(2, 2))(conv8), conv1], axis=-1)
up9 = Dropout(dropout)(up9)
conv9 = Conv2D(32, (3, 3), activation='relu', padding='same')(up9)
conv9 = Conv2D(32, (3, 3), activation='relu', padding='same')(conv9)
conv10 = Conv2D(1, (1, 1), activation='sigmoid')(conv9)
model = Model(inputs=input_img, outputs=conv10)
opt = Adam(lr=learning_rate, decay=learning_decay)
model.compile(optimizer=opt, loss=dice_coef_loss, metrics=[dice_coef])
return model
def define_model(input):
model = Sequential([
# For each training iteration, we're going to drop 10% of the input
# nodes, randomly. This helps avoid overfitting.
Dropout(0.1, input_shape=(len(input[0]), )),
# We normalize the inputs to make it easier to weight them correctly.
BatchNormalization(epsilon=0.001,
mode=0,
axis=-1,
momentum=0.99,
weights=None,
beta_init='zero',
gamma_init='one',
gamma_regularizer=None,
beta_regularizer=None),
GaussianNoise(0.01), # Just a tiny bit of noise.
# Ended up going with a lot of layers. Trains fast with large batch sizes.
# Integer input is the number of nodes in given layer.
Dense(
256,
W_regularizer=l2(0.01),
activity_regularizer=activity_l2(0.01),
),
# LeakyReLU is a modified Rectified Linear Unit.
# "Leaky" refers to a modification to the negative slope,
# which can get "stuck" in standard ReLUs.
LeakyReLU(alpha=0.1818),
Dense(32),
LeakyReLU(alpha=0.1818),
Dense(32),
LeakyReLU(alpha=0.1818),
Dense(32),
LeakyReLU(alpha=0.1818),
Dense(32),
LeakyReLU(alpha=0.1818),
Dense(32),
LeakyReLU(alpha=0.1818),
Dense(32),
LeakyReLU(alpha=0.1818),
Dense(2),
Activation("softmax"),
])
# Adam is a form of stochastic gradient descent with improved handling
# of altering the learning rate on the fly.
adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
# Compile the model with all our parameters.
# We use binary cross-entropy, also known as log-loss.
print("Training deep neural network model...")
model.compile(loss='binary_crossentropy',
optimizer=adam,
metrics=['accuracy'])
return model
def BuildUnet(self,
_input_shape,
_filter_num,
_depth,
dropout=True,
_kernel_size=3,
_stride=2,
_nchannel=1,
_data_format='channels_last',
use_softmax=False):
cut = [] # list used to save all cuts
filters = [] # list used to save all filter numbers
filter_num = _filter_num
if _data_format == 'channels_last':
axis = -1
else:
axis = 1
inputs = Input(shape=_input_shape)
x = GaussianNoise(0.03)(inputs)
x = Conv2D(_filter_num, 1, padding='same', data_format=_data_format)(x)
# Down sampling part
for i in range(_depth):
filters.append(filter_num)
x = self.ResidualBlock(_inputs=x, _filter_num=filter_num)
cut.append(x)
filter_num *= 2
x = self.DownsampleBlock(x, filter_num)
if i >= _depth - 2 and dropout:
x = Dropout(0.5)(x)
print(filters)
# One left over bottom block
x = self.ResidualBlock(_inputs=x, _filter_num=filter_num)
# Up sampling part
for i in range(_depth):
filter_num = filter_num // 2
x = self.UpsampleBlock(x, filter_num)
shortcut = cut[-(i + 1)]
x = keras.layers.concatenate([x, shortcut], axis=axis)
x = self.ResidualBlock(_inputs=x,
_filter_num=2 * filters[-(i + 1)])
# Output
if use_softmax:
# for cross entropy
outputs = Conv2D(_nchannel, (1, 1), activation='softmax')(x)
else:
# for iou loss
outputs = Conv2D(_nchannel, (1, 1), activation='sigmoid')(x)
model = Model(inputs=inputs, outputs=outputs)
return model
def aSDAE(S, X, latent_dim, hidden_unit_nums=[], stddev=0.1, lam=0.5):
S_noise = GaussianNoise(stddev)(S)
X_noise = GaussianNoise(stddev)(X)
h = S_noise
for num in hidden_unit_nums:
h_s = Dense(num,
kernel_regularizer=keras.regularizers.l2(lam),
bias_regularizer=keras.regularizers.l2(lam))(h)
h_x = Dense(num,
kernel_regularizer=keras.regularizers.l2(lam),
bias_regularizer=keras.regularizers.l2(lam))(X_noise)
h = Add()([h_s, h_x])
h = Activation("relu")(h)
latent_s = Dense(latent_dim,
kernel_regularizer=keras.regularizers.l2(lam),
bias_regularizer=keras.regularizers.l2(lam))(h)
latent_x = Dense(latent_dim,
kernel_regularizer=keras.regularizers.l2(lam),
bias_regularizer=keras.regularizers.l2(lam))(X_noise)
latent = Add()([latent_s, latent_x])
latent = Activation("relu")(latent)
h = latent
for num in hidden_unit_nums[::-1]:
h_s = Dense(num,
kernel_regularizer=keras.regularizers.l2(lam),
bias_regularizer=keras.regularizers.l2(lam))(h)
h_x = Dense(num,
kernel_regularizer=keras.regularizers.l2(lam),
bias_regularizer=keras.regularizers.l2(lam))(X_noise)
h = Add()([h_s, h_x])
h = Activation("relu")(h)
S_ = Dense(int(S.shape[1]),
kernel_regularizer=keras.regularizers.l2(lam),
bias_regularizer=keras.regularizers.l2(lam))(h)
X_ = Dense(int(X.shape[1]),
kernel_regularizer=keras.regularizers.l2(lam),
bias_regularizer=keras.regularizers.l2(lam))(h)
S_ = Activation("relu")(S_)
X_ = Activation("relu")(X_)
return latent, S_, X_
def model_arch(
self
): # Defines the modified sequential class with regularizers defined.
model = Sequential()
dropout_param = 0.5
activation_fn = "relu"
Gaussian_noise = 1
adam = Adam(lr=0.0005, beta_1=0.9, beta_2=0.999, epsilon=5e-09)
model.add(
Dense(32,
input_dim=2,
kernel_initializer="uniform",
activation=activation_fn,
kernel_constraint=maxnorm(3)))
model.add(Dropout(dropout_param))
model.add(BatchNormalization())
model.add(GaussianNoise(Gaussian_noise))
model.add(
Dense(128,
kernel_initializer="uniform",
activation=activation_fn,
kernel_constraint=maxnorm(3)))
model.add(Dropout(dropout_param))
model.add(BatchNormalization())
model.add(GaussianNoise(Gaussian_noise))
model.add(
Dense(32,
kernel_initializer="uniform",
activation=activation_fn,
kernel_constraint=maxnorm(3)))
model.add(Dropout(dropout_param))
model.add(BatchNormalization())
model.add(GaussianNoise(Gaussian_noise))
model.add(Dense(2, activation="softmax", kernel_initializer="normal"))
model.compile(loss="categorical_crossentropy",
optimizer=adam,
metrics=["accuracy"])
return model
def bn_layer(x, nchan, size, l2_reg, sigma=0.05):
"""An individual batchnorm layer"""
n = int(x.shape[-1]) - size + 1
y = Conv2D(nchan,
size,
data_format="channels_first",
kernel_regularizer=l2(l2_reg))(x)
y = Reshape((nchan, n, n))(BatchNormalization(axis=-1)(Flatten()(y)))
return Activation('relu')(GaussianNoise(sigma)(y))
def ANN_Custom(self, inputdimention, labelsnum):
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers.noise import GaussianNoise
from keras.constraints import maxnorm
self.__model = Sequential()
self.__model.add(
Dense(output_dim=inputdimention,
input_dim=inputdimention,
init=self.getInit(),
activation=self.getACTFUNC(),
W_constraint=maxnorm(1))) #2构建图模型
#add noise layer
self.__model.add(GaussianNoise(self.getGNsigma()))
##hidden layer
outputdimention = self.getOutDimention()
layersNum = self.getLayersNum()
subdim = int((inputdimention - outputdimention) / layersNum)
print "subdim", subdim
while layersNum > 0:
inputdim = inputdimention - subdim * (self.getLayersNum() -
layersNum)
outputdim = inputdim
print "outputdim", outputdim
self.__model.add(
Dense(output_dim=outputdim,
input_dim=inputdim,
init=self.getInit(),
activation=self.getACTFUNC(),
W_constraint=maxnorm(1)))
self.__model.add(Dropout(self.getDropoutRate()))
layersNum = layersNum - 1
print "layersNum--", layersNum
##full-conect layer
self.__model.add(
Dense(output_dim=outputdimention,
input_dim=outputdimention,
init=self.getInit(),
activation=self.getACTFUNC(),
W_constraint=maxnorm(1)))
##last layer
self.__model.add(
Dense(output_dim=labelsnum,
input_dim=outputdimention,
init=self.getInit(),
activation='softmax',
W_constraint=maxnorm(1))) #,W_constraint=maxnorm(1)
return self.__model
def build_gmodel():
graph = Graph()
graph.add_input(name='time_series', ndim=3)
graph.add_node(JZS3(63, 40), name='rnn1', input='time_series')
# # graph.add_node(JZS3(63, 40), name='rnn2', input='time_series')
# # graph.add_node(JZS3(63, 40), name='rnn3', input='time_series')
# # graph.add_node(JZS3(63, 40), name='rnn4', input='time_series')
graph.add_node(Dense(40, 40), name='dense1', input='rnn1')
# # graph.add_node(Dense(40, 40), name='dense2', input='rnn2')
# # graph.add_node(Dense(40, 40), name='dense3', input='rnn3')
# # graph.add_node(Dense(40, 40), name='dense4', input='rnn4')
graph.add_node(MaxoutDense(40, 20, nb_feature=4),
name='maxout1',
input='dense1')
# # graph.add_node(MaxoutDense(40, 80, nb_feature=4), name='maxout2', input='dense2')
# # graph.add_node(MaxoutDense(40, 80, nb_feature=4), name='maxout3', input='dense3')
# # graph.add_node(MaxoutDense(40, 80, nb_feature=4), name='maxout4', input='dense4')
graph.add_node(Dropout(0.5), name='dropout1', input='maxout1')
# # graph.add_node(Dropout(0.5), name='dropout2', input='maxout2')
# # graph.add_node(Dropout(0.5), name='dropout3', input='maxout3')
# # graph.add_node(Dropout(0.5), name='dropout4', input='maxout4')
# graph.add_node(Dense(320, 160, activation='softmax'), name='merge', inputs=['dropout1', 'dropout2', 'dropout3', 'dropout4'], merge_mode='concat')
# graph.add_node(MaxoutDense(160, 160, nb_feature=4), name='merge_maxout', input='merge')
# graph.add_node(Dropout(0.5), name='merge_dropout', input='merge_maxout')
graph.add_node(Dense(20, 1, activation='sigmoid'),
name='out_dense',
input='dropout1')
graph.add_input(name='enrollment', ndim=2)
graph.add_node(GaussianNoise(0.05), name='noise', input='enrollment')
# graph.add_node(Dense(54, 64), name='mlp_dense', inputs=['enrollment', 'out_dense'])
graph.add_node(Dense(53, 64), name='mlp_dense', input='noise')
graph.add_node(MaxoutDense(64, 64, nb_feature=4),
name='mlp_maxout',
input='mlp_dense')
graph.add_node(Dropout(0.5), name='mlp_dropout', input='mlp_maxout')
# graph.add_node(Dense(32, 16), name='mlp_dense2', input='mlp_dropout')
graph.add_node(Dense(64, 1, activation='sigmoid'),
name='mlp_dense3',
input='mlp_dropout')
graph.add_node(
Dense(4, 2, activation='softmax'),
name='mlp_dense4',
inputs=['mlp_dense3', 'out_dense', 'mlp_dense3', 'out_dense'],
merge_mode='concat')
graph.add_node(Dense(2, 1, activation='sigmoid'),
name='mlp_dense5',
input='mlp_dense4')
# graph.add_node(Dense(2, 1), name='my_output', inputs=['mlp_dense2', 'out_dense'], merge_mode='concat')
graph.add_output(name='output', input='mlp_dense5')
graph.compile('adam', {'output': 'binary_crossentropy'})
return graph
