def objective(params):
inputs = Input(shape=(m_rna.shape[1], ), name="inputs")
inputs_noise = GaussianNoise(stddev=params['gaussian_noise'])(inputs)
inputs_noise = GaussianDropout(
rate=params['gaussian_dropout']**2 /
(1 + params['gaussian_dropout']**2))(inputs_noise)
inputs_0 = BatchNormalization(name="inputs_0")(inputs_noise)
inputs_0 = Dropout(rate=params['dropout1'], name='dropout_1')(inputs_0)
inputs_1 = Dense(params['layer1'],
activation=params['activation1'],
name="inputs_1")(inputs_0)
inputs_2 = BatchNormalization(name="inputs_2")(inputs_1)
inputs_2 = Dropout(rate=params['dropout2'], name='dropout_2')(inputs_2)
inputs_3 = Dense(params['layer3'],
activation=params['activation3'],
name="inputs_3")(inputs_2)
inputs_4 = BatchNormalization(name="inputs_4")(inputs_3)
inputs_4 = Dropout(rate=params['dropout3'], name='dropout_3')(inputs_4)
h_q = Dense(params['n_z'],
activation=params['activation_hq'],
name="h_q")(inputs_4)
mu = Dense(params['n_z'], activation="linear", name="mu")(h_q)
log_sigma = Dense(params['n_z'], activation="linear",
name="log_sigma")(h_q)
def sample_z(args):
mu_samples, log_sigma_sample = args
eps = backend.random_normal(shape=(params['batch_size'],
params['n_z']),
mean=0.,
stddev=1.)
return mu_samples + backend.exp(log_sigma_sample / 2) * eps
# Sample z ~ Q(z|x)
z = Lambda(sample_z, name="lambda")([mu, log_sigma])
# P(x|z) -- decoder
decoder_hidden = Dense(params['layer_decoder_hidden'],
activation=params['activation_decoder_hidden'],
name="decoder_hidden")
decoded_tcga = Dense(m_rna.shape[1],
activation=params['activation_tcga'],
name="m_rna")
decoded_micro_rna = Dense(mi_rna.shape[1],
activation=params['activation_micro_rna'],
name="mi_rna")
decoded_cl_tissue = Dense(categorical_tissue.shape[1],
activation="softmax",
name="cl_tissue")
decoded_cl_disease = Dense(categorical_disease.shape[1],
activation="softmax",
name="cl_disease")
h_p = decoder_hidden(z)
outputs_tcga = decoded_tcga(h_p)
outputs_micro_rna = decoded_micro_rna(h_p)
outputs_cl_tissue = decoded_cl_tissue(h_p)
outputs_cl_disease = decoded_cl_disease(h_p)
lambda_value = params['lambda']
def vae_loss(y_true, y_pred):
""" Calculate loss = reconstruction loss + KL loss for each data_pred in minibatch """
# E[log P(x|z)]
recon = backend.mean(backend.square(y_true - y_pred), axis=1)
# D_KL(Q(z|x) || P(z|x)); calculate in closed form as both dist. are Gaussian
kl = 0.5 * backend.sum(
backend.exp(log_sigma) + backend.square(mu) - 1. - log_sigma,
axis=1)
return recon + lambda_value * kl
svae_dropout = Model(inputs, [
outputs_tcga, outputs_micro_rna, outputs_cl_tissue,
outputs_cl_disease
])
svae_dropout.compile(
optimizer=params['optimizer'],
loss=[vae_loss, "mse", "cosine_proximity", "cosine_proximity"],
loss_weights=[1e-3, 1e-3, 5e-1, 5e-1],
metrics={
"m_rna": ["mae", "mse"],
"mi_rna": ["mae", "mse"],
"cl_tissue": "acc",
"cl_disease": "acc"
})
svae_dropout.fit(m_rna_train, [
m_rna_train, mi_rna_train, categorical_tissue_train,
categorical_disease_train
],
batch_size=params['batch_size'],
epochs=params['nb_epochs'],
verbose=2)
score = svae_dropout.evaluate(m_rna_test, [
m_rna_test, mi_rna_test, categorical_tissue_test,
categorical_disease_test
],
verbose=0,
batch_size=params['batch_size'])
print(score)
with open(result_folder + 'hyperopt-Dropout-VAE.txt', 'ab') as file:
np.savetxt(file, score, delimiter=",")
return {
'loss':
np.mean([
score[1], score[2], score[5], score[6], score[7], score[8],
-score[9], -score[10]
]),
'status':
STATUS_OK
}
from sklearn.datasets import make_circles
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Activation
from keras.layers import GaussianNoise
from matplotlib import pyplot
# generate 2d classification dataset
X, y = make_circles(n_samples=100, noise=0.1, random_state=1)
# split into train and test
n_train = 30
trainX, testX = X[:n_train, :], X[n_train:, :]
trainy, testy = y[:n_train], y[n_train:]
# define model
model = Sequential()
model.add(Dense(500, input_dim=2))
model.add(GaussianNoise(0.1))
model.add(Activation('relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# fit model
history = model.fit(trainX, trainy, validation_data=(testX, testy), epochs=4000, verbose=0)
# evaluate the model
_, train_acc = model.evaluate(trainX, trainy, verbose=0)
_, test_acc = model.evaluate(testX, testy, verbose=0)
print('Train: %.3f, Test: %.3f' % (train_acc, test_acc))
# plot loss learning curves
pyplot.subplot(211)
pyplot.title('Cross-Entropy Loss', pad=-40)
pyplot.plot(history.history['loss'], label='train')
pyplot.plot(history.history['val_loss'], label='test')
pyplot.legend()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
ax = plt.subplot(2, n, i + 1 + n)
plt.imshow(image2.reshape(32, 32))
plt.gray()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.show()
X_train = train_images
X_test = test_images
noised_X_train = GaussianNoise(0.2)(X_train, training=True)
noised_X_test = GaussianNoise(0.2)(X_test, training=True)
from keras import backend as K
K.clear_session()
X_train = preprocess(X_train)
#nosied_X_train = preprocess(noised_X_train)
X_test = preprocess(X_test)
#noised_X_test = preprocess(X_test)
#FIXME
#Gaussian noise 넣을 때 전처리 함.
# Encoder
with tf.device('/gpu:0'):
def autoencoder(dims, noise_sd=0, init='glorot_uniform', act='relu'):
"""
Fully connected auto-encoder model, symmetric.
Arguments:
dims: list of number of units in each layer of encoder. dims[0] is input dim, dims[-1] is units in hidden layer.
The decoder is symmetric with encoder. So number of layers of the auto-encoder is 2*len(dims)-1
act: activation, not applied to Input, Hidden and Output layers
return:
Model of autoencoder
"""
n_stacks = len(dims) - 1
# input
sf_layer = Input(shape=(1, ), name='size_factors')
x = Input(shape=(dims[0], ), name='counts')
h = x
h = GaussianNoise(noise_sd, name='input_noise')(h)
# internal layers in encoder
for i in range(n_stacks - 1):
h = Dense(dims[i + 1], kernel_initializer=init,
name='encoder_%d' % i)(h)
h = BatchNormalization(center=True,
scale=False,
name='encoder_batchnorm_%d' % i)(h)
h = Activation(act, name='encoder_act_%d' % i)(h)
# hidden layer
h = Dense(dims[-1], kernel_initializer=init, name='encoder_hidden')(
h) # hidden layer, features are extracted from here
h = BatchNormalization(center=True,
scale=False,
name='encoder_hidden_batchnorm_%d' % i)(h)
h = Activation(act, name='encoder_hidden_act')(h)
# internal layers in decoder
for i in range(n_stacks - 1, 0, -1):
h = Dense(dims[i], kernel_initializer=init, name='decoder_%d' % i)(h)
h = BatchNormalization(center=True,
scale=False,
name='decoder_batchnorm_%d' % i)(h)
h = Activation(act, name='decoder_act_%d' % i)(h)
# output
pi = Dense(dims[0],
activation='sigmoid',
kernel_initializer=init,
name='pi')(h)
disp = Dense(dims[0],
activation=DispAct,
kernel_initializer=init,
name='dispersion')(h)
mean = Dense(dims[0],
activation=MeanAct,
kernel_initializer=init,
name='mean')(h)
output = ColWiseMultLayer(name='output')([mean, sf_layer])
output = SliceLayer(0, name='slice')([output, disp, pi])
return Model(inputs=[x, sf_layer], outputs=output)
def fit_model(lr=0.001,
dropout=0.55,
reg=0.0004,
yw=45,
hw=75,
fw=185,
final=275):
print("-----------------------1--------------------------")
print("lr:" + str(lr) + " dropout:" + str(dropout) + " yw:" + str(yw) +
" hw:" + str(hw) + " fw:" + str(fw) + " final:" + str(final))
total_width = yw + hw + fw
# yeast
y = Input(shape=(yeast_dims, ), name='yeast')
y1 = Dense(yw, activation='relu')(y)
y2 = Dropout(dropout, input_shape=(yw, ))(y1)
y3 = Dense(yw, activation='relu')(y2)
# hops
h = Input(shape=(
11,
hop_dims,
), name='hop')
conv_h = Convolution1D(32, (11, ),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001),
activation='relu')(h)
conv_h = BatchNormalization()(conv_h)
h4 = Dense(hw, activation='relu',
kernel_regularizer=regularizers.l2(reg))(conv_h)
h5 = Dropout(dropout, input_shape=(hw, ))(h4)
h6 = Dense(hw, activation='relu',
kernel_regularizer=regularizers.l2(reg))(h5)
h7 = Flatten()(h6)
# fermentables
f = Input(shape=(
6,
fermentable_dims,
), name='fermentable')
conv_f = Convolution1D(32, (6, ),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001),
activation='relu')(f)
conv_f = BatchNormalization()(conv_f)
f4 = Dense(fw, activation='relu',
kernel_regularizer=regularizers.l2(reg))(conv_f)
f5 = Dropout(dropout, input_shape=(fw, ))(f4)
f6 = Dense(fw, activation='relu',
kernel_regularizer=regularizers.l2(reg))(f5)
f7 = Flatten()(f6)
# with our networks combined...
a = Concatenate()([y3, h7, f7])
a1 = Dense(total_width, activation='relu')(a)
tn = GaussianNoise(0.01)(a1)
t = Dense(total_width, activation='relu')(tn)
td1 = GaussianDropout(dropout, input_shape=(total_width, ))(t)
a5 = Dense(final,
activation='relu',
kernel_regularizer=regularizers.l2(reg))(td1)
out = Dense(style_dims, activation='softmax', name='style')(a5)
model = Model(inputs=[y, h, f], outputs=out)
sgd = SGD(lr=lr, momentum=0.9, decay=0.0, nesterov=False)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
model.fit([yeast_data, hops_data, fermentables_data], [one_hot_labels],
epochs=epochs,
verbose=verbose,
validation_split=0.1,
callbacks=[tb_cb])
return model
def generator2(self):
if self.G2:
return self.G2
kernel_height = 5
depth = 32 * 16
dim = (self.input_shape[0] + 12) / 16
weight_decay = 1e-8
noise_std = 0.01
c2 = Input(shape=(self.latent_dim, ))
z2 = Input(shape=(self.noise_dim, ))
x1 = Input(shape=self.input_shape)
x1_flat = Flatten()(x1)
x = concatenate([c2, z2, x1_flat])
x = Dense(dim * 2 * depth, kernel_regularizer=l2(weight_decay))(x)
x = BatchNormalization(momentum=0.9)(x)
x = LeakyReLU(alpha=0.2)(x)
x = Reshape((dim, 2, depth))(x)
x = UpSampling2D((2, 1))(x)
x = Conv2DTranspose(int(depth / 2), (kernel_height, 2),
padding='same',
kernel_regularizer=l2(weight_decay))(x)
x = BatchNormalization(momentum=0.9)(x)
x = LeakyReLU(alpha=0.2)(x)
x = GaussianNoise(noise_std)(x)
x = UpSampling2D((2, 1))(x)
x = Conv2DTranspose(int(depth / 4), (kernel_height, 2),
padding='same',
kernel_regularizer=l2(weight_decay))(x)
x = BatchNormalization(momentum=0.9)(x)
x = LeakyReLU(alpha=0.2)(x)
x = GaussianNoise(noise_std)(x)
x = UpSampling2D((2, 1))(x)
x = Conv2DTranspose(int(depth / 8), (kernel_height, 2),
padding='same',
kernel_regularizer=l2(weight_decay))(x)
x = BatchNormalization(momentum=0.9)(x)
x = LeakyReLU(alpha=0.2)(x)
x = GaussianNoise(noise_std)(x)
x = UpSampling2D((2, 1))(x)
x = Conv2DTranspose(int(depth / 16), (kernel_height, 2),
padding='same',
kernel_regularizer=l2(weight_decay))(x)
x = BatchNormalization(momentum=0.9)(x)
x = LeakyReLU(alpha=0.2)(x)
x = GaussianNoise(noise_std)(x)
# Out: 100 x 2, xy coordinates, [-1.0,1.0] per coordinate
x = Conv2DTranspose(1, (kernel_height, 2),
padding='same',
kernel_regularizer=l2(weight_decay))(x)
x = Activation('tanh')(x)
x = Cropping2D((6, 0))(x)
self.G2 = Model(inputs=[c2, z2, x1], outputs=x)
self.G2.summary()
return self.G2
temp[i] = 1
data.append(temp)
data = np.array(data)
R = 3.0 / 3.0
n_channel = 3
for x in range(0,10):
input_signal = Input(shape=(M,))
encoded = Dense(M, activation='relu')(input_signal)
encoded1 = Dense(n_channel, activation='linear')(encoded)
encoded2 = CustomNormalization(1.0)(encoded1)
EbNo_train = np.power(10, 0.7)
alpha1 = pow((2 * R * EbNo_train), -0.5)
encoded3 = GaussianNoise(alpha1)(encoded2)
decoded = Dense(M, activation='relu')(encoded3)
decoded1 = Dense(M, activation='softmax')(decoded)
autoencoder = Model(input_signal, decoded1)
autoencoder.compile(optimizer='adam', loss='categorical_crossentropy')
autoencoder.summary()
N_val = 1500
val_label = np.random.randint(M, size=N_val)
val_data = []
for i in val_label:
temp = np.zeros(M)
temp[i] = 1
val_data.append(temp)
val_data = np.array(val_data)
dropout_val = 0.5
DIM_ORDERING = "tf"
l2_val = 0.000
#Load the VGG model
image_size = 224
#Load the VGG model
input_1 = Input(shape=(image_size, image_size, 3))
input_2 = Input(shape=(image_size, image_size, 3))
ll1 = Lambda(lambda cool: cool/127.5 - 1)(input_1)
ll2 = Lambda(lambda cool: cool/127.5 - 1)(input_2)
g1 = GaussianNoise(0.04)(ll1)
g2 = GaussianNoise(0.04)(ll2)
vgg_conv_1_base = VGG16(weights='imagenet', include_top=False)
vgg_conv_2_base = VGG16(weights='imagenet', include_top=False)
#vgg_conv_2_base.get_layer(name='vgg16').name='vgg16_1'
#for i in range(15):
# vgg_conv_1_base.layers.pop()
#for i in range(15):
# vgg_conv_2_base.layers.pop()
for layer in vgg_conv_1_base.layers:
layer.trainable = False
def build_model(self, n_features):
"""
The method builds a new member of the ensemble and returns it.
"""
# derived parameters
self.hyperparameters['n_members'] = self.hyperparameters[
'n_segments'] * self.hyperparameters['n_members_segment']
# initialize optimizer and early stopping
self.optimizer = Adam(lr=self.hyperparameters['lr'],
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.,
amsgrad=False)
self.es = EarlyStopping(monitor=f'val_{self.loss_name}',
min_delta=0.0,
patience=self.hyperparameters['patience'],
verbose=1,
mode='min',
restore_best_weights=True)
inputs = Input(shape=(n_features, ))
h = GaussianNoise(self.hyperparameters['noise_in'],
name='noise_input')(inputs)
for i in range(self.hyperparameters['layers']):
h = Dense(self.hyperparameters['neurons'],
activation=self.hyperparameters['activation'],
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_hidden'],
self.hyperparameters['l2_hidden']),
kernel_initializer='random_uniform',
bias_initializer='zeros',
name=f'hidden_{i}')(h)
h = Dropout(self.hyperparameters['dropout'],
name=f'hidden_dropout_{i}')(h)
mu = Dense(1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_mu'],
self.hyperparameters['l2_mu']),
kernel_initializer='random_uniform',
bias_initializer='zeros',
name='mu_output')(h)
mu = GaussianNoise(self.hyperparameters['noise_mu'],
name='noise_mu')(mu)
if self.hyperparameters['pdf'] == 'normal' or self.hyperparameters[
'pdf'] == 'skewed':
sigma = Dense(1,
activation='softplus',
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_sigma'],
self.hyperparameters['l2_sigma']),
kernel_initializer='random_uniform',
bias_initializer='zeros',
name='sigma_output')(h)
sigma = GaussianNoise(self.hyperparameters['noise_sigma'],
name='noise_sigma')(sigma)
if self.hyperparameters['pdf'] == 'skewed':
alpha = Dense(1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_alpha'],
self.hyperparameters['l2_alpha']),
kernel_initializer='random_uniform',
bias_initializer='zeros',
name='alpha_output')(h)
alpha = GaussianNoise(self.hyperparameters['noise_alpha'],
name='noise_alpha')(alpha)
if self.hyperparameters['pdf'] is None:
outputs = mu
elif self.hyperparameters['pdf'] == 'normal':
outputs = concatenate([mu, sigma])
elif self.hyperparameters['pdf'] == 'skewed':
outputs = concatenate([mu, sigma, alpha])
model = Model(inputs=inputs, outputs=outputs)
return model
return getDataAndTargets(matrix)
return getData(matrix)
trainX, trainY = getDataFromFile(train_file, 1)
testX = getDataFromFile(test_file, 0)
dataScaler = preprocessing.StandardScaler().fit(trainX)
trainX = dataScaler.transform(trainX)
testX = dataScaler.transform(testX)
trainY = trainY #/9.0
input_shape = trainX[0].shape
model = Sequential()
model.add(GaussianNoise(0.05, input_shape=input_shape))
model.add(Dense(30, activation='relu', input_shape=input_shape))
#model.add(Dense(15, activation='relu'))
#model.add(Dense(10, activation='relu', input_shape=input_shape))
#model.add(Dense(5, activation='relu'))
model.add(Dense(5, activation='relu'))
model.add(Dense(1, activation='linear', input_shape=input_shape))
sgd = SGD(lr=0.003, decay=0.000004, momentum=0.3, nesterov=False)
adagrad = Adagrad(lr=0.01, epsilon=None, decay=0.0)
model.compile(loss='mean_squared_error', optimizer=sgd)
learning_rate_reduction = keras.callbacks.ReduceLROnPlateau(monitor='val_loss',
patience=5,
verbose=1,
factor=0.8,
# Summarise stimuli
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
print(y_train.shape[1], 'training categories')
print(y_test.shape[1], 'testing categories')
# filters = 64
# NX = 32
# NY = 32
# NC = 1
# img_rows, img_cols, img_chns = NX, NY, NC
intermediate_dim = 1024
x = Input(shape=x_train[0].shape)
gn = GaussianNoise(noise_start)(x)
if retina_layers > 2:
conv1_nonlin = Conv2D(retina_hidden_channels, (filter_size, filter_size),
kernel_regularizer=keras.regularizers.l1(reg),
padding='same',
name='retina_1',
activation='relu',
input_shape=x_train.shape[1:])(gn)
retina_out = Conv2D(retina_hidden_channels, (filter_size, filter_size),
kernel_regularizer=keras.regularizers.l1(reg),
padding='same',
activation='relu',
name='retina_2',
trainable=True)(conv1_nonlin)
for iterationX in range(retina_layers - 2):
if iterationX == retina_layers - 3:
def GaussianTest():
inputs = Input((1,48,48,48))
noise = GaussianNoise(stddev=0.5,input_shape=(1,48,48,48))(inputs)
model = Model(inputs=inputs, outputs=noise)
return model
def gaussian_noise(layer, layer_in, layerId, tensor=True):
stddev = layer['params']['stddev']
out = {layerId: GaussianNoise(stddev=stddev)}
if tensor:
out[layerId] = out[layerId](*layer_in)
return out
trainFeat = pca.fit_transform(feature_train_bsif[train])
validateFeat = pca.transform(feature_train_bsif[validate])
trainFeat = feature_train_bsif[train]
validateFeat = feature_train_bsif[validate]
numFeatures_bsif = trainFeat.shape[1]
#Define model
model = None
model = Sequential()
model.add(
Dense(numFeatures_bsif,
input_dim=numFeatures_bsif,
init='glorot_uniform',
activation='relu'))
model.add(GaussianNoise(gaus_sigma))
model.add(Dropout(do))
#model.add(Dense(numFeatures_bsif, activation='relu'))
#model.add(Dropout(do))
#model.add(Dense(numFeatures_bsif, activation='relu'))
#model.add(Dropout(do))
model.add(Dense(1, activation='sigmoid'))
#compile model
opt = Adadelta()
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=['accuracy'])
#Fit and evaluate model
hist = model.fit(
def build(width, height, depth):
STRIDE = 2
POOLING = False
BIAS = True
assert POOLING + STRIDE == 2
inputs = Input(shape=(width, height, depth))
x = GaussianNoise(0.1)(inputs)
conv_1 = Conv2D(32, (3, 3), padding="same", strides=1, use_bias=BIAS)
x = conv_1(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
conv_2 = Conv2D(32, (3, 3),
padding="same",
strides=STRIDE,
use_bias=BIAS)
x = conv_2(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
if POOLING:
x = MaxPooling2D((2, 2))(x)
conv_3 = Conv2D(64, (3, 3), padding="same", strides=1, use_bias=BIAS)
x = conv_3(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
conv_4 = Conv2D(64, (3, 3),
padding="same",
strides=STRIDE,
use_bias=BIAS)
x = conv_4(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
if POOLING:
x = MaxPooling2D((2, 2))(x)
conv_5 = Conv2D(128, (3, 3), padding="same", strides=1, use_bias=BIAS)
x = conv_5(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
conv_6 = Conv2D(128, (3, 3),
padding="same",
strides=STRIDE,
use_bias=BIAS)
x = conv_6(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
if POOLING:
x = MaxPooling2D((2, 2))(x)
conv_7 = Conv2D(256, (3, 3), padding="same", strides=1, use_bias=BIAS)
x = conv_7(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
conv_8 = Conv2D(256, (3, 3),
padding="same",
strides=STRIDE,
use_bias=BIAS)
x = conv_8(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
if POOLING:
x = MaxPooling2D((2, 2))(x)
conv_9 = Conv2D(512, (3, 3), padding="same", strides=1, use_bias=BIAS)
x = conv_9(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
conv_10 = Conv2D(512, (3, 3),
padding="same",
strides=STRIDE,
use_bias=BIAS)
x = conv_10(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
if POOLING:
x = MaxPooling2D((2, 2))(x)
# flatten the network and then construct the latent vector
volumeSize = K.int_shape(x)
x = Flatten()(x)
latent = Dense(1024, name="encoded")
x = latent(x)
x = DenseTied(np.prod(volumeSize[1:]), tied_to=latent)(x)
# start building the decoder model which will accept the
# output of the encoder as its inputs
x = Reshape((volumeSize[1], volumeSize[2], volumeSize[3]))(x)
if POOLING:
x = UpSampling2D()(x)
x = TiedConv2DTranspose(512, (3, 3),
padding="same",
tied_to=conv_10,
strides=STRIDE,
use_bias=BIAS)(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
x = TiedConv2DTranspose(256, (3, 3),
padding="same",
tied_to=conv_9,
strides=1,
use_bias=BIAS)(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
if POOLING:
x = UpSampling2D()(x)
x = TiedConv2DTranspose(256, (3, 3),
padding="same",
tied_to=conv_8,
strides=STRIDE,
use_bias=BIAS)(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
x = TiedConv2DTranspose(128, (3, 3),
padding="same",
tied_to=conv_7,
strides=1,
use_bias=BIAS)(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
if POOLING:
x = UpSampling2D()(x)
x = TiedConv2DTranspose(128, (3, 3),
padding="same",
tied_to=conv_6,
strides=STRIDE,
use_bias=BIAS)(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
x = TiedConv2DTranspose(64, (3, 3),
padding="same",
tied_to=conv_5,
strides=1,
use_bias=BIAS)(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
if POOLING:
x = UpSampling2D()(x)
x = TiedConv2DTranspose(64, (3, 3),
padding="same",
tied_to=conv_4,
strides=STRIDE,
use_bias=BIAS)(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
x = TiedConv2DTranspose(32, (3, 3),
padding="same",
tied_to=conv_3,
strides=1,
use_bias=BIAS)(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
if POOLING:
x = UpSampling2D()(x)
x = TiedConv2DTranspose(32, (3, 3),
padding="same",
tied_to=conv_2,
strides=STRIDE,
use_bias=BIAS)(x)
x = BatchNormalization()(x)
x = PReLU(alpha_initializer=Constant(value=0.25))(x)
x = TiedConv2DTranspose(3, (3, 3),
padding="same",
tied_to=conv_1,
strides=1,
use_bias=BIAS)(x)
x = BatchNormalization()(x)
outputs = Activation("sigmoid")(x)
ae = Model(inputs=inputs, outputs=outputs)
return ae
def steg_model(pretrain=False):
if (pretrain):
model = load_model(PRETRAINED,
custom_objects={
'custom_loss_1': custom_loss_1,
'custom_loss_2': custom_loss_2
})
return model
# Inputs
secret = Input(shape=(224, 224, 3), name='secret')
cover = Input(shape=(224, 224, 3), name='cover')
# Prepare network - patches [3*3,4*4,5*5]
pconv_3x3 = Conv2D(50,
kernel_size=3,
padding="same",
activation='relu',
name='prep_conv3x3_1')(secret)
pconv_3x3 = Conv2D(50,
kernel_size=3,
padding="same",
activation='relu',
name='prep_conv3x3_2')(pconv_3x3)
pconv_3x3 = Conv2D(50,
kernel_size=3,
padding="same",
activation='relu',
name='prep_conv3x3_3')(pconv_3x3)
pconv_3x3 = Conv2D(50,
kernel_size=3,
padding="same",
activation='relu',
name='prep_conv3x3_4')(pconv_3x3)
pconv_4x4 = Conv2D(50,
kernel_size=4,
padding="same",
activation='relu',
name='prep_conv4x4_1')(secret)
pconv_4x4 = Conv2D(50,
kernel_size=4,
padding="same",
activation='relu',
name='prep_conv4x4_2')(pconv_4x4)
pconv_4x4 = Conv2D(50,
kernel_size=4,
padding="same",
activation='relu',
name='prep_conv4x4_3')(pconv_4x4)
pconv_4x4 = Conv2D(50,
kernel_size=4,
padding="same",
activation='relu',
name='prep_conv4x4_4')(pconv_4x4)
pconv_5x5 = Conv2D(50,
kernel_size=5,
padding="same",
activation='relu',
name='prep_conv5x5_1')(secret)
pconv_5x5 = Conv2D(50,
kernel_size=5,
padding="same",
activation='relu',
name='prep_conv5x5_2')(pconv_5x5)
pconv_5x5 = Conv2D(50,
kernel_size=5,
padding="same",
activation='relu',
name='prep_conv5x5_3')(pconv_5x5)
pconv_5x5 = Conv2D(50,
kernel_size=5,
padding="same",
activation='relu',
name='prep_conv5x5_4')(pconv_5x5)
pconcat_1 = concatenate([pconv_3x3, pconv_4x4, pconv_5x5],
axis=3,
name="prep_concat_1")
pconv_5x5 = Conv2D(50,
kernel_size=5,
padding="same",
activation='relu',
name='prep_conv5x5_f')(pconcat_1)
pconv_4x4 = Conv2D(50,
kernel_size=4,
padding="same",
activation='relu',
name='prep_conv4x4_f')(pconcat_1)
pconv_3x3 = Conv2D(50,
kernel_size=3,
padding="same",
activation='relu',
name='prep_conv3x3_f')(pconcat_1)
pconcat_f1 = concatenate([pconv_5x5, pconv_4x4, pconv_3x3],
axis=3,
name="prep_concat_2")
# Hiding network - patches [3*3,4*4,5*5]
hconcat_h = concatenate([cover, pconcat_f1], axis=3, name="hide_concat_1")
hconv_3x3 = Conv2D(50,
kernel_size=3,
padding="same",
activation='relu',
name='hide_conv3x3_1')(hconcat_h)
hconv_3x3 = Conv2D(50,
kernel_size=3,
padding="same",
activation='relu',
name='hide_conv3x3_2')(hconv_3x3)
hconv_3x3 = Conv2D(50,
kernel_size=3,
padding="same",
activation='relu',
name='hide_conv3x3_3')(hconv_3x3)
hconv_3x3 = Conv2D(50,
kernel_size=3,
padding="same",
activation='relu',
name='hide_conv3x3_4')(hconv_3x3)
hconv_4x4 = Conv2D(50,
kernel_size=4,
padding="same",
activation='relu',
name='hide_conv4x4_1')(hconcat_h)
hconv_4x4 = Conv2D(50,
kernel_size=4,
padding="same",
activation='relu',
name='hide_conv4x4_2')(hconv_4x4)
hconv_4x4 = Conv2D(50,
kernel_size=4,
padding="same",
activation='relu',
name='hide_conv4x4_3')(hconv_4x4)
hconv_4x4 = Conv2D(50,
kernel_size=4,
padding="same",
activation='relu',
name='hide_conv4x4_4')(hconv_4x4)
hconv_5x5 = Conv2D(50,
kernel_size=5,
padding="same",
activation='relu',
name='hide_conv5x5_1')(hconcat_h)
hconv_5x5 = Conv2D(50,
kernel_size=5,
padding="same",
activation='relu',
name='hide_conv5x5_2')(hconv_5x5)
hconv_5x5 = Conv2D(50,
kernel_size=5,
padding="same",
activation='relu',
name='hide_conv5x5_3')(hconv_5x5)
hconv_5x5 = Conv2D(50,
kernel_size=5,
padding="same",
activation='relu',
name='hide_conv5x5_4')(hconv_5x5)
hconcat_1 = concatenate([hconv_3x3, hconv_4x4, hconv_5x5],
axis=3,
name="hide_concat_2")
hconv_5x5 = Conv2D(50,
kernel_size=5,
padding="same",
activation='relu',
name='hide_conv5x5_f')(hconcat_1)
hconv_4x4 = Conv2D(50,
kernel_size=4,
padding="same",
activation='relu',
name='hide_conv4x4_f')(hconcat_1)
hconv_3x3 = Conv2D(50,
kernel_size=3,
padding="same",
activation='relu',
name='hide_conv3x3_f')(hconcat_1)
hconcat_f1 = concatenate([hconv_5x5, hconv_4x4, hconv_3x3],
axis=3,
name="hide_concat_3")
cover_pred = Conv2D(3, kernel_size=1, padding="same",
name='hide_conv_f')(hconcat_f1)
# Noise layer
noise_ip = GaussianNoise(0.1)(cover_pred)
# Reveal network - patches [3*3,4*4,5*5]
rconv_3x3 = Conv2D(50,
kernel_size=3,
padding="same",
activation='relu',
name='revl_conv3x3_1')(noise_ip)
rconv_3x3 = Conv2D(50,
kernel_size=3,
padding="same",
activation='relu',
name='revl_conv3x3_2')(rconv_3x3)
rconv_3x3 = Conv2D(50,
kernel_size=3,
padding="same",
activation='relu',
name='revl_conv3x3_3')(rconv_3x3)
rconv_3x3 = Conv2D(50,
kernel_size=3,
padding="same",
activation='relu',
name='revl_conv3x3_4')(rconv_3x3)
rconv_4x4 = Conv2D(50,
kernel_size=4,
padding="same",
activation='relu',
name='revl_conv4x4_1')(noise_ip)
rconv_4x4 = Conv2D(50,
kernel_size=4,
padding="same",
activation='relu',
name='revl_conv4x4_2')(rconv_4x4)
rconv_4x4 = Conv2D(50,
kernel_size=4,
padding="same",
activation='relu',
name='revl_conv4x4_3')(rconv_4x4)
rconv_4x4 = Conv2D(50,
kernel_size=4,
padding="same",
activation='relu',
name='revl_conv4x4_4')(rconv_4x4)
rconv_5x5 = Conv2D(50,
kernel_size=5,
padding="same",
activation='relu',
name='revl_conv5x5_1')(noise_ip)
rconv_5x5 = Conv2D(50,
kernel_size=5,
padding="same",
activation='relu',
name='revl_conv5x5_2')(rconv_5x5)
rconv_5x5 = Conv2D(50,
kernel_size=5,
padding="same",
activation='relu',
name='revl_conv5x5_3')(rconv_5x5)
rconv_5x5 = Conv2D(50,
kernel_size=5,
padding="same",
activation='relu',
name='revl_conv5x5_4')(rconv_5x5)
rconcat_1 = concatenate([rconv_3x3, rconv_4x4, rconv_5x5],
axis=3,
name="revl_concat_1")
rconv_5x5 = Conv2D(50,
kernel_size=5,
padding="same",
activation='relu',
name='revl_conv5x5_f')(rconcat_1)
rconv_4x4 = Conv2D(50,
kernel_size=4,
padding="same",
activation='relu',
name='revl_conv4x4_f')(rconcat_1)
rconv_3x3 = Conv2D(50,
kernel_size=3,
padding="same",
activation='relu',
name='revl_conv3x3_f')(rconcat_1)
rconcat_f1 = concatenate([rconv_5x5, rconv_4x4, rconv_3x3],
axis=3,
name="revl_concat_2")
secret_pred = Conv2D(3, kernel_size=1, padding="same",
name='revl_conv_f')(rconcat_f1)
model = Model(inputs=[secret, cover], outputs=[cover_pred, secret_pred])
# Multi GPU training (Uncomment the following line)
#model = multi_gpu_model(model, gpus=2)
# Compile model
model.compile(optimizer='adam', loss=losses, loss_weights=lossWeights)
return model
feature_train = feature_train.values
feature_train = preprocessing.scale(feature_train) #normalize
target_train = target_train.values
feature_test = feature_test.values
feature_test = preprocessing.scale(feature_test)
target_test = target_test.values
#Start model and ML here
# Model 1
model = Sequential()
model.add(
Dense(numFeatures,
input_dim=numFeatures,
init='glorot_uniform',
activation='relu'))
model.add(GaussianNoise(3))
model.add(Dropout(0.3))
#model.add(Dense(numFeatures, activation='relu'))
#model.add(Dropout(0.3))
#model.add(Dense(numFeatures, activation='relu'))
#model.add(Dropout(0.3))
model.add(Dense(1, activation='sigmoid'))
# Show some debug output
print(model.summary())
#compile model
opt = Adadelta()
model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])
#fit the model
def QuantizationAutoencoder(mod_size,
latent_dim,
num_layers,
hidden_dim,
common_layer,
latent_layer,
weight_reg,
local_seed,
verbose=False,
noise_sigma=1e-3,
passthrough=False):
# NN parameters
input_dim = mod_size
# Local seed
np.random.seed(local_seed)
# Generate integers to seed each non-sigma layer
seed_array = np.random.randint(low=0, high=2**31, size=2 * num_layers)
# Initializers
weight_init = []
for layer_idx in range(2 * num_layers):
weight_init.append(glorot_uniform(seed=seed_array[layer_idx]))
# Weights regularizers
l2_reg = weight_reg
# Input layer
input_bits = Input(shape=(input_dim, ))
# Universal encoder
encoded = Dense(hidden_dim[0],
activation=common_layer,
kernel_initializer=weight_init[0],
kernel_regularizer=l2(l2_reg),
name='enc_layer0')(input_bits)
for layer_idx in range(1, num_layers - 1):
encoded = Dense(hidden_dim[layer_idx],
activation=common_layer,
kernel_initializer=weight_init[layer_idx],
kernel_regularizer=l2(l2_reg),
name='enc_layer%d' % (layer_idx))(encoded)
# Final layer is tanh activated
encoded = Dense(latent_dim,
activation=latent_layer,
kernel_initializer=weight_init[num_layers - 1],
kernel_regularizer=l2(l2_reg),
name='enc_layer%d' % (num_layers - 1))(encoded)
# If passthrough is enabled, quantize in forward pass
if passthrough:
encoded = Lambda(lambda x: K.stop_gradient(K.sign(x) - x) + x)(encoded)
encoded_noisy = encoded
# Otherwise, add noise
else:
encoded_noisy = GaussianNoise(stddev=noise_sigma,
name='noise_layer')(encoded)
# List of decoders
# Keep a list of outputs
output_bit_list = []
for decoder_idx in range(mod_size):
decoded = Dense(hidden_dim[-1],
activation=common_layer,
kernel_initializer=weight_init[num_layers],
kernel_regularizer=l2(l2_reg),
name='dec_bit%d_layer0' % (decoder_idx))(encoded_noisy)
for layer_idx in range(num_layers + 1, 2 * num_layers - 1):
decoded = Dense(hidden_dim[-(layer_idx - num_layers + 1)],
activation=common_layer,
kernel_initializer=weight_init[layer_idx],
kernel_regularizer=l2(l2_reg),
name='dec_bit%d_layer%d' %
(decoder_idx, layer_idx - num_layers))(decoded)
# Final layer is tanh activated
output_bit = Dense(1,
activation='tanh',
kernel_initializer=weight_init[2 * num_layers - 1],
kernel_regularizer=l2(l2_reg),
name='dec_bit%d_layer%d' %
(decoder_idx, num_layers - 1))(decoded)
# Append to tensor list
output_bit_list.append(output_bit)
# Concatenate output tensors in single output
output_bits = Concatenate(axis=-1)(output_bit_list)
# this model maps end-to-end
autoencoder = Model(input_bits, output_bits)
# Save encoder model
encoder = Model(input_bits, encoded)
# Extract local decoder networks and local autoencoder networks
decoder_list = []
bit_list = []
# Global input
local_decoder_input = Input(shape=(latent_dim, ))
for decoder_idx in range(mod_size):
local_decoder = local_decoder_input
# Stack layers by name starting from the latent representation
for layer_idx in range(num_layers):
local_decoder = autoencoder.get_layer(
name='dec_bit%d_layer%d' %
(decoder_idx, layer_idx))(local_decoder)
# After stacking, save output
bit_list.append(local_decoder)
# Create local model
local_decoder = Model(inputs=local_decoder_input,
outputs=local_decoder)
# Append to list
decoder_list.append(local_decoder)
# Create local autoencoder models
autoencoder_list = []
for decoder_idx in range(mod_size):
# One-output
local_ae = Model(inputs=input_bits,
outputs=output_bit_list[decoder_idx])
autoencoder_list.append(local_ae)
# Shadow concatentation
bit_list = Concatenate(axis=-1)(bit_list)
# Joint decoder
decoder = Model(inputs=local_decoder_input, outputs=bit_list)
# Print model summary
if verbose:
autoencoder.summary()
return autoencoder, autoencoder_list, encoder, decoder, decoder_list
def get_model(args, num_tags=0, max_slen=0, num_words=0, wemb_dim=0, wemb_matrix=None, max_wlen=0, num_chars=0, cemb_dim=0, cemb_matrix=None):
"""
Obtains a neural model as a combination of layers
Inputs:
- args: parsed command line arguments
- num_tags: number of output tags
- max_slen: maximum number of words in a sentence
- num_words: size of the word embedding vocabulary
- wemb_dim: dimensionality of the word embedding vectors
- wemb_matrix: word embedding matrix
- max_wlen: maximum number of characters in a word
- num_chars: size of the character embedding vocabulary
- cemb_dim: dimensionality of the character embedding vectors
- cemb_matrix: character embedding matrix
Returns:
the compiled Keras neural model defined by the command line arguments
"""
## DEFINE NETWORK
if args.use_words:
# word input layer
word_input = Input(shape=(max_slen,))
# word embedding layer
word_model = Embedding(input_dim=num_words,
output_dim=wemb_dim,
weights = [wemb_matrix],
input_length = max_slen,
trainable = bool(args.word_embeddings_trainable))(word_input)
if args.use_chars:
# character input layer
char_input = Input(shape=(max_slen, max_wlen))
# character embedding layer
x = Reshape((max_slen * max_wlen, ))(char_input)
x = Embedding(input_dim=num_chars,
output_dim=cemb_dim,
weights = [cemb_matrix],
input_length = max_slen * max_wlen,
trainable = bool(args.char_embeddings_trainable))(x)
x = Reshape((max_slen, max_wlen, cemb_dim))(x)
# build word-like features from character features using a residual network
# the residual network is constructed by stacking residual blocks
shortcut = x
for _ in range(max(1, args.resnet_depth)):
# build a residual block
x = Conv2D(max_slen, kernel_size=(3, 3), padding='same', data_format='channels_first')(x)
if args.batch_normalization:
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.01)(x)
x = Conv2D(max_slen, kernel_size=(3, 3), padding='same', data_format='channels_first')(x)
if args.batch_normalization:
x = BatchNormalization()(x)
# merge input and shortcut
x = add([shortcut, x])
x = LeakyReLU(alpha=0.01)(x)
shortcut = x
# finish building the character model
char_model = Reshape((max_slen, max_wlen * cemb_dim))(x)
# concat word and character features if needed
if args.use_words and args.use_chars:
model = concatenate([word_model, char_model])
elif args.use_words:
model = word_model
elif args.use_chars:
model = char_model
# noise layer
if args.noise_sigma:
model = GaussianNoise(args.noise_sigma)(model)
# batch normalization layer
if args.batch_normalization:
model = BatchNormalization()(model)
# recurrent layers
num_units = args.model_size
for _ in range(args.num_layers):
layer = get_layer(args, num_units)
model = layer(model)
# batch normalization layer
if args.batch_normalization:
model = BatchNormalization()(model)
# output layer
if args.output_activation == 'crf':
# the crf layer optimizes marginal likelihoods of each class
# the joint likelihood becomes the product of marginal probabilities
crf = CRF(num_tags, learn_mode='marginal')
out = crf(model)
else:
out = TimeDistributed(Dense(num_tags, activation='softmax'))(model)
## DEFINE INPUT AND OUTPUT
model_output = out
if args.use_words and args.use_chars:
model_input = [word_input, char_input]
elif args.use_words:
model_input = word_input
elif args.use_chars:
model_input = char_input
model = Model(input=model_input, output=model_output)
## COMPILE NETWORK
# a single loss function is employed
# the crf layer uses categorical cross-entropy as a loss function when in marginal mode
if args.output_activation == 'crf':
model_losses = [crf.loss_function]
else:
model_losses = [get_loss(args.loss)]
model_loss_weights = [1.0]
# define metrics
# we employ Keras default accuracy and our strict accuracy metric
if args.output_activation == 'crf':
model_metrics = [strict_accuracy_K]
else:
model_metrics = [strict_accuracy_K]
# define optimizer
model_opt = get_optimizer(args.optimizer)
# compilation
model.compile(optimizer=model_opt,
loss=model_losses,
loss_weights=model_loss_weights,
metrics=model_metrics)
# return the built model ready to be used
return model
def create_model(inp_shape, out_shape, jlogz=c.jlogz):
model = keras.models.Sequential()
activation = 'elu'
padding = 'same'
kernel_size = (3, 11)
model.add(Lambda(random_channel_flip, input_shape=inp_shape, output_shape=inp_shape))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=32, kernel_size=kernel_size, activation=activation, padding=padding, input_shape=inp_shape))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=32, kernel_size=kernel_size, strides=(2,2), activation=activation, padding=padding))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=64, kernel_size=kernel_size, activation=activation, padding=padding))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=64, kernel_size=kernel_size, strides=(2,2), activation=activation, padding=padding))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=128, kernel_size=kernel_size, activation=activation, padding=padding))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=128, kernel_size=kernel_size, strides=(2,2), activation=activation, padding=padding))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=64, kernel_size=kernel_size, activation=activation, padding=padding))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=64, kernel_size=kernel_size, activation=activation, padding=padding))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=32, kernel_size=kernel_size, activation=activation, padding=padding))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=32, kernel_size=kernel_size, activation=activation, padding=padding))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=16, kernel_size=kernel_size, activation=activation, padding=padding))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=16, kernel_size=kernel_size, activation=activation, padding=padding))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=8, kernel_size=kernel_size, activation=activation, padding=padding))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=8, kernel_size=kernel_size, activation=activation, padding=padding))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=4, kernel_size=kernel_size, activation=activation, padding=padding))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=4, kernel_size=kernel_size, activation=activation, padding=padding))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=2, kernel_size=kernel_size, activation=activation, padding=padding))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=2, kernel_size=kernel_size, activation=activation, padding=padding))
model.add(BatchNormalization())
model.add(GaussianNoise(0.1))
model.add(Conv2D(filters=1, kernel_size=(3, 15), activation='linear', padding="valid"))
model.add(Flatten())
model.add(Lambda(lambda x: K.tf.add(K.tf.multiply(x, K.variable(scale.squeeze)),
K.variable(mean.squeeze))))
return model
def model_tenth(X,
drop_ra=0.,
l1_reg=0.,
bias=0.,
g_noise=0.,
ker_init=None,
nodes=[32, 16, 5, 16, 32]):
#a list of layers, each comprised of a dictionary of layer elements
#K.clear_session()
input_dim = X.shape[1]
layers = []
layers.append({
'layer':
Dense(nodes[0],
input_dim=input_dim,
kernel_initializer=ker_init,
bias_initializer=Constant(value=bias)),
'advanced_activation':
PReLU(),
'noise':
GaussianNoise(g_noise)
})
layers.append({
'layer':
Dense(nodes[1],
kernel_initializer=ker_init,
bias_initializer=Constant(value=bias)),
'advanced_activation':
PReLU(),
'noise':
GaussianNoise(g_noise),
'dropout_rate':
drop_ra,
'normalisation':
BatchNormalization()
})
layers.append({'layer': MaxoutDense(nodes[2], init=ker_init)})
layers.append({
'layer':
Dense(nodes[3],
bias_initializer=Constant(value=bias),
kernel_initializer=ker_init),
'advanced_activation':
PReLU(),
'dropout_rate':
drop_ra
})
layers.append({
'layer':
Dense(nodes[4],
bias_initializer=Constant(value=bias),
kernel_initializer=ker_init),
'advanced_activation':
PReLU(),
'dropout_rate':
drop_ra
})
layers.append({
'layer':
Dense(input_dim,
kernel_regularizer=l1(l1_reg),
kernel_initializer=ker_init)
})
return layers
def __init__(self, start_target_size=(672, 4), mode='vanilla'):
#set of conv blocks wrapper
def conv_block(x, dim):
x1 = Conv1D(dim,
kernel_size=1,
strides=1,
padding='same',
activation='relu')(x)
x1 = BatchNormalization()(x1)
x1 = Conv1D(dim,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x1)
x1 = BatchNormalization()(x1)
return x1
# set of separable conv wrapper
def sepa_conv_block(x, dim):
x1 = SeparableConv1D(dim,
kernel_size=1,
strides=1,
padding='same',
activation='relu')(x)
x1 = BatchNormalization()(x1)
x1 = SeparableConv1D(dim,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x1)
x1 = BatchNormalization()(x1)
return x1
# dense block wrapper
def dense_block(inlayer, convs, dims):
conv_list = []
ministem = conv_block(
inlayer, dims) if mode != 'separable' else sepa_conv_block(
inlayer, dims)
ministem = BatchNormalization()(ministem)
conv_list.append(ministem)
ministem = conv_block(
conv_list[0],
dims) if mode != 'separable' else sepa_conv_block(
conv_list[0], dims)
ministem = BatchNormalization()(ministem)
conv_list.append(ministem)
for _ in range(convs - 2):
x = Concatenate()([layer for layer in conv_list])
x = conv_block(
x, dims) if mode != 'separable' else sepa_conv_block(
x, dims)
x = BatchNormalization()(x)
conv_list.append(x)
return conv_list[-1]
## build our model
# stem
inputs = Input(shape=start_target_size)
x = GaussianNoise(0.3)(inputs)
x = Conv1D(512,
kernel_size=7,
strides=2,
padding='same',
activation='relu')(x)
x = MaxPooling1D(pool_size=3, strides=2)(x)
# dense block 1
d1 = dense_block(x, 6, 64)
#transition
t = Conv1D(64,
kernel_size=1,
strides=1,
padding='same',
activation='relu')(d1)
t = MaxPooling1D(pool_size=2, strides=2)(t)
# dense block 2
d2 = dense_block(t, 12, 64)
# optional depth, doesn't seem to help
'''
#transition
t2 = Conv1D(64, kernel_size=1, strides=1, padding='same', activation='relu')(d2)
t2 = AveragePooling1D(pool_size=2, strides=2)(t2)
# dense block 2
d3 = dense_block(t2, 6, 64)
'''
# exit stem
fc = Conv1D(64,
kernel_size=1,
strides=1,
padding='same',
activation='relu')(d2)
if mode == 'long stem':
fc = MaxPooling1D(pool_size=2, strides=2)(fc)
fc = Flatten()(fc)
fc = Dense(1024, activation='relu')(fc)
fc = Dropout(0.5)(fc)
predictions = Dense(1, activation='sigmoid')(fc)
model = Model(inputs=inputs, outputs=predictions)
model.compile(loss='binary_crossentropy',
optimizer=SGD(lr=1e-3, momentum=0.9),
metrics=['binary_accuracy'])
model.summary()
self.model = model
self.mode = mode
def main(nb_epoch=1,
data_augmentation=True,
noise=True,
maxout=True,
dropout=True,
l1_reg=False,
l2_reg=True,
max_pooling=True,
deep=False,
noise_sigma=0.01):
# l1 and l2 regularization shouldn't be true in the same time
if l1_reg and l2_reg:
print("No need to run l1 and l2 regularization in the same time")
quit()
# print settings for this experiment
print("number of epoch: {0}".format(nb_epoch))
print("data augmentation: {0}".format(data_augmentation))
print("noise: {0}".format(noise))
print("sigma: {0}".format(sigma))
print("maxout: {0}".format(maxout))
print("dropout: {0}".format(dropout))
print("l1: {0}".format(l1_reg))
print("l2: {0}".format(l2_reg))
print("max_pooling: {0}".format(max_pooling))
print("deep: {0}".format(deep))
# the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = cifar100.load_data()
# split the validation dataset
X_train, X_valid, y_train, y_valid = train_test_split(X_train,
y_train,
test_size=0.2,
random_state=0)
# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_valid = np_utils.to_categorical(y_valid, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
X_train = X_train.astype('float32')
X_valid = X_valid.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_valid /= 255
X_test /= 255
##### try loading data using data_loader.py ####
# data_loader.download_and_extract(data_path, data_url)
# class_names = data_loader.load_class_names()
# print(class_names)
# images_train, cls_train, labels_train = data_loader.load_training_data()
# images_test, cls_test, labels_test = data_loader.load_test_data()
# X_train, Y_train = images_train, labels_train
# X_test, Y_test = images_test, labels_test
# X_train, X_valid, Y_train, Y_valid = train_test_split(X_train, Y_train, test_size=0.2, random_state=0)
print("Size of:")
print("- Training-set:\t\t{}".format(len(X_train)))
print("- Validation-set:\t\t{}".format(len(X_valid)))
print("- Test-set:\t\t{}".format(len(X_test)))
model = Sequential()
if noise:
model.add(
GaussianNoise(noise_sigma,
input_shape=(img_channels, img_rows, img_cols)))
model.add(
Convolution2D(32,
3,
3,
border_mode='same',
input_shape=(img_channels, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(Convolution2D(32, 3, 3))
model.add(Activation('relu'))
if max_pooling:
model.add(MaxPooling2D(pool_size=(2, 2)))
if dropout:
model.add(Dropout(0.25))
if max_pooling:
model.add(MaxPooling2D(pool_size=(2, 2)))
if dropout:
model.add(Dropout(0.25))
model.add(Flatten())
if maxout:
model.add(MaxoutDense(512, nb_feature=4, init='glorot_uniform'))
else:
if not (l1_reg or l2_reg):
model.add(Dense(512))
# activation regularization not implemented yet
if l1_reg:
model.add(Dense(512, W_regularizer=l1(l1_weight)))
elif l2_reg:
model.add(Dense(512, W_regularizer=l2(l2_weight)))
model.add(Activation('relu'))
if dropout:
model.add(Dropout(0.5))
if deep:
model.add(Dense(512))
model.add(Dense(512))
model.add(Dense(512))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
# let's train the model using SGD + momentum (how original).
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
start_time = time.time()
if not data_augmentation:
his = model.fit(X_train,
Y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
validation_data=(X_valid, Y_valid),
shuffle=True)
else:
# this will do preprocessing and realtime data augmentation
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=
False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=True, # apply ZCA whitening
rotation_range=
0, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=
0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=
0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=False) # randomly flip images
# compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied)
datagen.fit(X_train)
# fit the model on the batches generated by datagen.flow()
his = model.fit_generator(datagen.flow(X_train,
Y_train,
batch_size=batch_size),
samples_per_epoch=X_train.shape[0],
nb_epoch=nb_epoch,
validation_data=(X_valid, Y_valid))
# evaluate our model
score = model.evaluate(X_test, Y_test, verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])
print('training time', time.time() - start_time)
file_path = os.path.join(output_path, output_directory)
print("outputs should be store at %s" % file_path)
# Check if the file already exists.
# If it exists then we assume it has also been extracted,
# otherwise we need to download and extract it now.
if not os.path.exists(file_path):
print("creat output directory fro storing output")
# Check if the download directory exists, otherwise create it.
os.makedirs(file_path)
# wirte test accuracy to a file
output_file_name = os.path.join(
file_path,
'train_val_loss_with_dropout_epochs_{0}_data_augmentation_{1}_noise_{2}_sigma{12}_maxout_{3}_dropout_{4}_l1_{5}_l2_{6}_sigma_{7}_l1weight_{8}_l2weight_{9}_maxout_{10}_deep_{11}.txt'
.format(nb_epoch, data_augmentation, noise, maxout, dropout, l1_reg,
l2_reg, sigma, l1_weight, l2_weight, maxpooling, deep, sigma))
print("save file at {}".output_file_name)
with open(output_file_name, "w") as text_file:
text_file.write('Test score: {}\n'.format(score[0]))
text_file.write('Test accuracy: {}\n'.format(score[1]))
text_file.write('Training time: {}\n'.format(time.time() - start_time))
text_file.close()
# visualize training history
train_loss = his.history['loss']
val_loss = his.history['val_loss']
plt.plot(range(1,
len(train_loss) + 1),
train_loss,
color='blue',
label='train loss')
plt.plot(range(1,
len(val_loss) + 1),
val_loss,
color='red',
label='val loss')
plt.legend(loc="upper left", bbox_to_anchor=(1, 1))
plt.xlabel('#epoch')
plt.ylabel('loss')
output_fig_name = os.path.join(
file_path,
'train_val_loss_with_dropout_epochs_{0}_data_augmentation_{1}_noise_{2}_sigma{12}_maxout_{3}_dropout_{4}_l1_{5}_l2_{6}_sigma_{7}_l1weight_{8}_l2weight_{9}_maxout_{10}_deep_{11}.png'
.format(nb_epoch, data_augmentation, noise, maxout, dropout, l1_reg,
l2_reg, sigma, l1_weight, l2_weight, maxpooling, deep, sigma))
plt.savefig(output_fig_name, dpi=300)
plt.show()
def fcn_model(colorshape=[3, 384, 512]):
cls = 5
color_input = Input(shape=colorshape)
x = GaussianNoise(0.1)(color_input)
# use VGG
# Block 1
x = Convolution2D(64,
3,
3,
activation='relu',
border_mode='same',
name='block1_conv1')(x)
x = Convolution2D(64,
3,
3,
activation='relu',
border_mode='same',
name='block1_conv2')(x)
mp1 = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
# Block 2
x = Convolution2D(128,
3,
3,
activation='relu',
border_mode='same',
name='block2_conv1')(mp1)
x = Convolution2D(128,
3,
3,
activation='relu',
border_mode='same',
name='block2_conv2')(x)
mp2 = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
# Block 3
x = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
name='block3_conv1')(mp2)
x = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
name='block3_conv2')(x)
x = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
name='block3_conv3')(x)
mp3 = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
# Block 4
x = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='block4_conv1')(mp3)
x = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='block4_conv2')(x)
x = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='block4_conv3')(x)
mp4 = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
# Block 5
x = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='block5_conv1')(mp4)
x = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='block5_conv2')(x)
x = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='block5_conv3')(x)
mp5 = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
# reduce channel
x = Convolution2D(4096,
7,
7,
activation='relu',
border_mode='same',
name='fc6')(mp5) # => [?, 4096, 12, 16]
x = Dropout(0.5)(x)
x = Convolution2D(4096,
1,
1,
activation='relu',
border_mode='same',
name='fc7')(x) # => [?, 4096, 12, 16]
x = Dropout(0.5)(x)
# Deconv Layer
x = Convolution2D(cls, 4, 4, activation='relu',
border_mode='same')(x) # => [?, 21, 12, 16]
x = UpSampling2D()(x) # => [?, 21, 24, 16]
x = merge([x, mp4], mode='concat', concat_axis=-3)
x = Convolution2D(cls, 4, 4, activation='relu',
border_mode='same')(x) # => [?, 21, 12, 16]
x = UpSampling2D()(x) # => [?, 21, 24, 16]
x = merge([x, mp3], mode='concat', concat_axis=-3)
x = Convolution2D(cls, 4, 4, activation='relu',
border_mode='same')(x) # => [?, 21, 12, 16]
x = UpSampling2D()(x) # => [?, 21, 24, 16]
x = merge([x, mp2], mode='concat', concat_axis=-3)
x = Convolution2D(cls, 4, 4, activation='relu',
border_mode='same')(x) # => [?, 21, 12, 16]
x = UpSampling2D()(x) # => [?, 21, 24, 16]
x = merge([x, mp1], mode='concat', concat_axis=-3)
x = Convolution2D(cls, 4, 4, activation='relu',
border_mode='same')(x) # => [?, 21, 12, 16]
x = UpSampling2D()(x) # => [?, 21, 24, 16]
x = merge([x, color_input], mode='concat', concat_axis=-3)
y = Convolution2D(cls, 4, 4, activation='sigmoid',
border_mode='same')(x) # => [?, 21, 12, 16]
model = Model(input=color_input, output=y)
return model
X_train = scaler.transform(X_train) X_val = scaler.transform(X_val) # One hot-encoding #enc = preprocessing.OneHotEncoder(sparse=False).fit(t_train) #t_train = enc.transform(t_train) #t_val = enc.transform(t_val) #%% TRAIN FFNN lr = 0.05 momentum = 0.9 inputs = Input(shape=(X.shape[1], )) x = GaussianNoise(1.0)(inputs) x = Dense(25, activation='linear')(x) encoded = Dense(2, activation='linear')(x) x = Dense(25, activation='linear')(encoded) decoded = Dense(X_train.shape[1], activation='linear')(x) autoencoder = Model(inputs, decoded, name='Autoencoder') encoder = Model(inputs, encoded) # Optimizer sgd = SGD(lr=lr, momentum=momentum, nesterov=True)
# The data, shuffled and split between train and test sets:
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = np.mean(x_train, 3, keepdims=True)
x_test = np.mean(x_test, 3, keepdims=True)
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
# Convert class vectors to binary class matrices.
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
model = Sequential()
#model.add(UpSampling2D(size=(1, 1), data_format=None, input_shape=x_train.shape[1:]))
model.add(GaussianNoise(noise_start, input_shape=x_train.shape[1:]))
'''
if bottleneck_mode == 'append_retina':
#RETINA net
model.add(Conv2D(2, (5, 5), strides=(5, 5), padding='same', input_shape=x_train.shape[1:]))
model.add(Activation('tanh'))
model.add(Conv2D(30, (5, 5), strides=(11, 11)))
model.add(Activation('tanh'))
model.add(GaussianNoise(noise_end))
#INVERSE RETINA net
model.add(Conv2DTranspose(30, (5, 5), strides=(11, 11)))
model.add(Activation('tanh'))
model.add(Conv2DTranspose(30, (5, 5), strides=(5, 5)))
model.add(Activation('tanh'))
def model7():
model = Sequential()
model.add(
Conv2D(filters=16,
kernel_size=(3, 3),
input_shape=(48, 48, 1),
padding='same'))
model.add(BatchNormalization(axis=-1, momentum=0.5))
model.add(LeakyReLU())
model.add(Conv2D(filters=32, kernel_size=(3, 3), padding='same'))
model.add(GaussianNoise(0.1))
model.add(BatchNormalization(axis=-1, momentum=0.5))
model.add(LeakyReLU())
model.add(Conv2D(filters=64, kernel_size=(3, 3), padding='same'))
model.add(BatchNormalization(axis=-1, momentum=0.5))
model.add(LeakyReLU())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.1))
model.add(Conv2D(filters=128, kernel_size=(3, 3), padding='same'))
model.add(BatchNormalization(axis=-1, momentum=0.5))
model.add(LeakyReLU())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(Conv2D(filters=256, kernel_size=(3, 3), padding='same'))
model.add(BatchNormalization(axis=-1, momentum=0.5))
model.add(LeakyReLU())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(Conv2D(filters=512, kernel_size=(3, 3), padding='same'))
model.add(Conv2D(filters=512, kernel_size=(3, 3), padding='same'))
model.add(BatchNormalization(axis=-1, momentum=0.5))
model.add(LeakyReLU())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(BatchNormalization(axis=-1, momentum=0.5))
model.add(LeakyReLU())
model.add(Dropout(0.5))
model.add(Dense(512))
model.add(LeakyReLU())
model.add(Dense(7))
model.add(Activation('softmax'))
adam = Adam(lr=1e-3, decay=5e-6)
model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=['accuracy'])
datagen = ImageDataGenerator(rotation_range=30.0,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.1,
zoom_range=0.2,
horizontal_flip=True,
fill_mode='constant',
vertical_flip=False)
return model, datagen
input_signal = Input(shape=(M, ))
#encoded01 = Dense(M, activation='linear')(input_signal)
encoded = Dense(M,
activation='relu')(input_signal) # Transmitter: signal input
encoded1 = Dense(n_channel, activation='linear')(
encoded) # Transmitter: n_Channel transform
#encoded2 = Lambda(lambda x: x / K.sqrt(K.mean(x**2)))(encoded1) # from Berke
encoded2 = Lambda(lambda x: np.sqrt(n_channel) * K.l2_normalize(x, axis=1))(
encoded1) #from T O'Shea
# Transmitter: normalization
# Channel: AWGN
SNR_train = 10 # coverted SNR in dB
EbNo_train = 1 / 2 / k * 10.0**(SNR_train / 10.0)
# EbNo_train = 5.01187 # coverted 7 db of EbNo
encoded3 = GaussianNoise(np.sqrt(1 / (2 * R * EbNo_train)))(encoded2)
# Training the autoencoder at 7dB, while it is used in SNR range [-4 8.5] below.
# Receiver
#decoded01= Dense(M, activation='linear')(encoded3)
decoded = Dense(M, activation='relu')(encoded3)
decoded1 = Dense(M, activation='softmax')(decoded)
# output is the probability of all elements suming as 1
# From TX to RX
autoencoder = Model(input_signal, decoded1)
# adam = Adam(lr=0.001) # learning rate
autoencoder.compile(
optimizer='adam',
# loss='mean_squared_error')
loss='categorical_crossentropy')
def build_attention_RNN(embeddings,
classes,
max_length,
unit=LSTM,
cells=64,
layers=1,
**kwargs):
"""Builds RNN model with attention based on given parameters"""
# parameters
bi = kwargs.get("bidirectional", False)
noise = kwargs.get("noise", 0.)
dropout_words = kwargs.get("dropout_words", 0)
dropout_rnn = kwargs.get("dropout_rnn", 0)
dropout_rnn_U = kwargs.get("dropout_rnn_U", 0)
dropout_attention = kwargs.get("dropout_attention", 0)
dropout_final = kwargs.get("dropout_final", 0)
attention = kwargs.get("attention", None)
final_layer = kwargs.get("final_layer", False)
clipnorm = kwargs.get("clipnorm", 1)
loss_l2 = kwargs.get("loss_l2", 0.)
lr = kwargs.get("lr", 0.001)
model = Sequential()
model.add(
embeddings_layer(max_length=max_length,
embeddings=embeddings,
trainable=False,
masking=True,
scale=False,
normalize=False))
if noise > 0:
model.add(GaussianNoise(noise))
if dropout_words > 0:
model.add(Dropout(dropout_words))
for i in range(layers):
rs = (layers > 1 and i < layers - 1) or attention
model.add(
get_RNN(unit,
cells,
bi,
return_sequences=rs,
dropout_U=dropout_rnn_U))
if dropout_rnn > 0:
model.add(Dropout(dropout_rnn))
if attention == "memory":
model.add(AttentionWithContext())
if dropout_attention > 0:
model.add(Dropout(dropout_attention))
if attention == "simple":
model.add(Attention())
if dropout_attention > 0:
model.add(Dropout(dropout_attention))
if final_layer:
model.add(MaxoutDense(100, W_constraint=maxnorm(2)))
if dropout_final > 0:
model.add(Dropout(dropout_final))
model.add(Dense(classes, activity_regularizer=l2(loss_l2)))
model.add(Activation("softmax"))
model.compile(optimizer=Adam(clipnorm=clipnorm, lr=lr),
loss="categorical_crossentropy")
return model
x_test_std = Scaler.transform(x_test)
test_pine = Scaler.transform(test_pine)
test_corn = Scaler.transform(test_corn)
test_coal = Scaler.transform(test_coal)
else:
x_train_std = x_train
#------------------------------------------------------------------------------
# build network
#------------------------------------------------------------------------------
#from keras.layers.advanced_activations import PReLU
input_dim = x_train_std.shape[1]
feature = Input(shape = (input_dim, 1))
x = GaussianNoise(0.1)(feature)
#
# =============================================================================
# x = GaussianNoise(0.01)(feature)
# x = Conv1D(filters= 8, kernel_size = 4, strides=4, padding='valid',
# activation='relu',name = 'conv1D_1')(x)
# x = MaxPooling1D(pool_size=2, strides=2, name = 'MP_1')(x)
# x = Flatten(name = 'flat_1')(x)
#
# x_x = GaussianNoise(0.01)(feature)
# x_x = Conv1D(filters= 12, kernel_size = 6, strides= 6, padding='valid',
# activation='relu',name = 'conv1D_2')(x_x)
# x_x = MaxPooling1D(pool_size=2, strides=2, name = 'MP_2')(x_x)
# x_x = Flatten(name = 'flat_2')(x_x)
#
# x_x_x = GaussianNoise(0.01)(feature)
