def create_model(train_data):
model = Sequential()
model.add(Flatten(input_shape=train_data.shape[1:]))
model.add(Dense(128, activation='sigmoid', name='dense'))
model.add(Dropout(0.75))
model.add(
Dense(config.num_classes,
activation='sigmoid',
kernel_regularizer=L1L2(0, 0.01),
activity_regularizer=L1L2(0, 0.01),
name='out'))
#optimizer = Adam(lr=0.01)
optimizer = RMSprop(lr=0.005)
#optimizer = SGD(lr=0.005, decay=1e-6, momentum=0.9, nesterov=True)
# In Keras we need to compile the model so it can be trained.
model.compile(optimizer=optimizer,
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def lstm_keras(inp_dim,
vocab_size,
embed_size,
use_word_embeddings=False,
embedding_matrix=None,
embedding_trainable=False):
# K.clear_session()
model = Sequential()
if use_word_embeddings == True:
model.add(
Embedding(vocab_size,
embed_size,
weights=[embedding_matrix],
input_length=inp_dim,
trainable=embedding_trainable))
else:
model.add(
Embedding(vocab_size,
embed_size,
input_length=inp_dim,
trainable=True))
model.add(Dropout(0.25))
model.add(
LSTM(embed_size,
kernel_regularizer=L1L2(l1=0.0, l2=0.00000001),
bias_regularizer=L1L2(l2=0.00000001)))
model.add(Dropout(0.50))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
#print (model.summary())
return model
def build_module(self):
'''
Build the module from the initialized arguments above
# Returns
---------
collection: list
list of layers according to the specified hyperparameters
'''
collection = []
if self.timedistributed:
for i in range(self.layers):
tmp = TimeDistributed(Dense(self.sizes[i]))
activation = self.activations[i]
collection.append(tmp)
collection.append(activation)
if self.dropout:
tmp = Dropout(0.25)
collection.append(tmp)
if (self.L1 + self.L2) > 0.0:
tmp = L1L2(self.L1, self.L2)
collection.append(tmp)
else:
for i in range(self.layers):
tmp = Dense(self.sizes[i], activation=self.activations[i])
collection.append(tmp)
if self.dropout:
tmp = Dropout(0.3)
collection.append(tmp)
if (self.L1 + self.L2) > 0.0:
tmp = L1L2(self.L1, self.L2)
collection.append(tmp)
return collection
def run():
# load dataset
series = read_csv('shampoo-sales.csv',
header=0,
parse_dates=[0],
index_col=0,
squeeze=True,
date_parser=parser)
# configure the experiment
n_lag = 1
n_repeats = 30
n_epochs = 1000
n_batch = 4
n_neurons = 3
regularizers = [
L1L2(l1=0.0, l2=0.0),
L1L2(l1=0.01, l2=0.0),
L1L2(l1=0.0, l2=0.01),
L1L2(l1=0.01, l2=0.01)
]
# run the experiment
results = DataFrame()
for reg in regularizers:
name = ('l1 %.2f,l2 %.2f' % (reg.l1, reg.l2))
results[name] = experiment(series, n_lag, n_repeats, n_epochs, n_batch,
n_neurons, reg)
# summarize results
print(results.describe())
# save boxplot
results.boxplot()
pyplot.savefig('experiment_reg_bias.png')
def tune_weight_regularization(train_X, train_y, validation_X, validation_y):
# define scope of search
regularizers = {
1: L1L2(l1=0.0, l2=0.01),
2: L1L2(l1=0.01, l2=0.0),
3: L1L2(l1=0.0, l2=0.0),
4: L1L2(l1=0.01, l2=0.01)
}
n_repeats = 5
# grid search parameter values
scores = DataFrame()
for reg in regularizers.keys():
# repeat each experiment multiple times
loss_values = list()
for i in range(n_repeats):
loss = fit_weight_regularization_model(regularizers[reg], train_X,
train_y, validation_X,
validation_y)
loss_values.append(loss)
print('>%d/%d param=%f, loss=%f' % (i + 1, n_repeats, reg, loss))
# store results for this parameter
scores[str(reg)] = loss_values
# summary statistics of results
print(scores.describe())
# box and whisker plot of results
scores.boxplot()
pyplot.show()
def experiment():
dataset = pd.read_csv(
r'G:\Final Dev\12_Classifier\Data Merged\train_file.csv', header=None)
neuron = 26
n_epochs = 5
n_batch = 144
k_fold = 3
x, y = read_data(dataset)
results = pd.DataFrame()
regularizer = [
L1L2(l1=0.0, l2=0.0),
L1L2(l1=0.001, l2=0.0),
L1L2(l1=0.0, l2=0.001),
L1L2(l1=0.001, l2=0.001)
]
for reg in regularizer:
name = ("l1 %.3f l2 %.3f" % (reg.l1, reg.l2))
results[name] = model_fit(x, y, neuron, n_epochs, n_batch, k_fold, reg)
print(results)
print(results.describe())
stop = timeit.default_timer()
print('Time :', stop - start)
# save boxplot
results.boxplot()
plt.ylabel('Accuracy')
plt.xlabel('Regularizer')
Box_Name = input("Nama Figure: ")
plt.savefig(r'G:\Final Dev\12_Classifier\Box Plot Model\%s.png' % Box_Name)
def default_model(n_fft):
"""
This model is a bit too large for Tegra, but it is proven
"""
assert n_fft == 257, "Default model cannot handle non-257 fft sizes"
input_lower = Input((None, 257), name="input_lf")
layer = Lambda(K.expand_dims)(input_lower)
layer = LeakyReLU(0.01)(Conv2D(12, kernel_size=(9, 1),
activation='linear')(layer))
layer = LeakyReLU(0.01)(Conv2D(12, kernel_size=(1, 5),
activation='linear')(layer))
layer = LeakyReLU(0.01)(Conv2D(12, kernel_size=(9, 1),
activation='linear')(layer))
layer = LeakyReLU(0.01)(Conv2D(12, kernel_size=(1, 5),
activation='linear')(layer))
layer = TimeDistributed(Flatten())(layer)
layer = LeakyReLU(0.01)(Dense(1024,
kernel_regularizer=L1L2(l1=1e-5))(layer))
layer = LeakyReLU(0.01, name='hidden')(Dense(
512, kernel_regularizer=L1L2(l1=1e-5))(layer))
layer = LeakyReLU(0.01)(Dense(350,
kernel_regularizer=L1L2(l2=1e-5))(layer))
layer = Dense(257)(layer)
mdl = Model(input_lower, layer)
mdl.summary()
return mdl
def fit(self, X, y, weights):
input_shape = X.shape[1]
self.reg.add(
Dense(int(np.floor(input_shape / 2)),
input_dim=input_shape,
activation='relu',
kernel_regularizer=L1L2(l1=0.0, l2=0.1)))
self.reg.add(Dense(1, kernel_regularizer=L1L2(l1=0.0, l2=0.1)))
self.reg.compile(optimizer='adam', loss='mean_squared_error')
self.reg.fit(X, y, sample_weight=weights, verbose=True)
def create_model(self):
lstm_units = 100
# Comment input
encoder_inputs = Input(shape=(self.comlen, ))
encoder_embedding = Embedding(output_dim=10,
input_dim=self.comvocabsize,
mask_zero=False)(encoder_inputs)
encoder = Bidirectional(
CuDNNLSTM(lstm_units,
return_state=True,
bias_regularizer=L1L2(0.01, 0.0)))
encoder_outputs, forward_h, forward_c, backward_h, backward_c = encoder(
encoder_embedding)
state_h = concatenate([forward_h, backward_h])
state_c = concatenate([forward_c, backward_c])
encoder_states = [state_h, state_c]
# Tag input
decoder_inputs = Input(shape=(self.taglen, ))
decoder_embedding = Embedding(output_dim=2,
input_dim=self.tagvocabsize,
mask_zero=False)(decoder_inputs)
decoder_lstm = CuDNNLSTM(lstm_units * 2,
return_sequences=True,
return_state=True,
bias_regularizer=L1L2(0.01, 0.0))
decoder_outputs, _, _ = decoder_lstm(decoder_embedding,
initial_state=encoder_states)
#decoder_dropout = Dropout(0.5)(decoder_outputs)
#decoder_d = TimeDistributed(Dense(400))(decoder_dropout)
decoder_dense = Dense(self.tagvocabsize, activation='softmax')
decoder_outputs = decoder_dense(decoder_outputs)
train_model = Model(inputs=[encoder_inputs, decoder_inputs],
outputs=decoder_outputs)
# Define Inference Model
# Inference Encoder
encoder_model = Model(encoder_inputs, encoder_states)
# Inference Decoder
decoder_state_h = Input(shape=(lstm_units * 2, ))
decoder_state_c = Input(shape=(lstm_units * 2, ))
decoder_state_inputs = [decoder_state_h, decoder_state_c]
decoder_outputs, state_h, state_c = decoder_lstm(
decoder_embedding, initial_state=decoder_state_inputs)
#decoder_dropout = Dropout(0.5)(decoder_outputs)
#decoder_d = TimeDistributed(Dense(400))(decoder_dropout)
decoder_states = [state_h, state_c]
decoder_outputs = decoder_dense(decoder_outputs)
decoder_model = Model([decoder_inputs] + decoder_state_inputs,
[decoder_outputs] + decoder_states)
return train_model, encoder_model, decoder_model
def build_model(stateful, batch_size=None):
i = Input(shape=IN_SHAPE[1:])
i = Permute((1, 3, 4, 2), batch_input_shape=IN_SHAPE)(i)
R = representation_rnn()
C = consciousness_rnn()
G = generator_rnn()
D = decoder_rnn(nb_actions)
h = R(i) # Get h from R
c_A, c_B, c_A_soft, c_B_soft = C(h) # Get masks c_A and c_B from C
b = multiply([h, c_B],
name='b') # Get b through elementwise multiplication
a_hat = G([c_A, c_B, b]) # Send c_A, c_B and b to G to get a_hat
a_hat = Lambda(lambda x: x[:, :-1, :], output_shape=(None, latent_dim))(
a_hat) # Slice dimensions to align vectors
h_A = Lambda(lambda x: x[:, 1:, :], output_shape=(None, latent_dim))(
h) # Slice dimensions to align vectors
c_A = Lambda(lambda x: x[:, :-1, :], output_shape=(None, latent_dim))(
c_A) # Slice dimensions to align vectors
h_A = multiply([h_A, c_A]) # Calculate h[A] to compare against a_hat
a_hat = multiply([a_hat, c_A]) # Mask a_hat
consciousness_error = subtract([a_hat, h_A])
consciousness_error = Regularize(
L1L2(l1=0., l2=1. * reg_lambda),
name='Consciousness_Generator_Error')(consciousness_error)
b_transformed = Dense(latent_dim, activation='linear')(
b) # Create a layer that attempts to make b independent from h[A]
b_transformed = Lambda(lambda x: x[:, :-1, :],
output_shape=(None, latent_dim))(b_transformed)
b_transformed = multiply([b_transformed, c_A])
transformation_error = subtract([b_transformed, h_A])
transformation_error = Regularize(
L1L2(l1=0., l2=1. * reg_lambda),
name='Transformation_Error')(transformation_error)
intelligence_error = concatenate([
c_A_soft, c_B_soft
]) # The more elements we choose to predict, the more "intelligent" we are
intelligence_error = Flatten()(intelligence_error)
intelligence_error = Regularize(
LinearRegularizer(c=1. * reg_lambda),
name='Intelligence_Level')(intelligence_error)
x_hat = D(a_hat)
x_hat = ApplyRegularization()(
[x_hat, consciousness_error, transformation_error, intelligence_error])
## Compile the model and start training
CN = Model(inputs=i, outputs=[x_hat])
return CN
def lstm_bert(embed_size):
model = Sequential()
model.add(
LSTM(embed_size,
kernel_regularizer=L1L2(l1=0.0, l2=0.00000001),
bias_regularizer=L1L2(l2=0.00000001)))
model.add(Dropout(0.50))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def build(self, input_shape):
self.W = self.add_weight(name='Attention_Dot_Weight',
shape=(input_shape[1], input_shape[1]),
regularizer=L1L2(0.0000032),
initializer='uniform',
trainable=True)
self.b = self.add_weight(name='Attention_Dot_Bias',
regularizer=L1L2(0.00032),
shape=(input_shape[1], ),
initializer='uniform',
trainable=True)
super().build(input_shape)
def setup(self, input_dim: tuple, **kwargs) -> None:
"""
Method that builds the NN architecture
:param input_dim: The dimension of the inputs
:param kwargs: Other arguments. Not used here
:return: -
"""
assert not self.built
# Defining layers
batchNormLayer1 = BatchNormalization(input_shape=input_dim)
dropoutInputLayer = Dropout(rate=.2)
denseLayer1 = Dense(32,
activation='relu',
kernel_regularizer=L1L2(l1=.0, l2=.1),
kernel_initializer='uniform')
bathNormLayer2 = BatchNormalization()
dropoutDenseLayer1 = Dropout(rate=.2)
denseLayer2 = Dense(16,
activation='relu',
kernel_regularizer=L1L2(l1=.0, l2=.1),
kernel_initializer='uniform')
bathNormLayer3 = BatchNormalization()
dropoutDenseLayer2 = Dropout(rate=.2)
outputLayer = Dense(2,
activation='softmax',
kernel_initializer='uniform')
# adding the layers
self.model.add(batchNormLayer1)
self.model.add(dropoutInputLayer)
self.model.add(denseLayer1)
self.model.add(bathNormLayer2)
self.model.add(dropoutDenseLayer1)
self.model.add(denseLayer2)
self.model.add(bathNormLayer3)
self.model.add(dropoutDenseLayer2)
self.model.add(outputLayer)
# compiling the model
self.model.compile(optimizer=Adam(),
loss='categorical_crossentropy',
metrics=['accuracy'])
self.built = True
def model_CNN_LSTM_layers():
'''
returns a CNN+LSTM model with 2 conv and 3 lstm layers.
Parameters
----------
None
Returns
-------
model: keras model with the architecture described
'''
model = Sequential()
model.add(
TimeDistributed(Conv1D(filters=32,
kernel_size=2,
padding='valid',
activation='relu'),
input_shape=(None, nTimeSteps, nFeatures)))
model.add(TimeDistributed(Dropout(0.05)))
model.add(
TimeDistributed(Conv1D(filters=64,
kernel_size=2,
padding='valid',
activation='relu'),
input_shape=(None, nTimeSteps, nFeatures)))
model.add(TimeDistributed(Dropout(0.05)))
model.add(TimeDistributed(Flatten()))
model.add(
LSTM(50,
return_sequences=True,
activation='relu',
dropout=0.2,
kernel_regularizer=L1L2(l1=0.01, l2=0.01)))
model.add(
LSTM(50,
return_sequences=True,
activation='relu',
dropout=0.2,
kernel_regularizer=L1L2(l1=0.01, l2=0.01)))
model.add(LSTM(30))
model.add(Dense(1, activation='tanh')) #Final activation tanh function
print("\nModel Architecture \n")
model.summary()
model.compile(loss='mean_absolute_error',
optimizer=keras.optimizers.Adam(learning_rate=0.001),
metrics=['accuracy'])
return model
def main():
# load data set
open_db()
origin = raw_input("Origin: ")
destination = raw_input("Destination: ")
end = raw_input("End Date (MM/YY): ")
end_date = "20" + end[3:6] + "-" + end[0:2]
segments = concatenate_segment(origin, destination, end_date)
series = segments[pd.notnull(segments.passengers)].passengers
# configure the experiment
n_lag = 1
n_repeats = 10
n_epochs = 1000
n_batch = (len(series) - 2) % 12
n_neurons = 3
n_dropout = [0.0, 0.2, 0.4, 0.6]
input_regularizers = [
L1L2(l1=0.0, l2=0.0),
L1L2(l1=0.01, l2=0.0),
L1L2(l1=0.0, l2=0.01),
L1L2(l1=0.01, l2=0.01)
]
# run the experiment
results = pd.DataFrame()
for dropout in n_dropout:
for input_reg in input_regularizers:
name = ('l1 %.2f,l2 %.2f - dropout: %.2f' %
(input_reg.l1, input_reg.l2, dropout))
results[name] = experiment(series, n_lag, n_repeats, n_epochs,
n_batch, n_neurons, input_reg, dropout)
"""
n_dim = series.shape[1]
for dropout in n_dropout:
for input_reg in input_regularizers:
name = ('l1 %.2f,l2 %.2f - dropout: %.2f' % (input_reg.l1, input_reg.l2, dropout))
results[name] = experiment(series, n_lag, n_repeats, n_epochs, n_batch, n_neurons, input_reg, dropout)
"""
# summarize results
print(results.describe())
# save boxplot
pyplot.title("%s - %s" % (origin, destination))
results.boxplot()
pyplot.savefig('~/experiment_baseline.png')
pyplot.figure()
pyplot.boxplot(results)
def generate_model2(con_cols, lstm_list, M, cells):
# Initialize input
k = 0
inputs = [[], [], []]
for m in M:
inputs[0].append(Input(shape=(None, 1)))
inputs[1].append(Embedding(m[0], min(m[0], m[1]))(inputs[0][-1]))
inputs[2].append(Reshape((-1, min(m[0], m[1])))(inputs[1][-1]))
k += min(m[0], m[1])
cont_input = Input(shape=(None, len(con_cols)), name='cont_input')
inputs[2].append(cont_input)
# input concatenation
concat1 = Concatenate(name='profile_concat')(inputs[2])
# LSTM layers
lstm_input = Input(shape=(None, len(lstm_list)))
k += len(lstm_list)
lstm = LSTM(cells,
return_sequences=True,
input_shape=(None, k),
stateful=False,
dropout=0.1,
recurrent_regularizer=L1L2(l1=0.0))(lstm_input)
lstm1 = LSTM(12,
return_sequences=True,
input_shape=(None, k),
stateful=False,
dropout=0.1,
recurrent_regularizer=L1L2(l1=0.0))(lstm)
lstm2 = Concatenate(axis=-1)(inputs[2] + [lstm1])
print(k)
# Dense layers
dns1 = Dense(128, activation='relu')(lstm2)
con3 = Dropout(0.1)(dns1)
dns2 = Dense(64, activation='relu')(con3)
dp1 = Dropout(0.2)(dns2)
dns3 = Dense(1, activation='sigmoid')(dp1)
# Model
model = Model(inputs[0] + [cont_input, lstm_input], dns3)
print(model.summary(90))
model.compile(loss='binary_crossentropy',
optimizer='adam',
sample_weight_mode='temporal',
metrics=['binary_crossentropy'])
return model
def _build_model(self, input_dim, output_dim):
model = Sequential()
model.add(
Dense(500,
activation='relu',
kernel_regularizer=L1L2(l1=0.0, l2=0.002),
input_dim=input_dim))
model.add(
Dense(output_dim,
activation='softmax',
kernel_regularizer=L1L2(l1=0.0, l2=0.002)))
model.compile(optimizer='sgd',
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def LR(inp_dim):
print("Model LR")
model = Sequential()
model.add(
Dense(1,
activation='sigmoid',
input_dim=inp_dim,
kernel_regularizer=L1L2(l1=0.0, l2=0.00000001),
bias_regularizer=L1L2(l2=0.00000001)))
model.compile(optimizer="adam",
loss='binary_crossentropy',
metrics=['accuracy'])
#print(model.summary())
return model
def model_discriminator():
nch = 256
h = 5
reg = lambda: L1L2(l1=1e-7, l2=1e-7)
c1 = Conv2D(int(nch / 4), (h, h), padding='same', kernel_regularizer=reg(),
input_shape=(32, 32, 3))
c2 = Conv2D(int(nch / 2), (h, h), padding='same', kernel_regularizer=reg())
c3 = Conv2D(nch, (h, h), padding='same', kernel_regularizer=reg())
c4 = Conv2D(1, (h, h), padding='same', kernel_regularizer=reg())
model = Sequential()
model.add(c1)
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(0.2))
model.add(c2)
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(0.2))
model.add(c3)
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(0.2))
model.add(c4)
model.add(AveragePooling2D(pool_size=(4, 4), padding='valid'))
model.add(Flatten())
model.add(Activation('sigmoid'))
return model
def __init__(self):
'''
self.data_dir="data/images_data/the-simpsons-characters-dataset/simpsons_dataset"
self.model_dir = "model/image_synthesis_GAN"
self.result_dir = "results/gan_images"
self.loss_fn = "binary_crossentropy"
self.generator_layers=[256,256,512]
self.discriminator_layers=[512,256]
self.latent_dim = 100
self.img_rows = 32
self.img_cols = 32
self.n_channels = 3
self.use_channel_in_img = True
self.learning_rate=0.01
self.batch_size=200
self.batch_num = 10
self.generate_images_flag = False
self.load_previous_model = False
self.use_cifar_flag=True
self.plot_rows=5
self.plot_cols=5
'''
self.read_configuration("config/image_synthesis_GAN.prop")
self.reg = lambda: L1L2(1e-5, 1e-5)
self.img_shape = (self.img_rows,self.img_cols,self.n_channels)
self.run_image_synthesis()
def setup(self, input_dim: tuple, **kwargs) -> None:
"""
Method used to setup the Keras machine learning model
:param input_dim: Dimensions of the input vectors
:param kwargs: Other potential arguments. Not applicable here.
:return: -
"""
# Defining layers
batchNormLayer = BatchNormalization(input_shape=input_dim)
denseLayer = Dense(2,
activation='softmax',
kernel_regularizer=L1L2(l1=.0, l2=.1))
# Adding layers
self.model.add(batchNormLayer)
self.model.add(denseLayer)
# Compiling model
self.model.compile(loss='categorical_crossentropy',
optimizer=SGD(lr=0.001),
metrics=['accuracy'])
self.built = True
def classify_participant_independent(csv, trainingid, validationid, testingid, normalizer_type, normalizer1, normalizer2, totalDays):
X_train = []
Y_train = []
X_val = []
Y_val = []
X_test = []
Y_test = []
for i in range(totalDays, len(csv)):
days = data_collector.collectDayData(csv, i, totalDays)
userid = days[0][1]
if data_collector.isSameUserAcross(days):
x = transform_to_train_vec.transform(
days,
False,
"LSTM",
totalDays,
normalizer_type,
normalizer1,
normalizer2
)
# put the x vector into the appropriate set (i.e. training, validation, testing)
if days[0][EMA_INDEX] != '':
X_train, Y_train, X_val, Y_val, X_test, Y_test = assign_data.independent_assign(
True,
days[0][1],
trainingid,
validationid,
testingid,
X_train,
Y_train,
X_val,
Y_val,
X_test,
Y_test,
x,
days[0][EMA_INDEX]
)
model = Sequential()
model.add(LSTM(64, return_sequences=True, input_shape=(totalDays, len(X_train[0][0]))))
model.add(LSTM(64, return_sequences=True, recurrent_dropout=0.2))
model.add(LSTM(64))
model.add(Dense(4, activation='softmax', kernel_regularizer=L1L2(l1=0.0, l2=0.0)))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['categorical_accuracy'])
if len(X_val) == 0:
model.fit(X_train, Y_train, epochs=5)
else:
model.fit(X_train, Y_train, epochs=5, validation_data=(X_val, Y_val))
y_pred = model.predict(X_test)
return prediction_utilities.convert_preds_into_ema(y_pred), Y_test
def build_mlp(input_layer, num_units=16, activation="selu", n_layers=1, p_dropout=0.0,
l2_weight=0.0, with_bn=False, with_bias=True, **kwargs):
last_layer = input_layer
for i in range(n_layers):
last_layer = Dense(num_units,
kernel_regularizer=L1L2(l2=l2_weight),
bias_regularizer=L1L2(l2=l2_weight),
use_bias=with_bias)(last_layer)
if with_bn:
last_layer = BatchNormalization(beta_regularizer=L1L2(l2=l2_weight),
gamma_regularizer=L1L2(l2=l2_weight))(last_layer)
last_layer = Activation(activation)(last_layer)
if p_dropout != 0.0:
last_layer = Dropout(p_dropout)(last_layer)
return last_layer
def get_model(self, batch_size):
dengue_input = Input(batch_shape=(batch_size, self.history_length, 1))
dropout_input = Dropout(self.dropout_input)(dengue_input)
emb_input = Input(batch_shape=(batch_size, self.history_length))
emb = Embedding(52, self.neurons_emb,
input_length=self.history_length)(emb_input)
dropout_emb = Dropout(self.dropout)(emb)
merged = keras.layers.concatenate([dropout_input, dropout_emb])
lstm = LSTM(self.neurons,
stateful=True,
kernel_regularizer=L1L2(l1=self.regularization_l1,
l2=self.regularization_l2))(merged)
dropout_lstm = Dropout(self.dropout)(lstm)
output_dense = Dense(1, activation='sigmoid',
use_bias=True)(dropout_lstm)
model_temp = Model(inputs=[dengue_input, emb_input],
outputs=output_dense)
op = None
if self.optimizer == OpUtil.RMSPROP.value:
op = keras.optimizers.RMSprop(lr=self.learning_rate,
rho=0.9,
epsilon=1e-6,
decay=0.0)
model_temp.compile(loss='mean_squared_error', optimizer=op)
return model_temp
def model_discriminator():
nch = 256
h = 5
reg = lambda: L1L2(l1=1e-7, l2=1e-7)
c1 = Conv2D(int(nch / 4), (h, h),
padding="same",
kernel_regularizer=reg(),
input_shape=dim_ordering_shape((3, 32, 32)))
c2 = Conv2D(int(nch / 2), (h, h), padding="same", kernel_regularizer=reg())
c3 = Conv2D(nch, (h, h), padding="same", kernel_regularizer=reg())
c4 = Conv2D(1, (h, h), padding="same", kernel_regularizer=reg())
def m(dropout):
model = Sequential()
model.add(c1)
model.add(SpatialDropout2D(dropout))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(0.2))
model.add(c2)
model.add(SpatialDropout2D(dropout))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(0.2))
model.add(c3)
model.add(SpatialDropout2D(dropout))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(0.2))
model.add(c4)
model.add(AveragePooling2D(pool_size=(4, 4), padding="valid"))
model.add(Flatten())
model.add(Activation("sigmoid"))
return model
return m
def model_generator():
model = Sequential()
nch = 256
reg = lambda: L1L2(l1=1e-7, l2=1e-7)
h = 5
model.add(Dense(nch * 4 * 4, input_dim=100, kernel_regularizer=reg()))
model.add(BatchNormalization())
model.add(Reshape(dim_ordering_shape((nch, 4, 4))))
model.add(
Conv2D(int(nch / 2), (h, h), padding="same", kernel_regularizer=reg()))
model.add(BatchNormalization())
model.add(LeakyReLU(0.2))
model.add(UpSampling2D(size=(2, 2)))
model.add(
Conv2D(int(nch / 2), (h, h), padding="same", kernel_regularizer=reg()))
model.add(BatchNormalization())
model.add(LeakyReLU(0.2))
model.add(UpSampling2D(size=(2, 2)))
model.add(
Conv2D(int(nch / 4), (h, h), padding="same", kernel_regularizer=reg()))
model.add(BatchNormalization())
model.add(LeakyReLU(0.2))
model.add(UpSampling2D(size=(2, 2)))
model.add(Conv2D(3, (h, h), padding="same", kernel_regularizer=reg()))
model.add(Activation("sigmoid"))
return model
def create_model(train_X, train_y, val_X, val_y):
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)))
# design network
model = Sequential()
model.add(
LSTM(units=128,
return_sequences=True,
input_shape=(train_X.shape[1], train_X.shape[2]),
bias_regularizer=L1L2(l1=0.1, l2=0.05)))
model.add(Dropout({{uniform(0, 1)}}))
model.add(LSTM(units=64))
model.add(Dropout({{uniform(0, 1)}}))
# model.add(Dense(16,init='uniform',activation='relu'))
model.add(Dense({{choice([64, 32, 16, 8])}}))
model.add(Dense(units=1))
model.compile(optimizer={{choice(['adam'])}}, loss=root_mean_squared_error)
# fit network
history = model.fit(train_X,
train_y,
epochs=500,
batch_size={{choice([25])}},
verbose=2,
shuffle=False,
validation_split=0.1)
#get the highest validation accuracy of the training epochs
validation_acc = np.amin(history.history['val_loss'])
print('Best validation loss of epoch:', validation_acc)
return {'loss': validation_acc, 'status': STATUS_OK, 'model': model}
def network_initializer(self, max_timesteps=None):
### build encoder
enc_input = Input(shape=(self.max_length, ),
dtype='int32',
name='input')
enc_embedding = Embedding(self.vocab_size,
self.embedding_size,
name='embedding')(enc_input)
# stacking encoding LSTMs
convos = []
for i, hs in enumerate(self.kernel_sizes):
if hs > 0:
convo_layer = Conv1D(filters=hs,
kernel_size=i + 1,
padding='same')(enc_embedding)
if self.L_pooling > 1:
convo_layer = MaxPooling1D(
pool_size=self.L_pooling)(convo_layer)
convo_layer = Flatten()(convo_layer)
convos.append(convo_layer)
enc_output = Concatenate()(convos) if len(convos) > 1 else convos[0]
enc_output = Lambda(lambda x: K.l2_normalize(x, axis=1),
name=BioactivityLSTM.ENCODING_NAME)(enc_output)
### output layer
out_layer = Dense(1, kernel_regularizer=L1L2(l2=self.l2))(enc_output)
self.model = Model(inputs=enc_input, outputs=out_layer)
self.model.compile(optimizer=self.optimizer, loss='mse')
def network_initializer(self, max_timesteps=None):
### build encoder
enc_input = Input(shape=(None, ), dtype='int32', name='input')
enc_embedding = Embedding(self.vocab_size,
self.embedding_size,
mask_zero=True,
name='embedding')(enc_input)
# stacking encoding LSTMs
hidden_states = []
enc_layer = enc_embedding
for i, layer_size in enumerate(self.layer_sizes):
return_sequences = (i != len(self.layer_sizes) - 1)
enc_layer, hidden_state, cell_state = LSTM(
layer_size,
return_sequences=return_sequences,
return_state=True,
name='lstm_%d' % (i + 1))(enc_layer)
hidden_states += [hidden_state, cell_state]
# concatenating LSTMs' states and normalizing their norms
enc_output = Concatenate()(hidden_states)
enc_output = Lambda(lambda x: K.l2_normalize(x, axis=1),
name=BioactivityLSTM.ENCODING_NAME)(enc_output)
### output layer
out_layer = Dense(1, kernel_regularizer=L1L2(l2=self.l2))(enc_output)
self.model = Model(inputs=enc_input, outputs=out_layer)
self.model.compile(optimizer=self.optimizer, loss='mse')
def dcgan_discriminator_max_pool(channel=1):
nch = 256
h = 5
reg = lambda: L1L2(l1=1e-7, l2=1e-7) #L1L2(l1=1e-7, l2=1e-7)
c1 = Convolution2D(int(nch / 4), (h, h),
padding='same',
kernel_regularizer=reg(),
input_shape=(64, 64, channel))
c2 = Convolution2D(int(nch / 2), (h, h),
padding='same',
kernel_regularizer=reg())
c3 = Convolution2D(nch, (h, h), padding='same', kernel_regularizer=reg())
c4 = Convolution2D(nch, (h, h), padding='same', kernel_regularizer=reg())
model = Sequential()
model.add(c1)
model.add(MaxPooling2D(pool_size=(2, 2), data_format='channels_last'))
model.add(LeakyReLU(0.2))
model.add(c2)
model.add(MaxPooling2D(pool_size=(2, 2), data_format='channels_last'))
model.add(LeakyReLU(0.2))
model.add(c3)
model.add(MaxPooling2D(pool_size=(4, 4), data_format='channels_last'))
model.add(LeakyReLU(0.2))
model.add(c4)
model.add(
MaxPooling2D(pool_size=(4, 4),
data_format='channels_last')) #, border_mode='valid')
model.add(LeakyReLU(0.2))
model.add(Flatten())
#model.add(Dense(1, activation='linear'))
model.add(Dense(1, activation='sigmoid'))
return model