def test_specify_initial_state_keras_tensor(self, layer_class):
input_size = 10
timesteps = 6
units = 2
num_samples = 32
num_states = 2 if layer_class is keras.layers.CuDNNLSTM else 1
inputs = keras.Input((timesteps, input_size))
initial_state = [keras.Input((units,)) for _ in range(num_states)]
layer = layer_class(units)
if len(initial_state) == 1:
output = layer(inputs, initial_state=initial_state[0])
else:
output = layer(inputs, initial_state=initial_state)
self.assertIn(initial_state[0], layer._inbound_nodes[0].input_tensors)
model = keras.models.Model([inputs] + initial_state, output)
model.compile(
loss='categorical_crossentropy',
optimizer=RMSprop(learning_rate=0.001),
run_eagerly=testing_utils.should_run_eagerly(),
experimental_run_tf_function=testing_utils.should_run_tf_function())
inputs = np.random.random((num_samples, timesteps, input_size))
initial_state = [
np.random.random((num_samples, units)) for _ in range(num_states)
]
targets = np.random.random((num_samples, units))
model.fit([inputs] + initial_state, targets)
def test_weightnorm_dense_train(self):
model = keras.models.Sequential()
model.add(
wrappers.WeightNormalization(keras.layers.Dense(2),
input_shape=(3, 4)))
model.compile(optimizer=RMSprop(learning_rate=0.001), loss='mse')
model.fit(np.random.random((10, 3, 4)),
np.random.random((10, 3, 2)),
epochs=3,
batch_size=10)
self.assertTrue(hasattr(model.layers[0].layer, 'g'))
def main():
# Configure IPUs.
cfg = ipu.utils.create_ipu_config()
cfg = ipu.utils.auto_select_ipus(cfg, 1)
ipu.utils.configure_ipu_system(cfg)
# Set up IPU strategy.
strategy = ipu.ipu_strategy.IPUStrategy()
with strategy.scope():
model = get_model()
model.compile(loss='sparse_categorical_crossentropy', optimizer=RMSprop(learning_rate=0.01))
model.fit(get_dataset(), steps_per_epoch=100, epochs=10)
def get_hyperparam_model():
# Fourth model with hyper parameter tuning
model_4 = Sequential()
model_4.add(
Conv2D(filters=32,
kernel_size=(5, 5),
strides=1,
padding='Same',
activation='relu',
input_shape=(28, 28, 1)))
model_4.add(BatchNormalization())
model_4.add(
Conv2D(filters=32,
kernel_size=(5, 5),
strides=1,
padding='Same',
activation='relu'))
model_4.add(BatchNormalization())
model_4.add(Dropout(0.4))
model_4.add(
Conv2D(filters=64,
kernel_size=(3, 3),
strides=2,
padding='Same',
activation='relu'))
model_4.add(BatchNormalization())
model_4.add(
Conv2D(filters=64,
kernel_size=(3, 3),
strides=2,
padding='Same',
activation='relu'))
model_4.add(BatchNormalization())
model_4.add(Dropout(0.4))
model_4.add(Flatten())
model_4.add(Dense(256, activation="relu"))
model_4.add(Dropout(0.4))
model_4.add(Dense(10, activation="softmax"))
# Define the optimizer and compile the model
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
model_4.compile(optimizer=optimizer,
loss="categorical_crossentropy",
metrics=["accuracy"])
return model_4
def test_weightnorm_conv2d(self):
model = keras.models.Sequential()
model.add(
wrappers.WeightNormalization(keras.layers.Conv2D(5, (2, 2),
padding='same'),
input_shape=(4, 4, 3)))
model.add(keras.layers.Activation('relu'))
model.compile(optimizer=RMSprop(learning_rate=0.001), loss='mse')
model.fit(np.random.random((2, 4, 4, 3)),
np.random.random((2, 4, 4, 5)),
epochs=3,
batch_size=10)
self.assertTrue(hasattr(model.layers[0].layer, 'g'))
def net_work_without_embedding():
model = models.Sequential([
layers.Flatten(input_shape=(25, )),
layers.Dense(64, activation='relu'),
layers.Dropout(0.3),
layers.Dense(128, activation='relu'),
layers.Dense(64, activation='relu'),
layers.Dropout(0.3),
layers.Dense(32, activation='relu'),
layers.Dense(1, activation='sigmoid')
])
model.compile(optimizer=RMSprop(lr=1e-3),
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def test_time_major_input(self, layer_class):
input_size = 10
timesteps = 6
units = 2
num_samples = 32
model = keras.models.Sequential()
model.add(
keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2])))
layer = layer_class(units, time_major=True, return_sequences=True)
model.add(layer)
model.add(
keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2])))
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(learning_rate=0.001))
model.fit(np.ones((num_samples, timesteps, input_size)),
np.ones((num_samples, timesteps, units)))
out = model.predict(np.ones((num_samples, timesteps, input_size)))
self.assertEqual(out.shape, (num_samples, timesteps, units))
def train_lstm_model(params: Dict,
full_text: str,
model_path=None,
weights_path=None):
"""
Train function, builds the model, with metrics and checkpoints, import model
config and train with number of epochs and parameters provided in config
Args:
params: json_config for the model
full_text: full text of data to process
model_path: optional path to previously saved model
weights_path: optional path to previously saved weights
Returns:
model history
"""
# load char2int encoder
char2int_encoder = load_json_file(params['char2int_encoder_path'])
# Load model from previous training session
if model_path and weights_path:
model = load_model_from_json_and_weights(model_path, weights_path)
# Create new model if no previous one
else:
lstm_model = LSTMModel(sequence_length=params['sequence_length'],
step=params['step'],
lstm_units=params['lstm_units'],
char_encoder=char2int_encoder)
model = lstm_model.build_model()
# Set optimizer
optimizer = RMSprop()
# Metrics
precision = Precision()
recall = Recall()
categorical_accuracy = CategoricalAccuracy()
metrics = [precision, recall, categorical_accuracy]
model.compile(optimizer=optimizer,
loss=params['loss'],
metrics=metrics,
run_eagerly=False)
# Define callbacks
if weights_path:
last_epoch = max([
int(
re.search(r"weights\.0?(?P<epoch>\d\d?)-",
filename).group("epoch"))
for filename in os.listdir(params['model_path'])
if filename.endswith("hdf5")
])
file_path = params["model_path"] + '/weights.' + str(
last_epoch) + '-{epoch:02d}-{val_loss:.2f}.hdf5'
else:
file_path = params[
"model_path"] + '/weights.{epoch:02d}-{val_loss:.2f}.hdf5'
checkpoint = ModelCheckpoint(monitor='val_loss',
filepath=file_path,
verbose=1,
save_freq='epoch')
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=1,
verbose=1,
mode='auto',
epsilon=0.0001,
cooldown=0,
min_lr=0)
callbacks_fit = [checkpoint, reduce_lr]
# Save model json
if not model_path:
with open(params["model_path"] + '/model_result.json',
'w') as json_file:
json_file.write(model.to_json())
x, y = extract_data_with_labels(full_text, params, char2int_encoder)
# Fit model
logging.info('Start training')
history = model.fit(x,
y,
batch_size=params['batch_size'],
epochs=params['epochs'],
verbose=1,
callbacks=callbacks_fit,
validation_split=0.2)
# Print results
history = history.history
logging.info(history)
model.save_weights(params["model_path"] + '/model_result_weights.h5')
return history['val_categorical_accuracy'], history['val_loss']
model.add(Dense(128, activation='selu'))
model.add(Dropout(0.1))
model.add(Dense(64, activation='selu'))
model.add(Dropout(0.1))
model.add(Dense(32, activation='selu'))
model.add(Dropout(0.1))
model.add(Dense(16, activation='selu'))
model.add(Dropout(0.1))
model.add(Dense(2, activation='selu'))
model.add(Dropout(0.1))
model.add(Dense(1, activation='sigmoid'))
# compile the keras model
# model.compile(loss='mean_absolute_error', optimizer='adam', metrics=['accuracy'])
# model.compile(loss='categorical_crossentropy', optimizer=RMSprop(), metrics=['accuracy'])
model.compile(optimizer=RMSprop(),
loss='binary_crossentropy',
metrics=['BinaryAccuracy'])
# fit the keras model on the dataset
# model.fit(X_train, y_train, epochs=500, batch_size=128, validation_data=(X_test, y_test))
model.fit(X_train,
y_train,
epochs=1500,
batch_size=50,
validation_data=(X_test, y_test),
callbacks=[EarlyStopping(monitor='binary_accuracy', patience=10)])
# evaluate the keras model
oldModel = keras.models.load_model('model/spreadpredictionmodel')
_, oldAccuracy = oldModel.evaluate(X_val, y_val)
