def reuse_conv_base():
"""
This function will reuse the bottom of the VGG16 network to extract the features of the images and then use
a new classifier to perform the classification. The issue with this approach is that the classifier overfits
almost immediately. The chapter in the book goes into detail into how this can be avoided by incorporating
the base into the model and training end - to - end with augmentation, however this is too expensive to run
on CPU and is essentially a mix of train_model_with_augmentation() and reuse_conv_base()
:return: None
"""
# Extract the convolutional base of VGG16 and print the summary
conv_base = VGG16(weights='imagenet',
include_top=False,
input_shape=(150, 150, 3))
conv_base.summary()
# Prepare the image generator
datagen = ImageDataGenerator(rescale=1. / 255)
batch_size = 20
def extract_features(directory, sample_count):
"""
Simple function to pass the inputs from the dataset through the convolutional base to extract their
features
:param directory: Directory to the dataset of interest
:param sample_count: Number of samples to preprocess
:return: The processed features and their labels
"""
features = np.zeros(shape=(sample_count, 4, 4, 512))
labels = np.zeros(shape=(sample_count))
generator = datagen.flow_from_directory(directory,
target_size=(150, 150),
batch_size=batch_size,
class_mode='binary')
i = 0
for inputs_batch, labels_batch in generator:
features_batch = conv_base.predict(inputs_batch)
features[i * batch_size:(i + 1) *
batch_size] = features_batch
labels[i * batch_size:(i + 1) * batch_size] = labels_batch
i += 1
if i * batch_size >= sample_count:
break
return features, labels
# Pre-process data for training, validation and testing
train_features, train_labels = extract_features(train_dir, 2000)
validation_features, validation_labels = extract_features(
validation_dir, 1000)
test_features, test_labels = extract_features(test_dir, 1000)
train_features = np.reshape(train_features, (2000, 4 * 4 * 512))
validation_features = np.reshape(validation_features,
(1000, 4 * 4 * 512))
test_features = np.reshape(test_features, (1000, 4 * 4 * 512))
# Layout the classifier (very simple) and train. Record the training and validation performance
model = models.Sequential()
model.add(
layers.Dense(256, activation='relu', input_dim=4 * 4 * 512))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(optimizer=optimizers.RMSprop(lr=2e-5),
loss='binary_crossentropy',
metrics=['acc'])
history = model.fit(train_features,
train_labels,
epochs=30,
batch_size=20,
validation_data=(validation_features,
validation_labels))
# Save the model and plot the performance
model.save(
filepath=
'C:\\Datasets\\dogs-vs-cats\\cats_and_dogs_small\\cats_and_dogs_small_ReuseBottom.h5'
)
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(acc) + 1)
plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and Validation Accuracy vs. Epoch #')
plt.legend()
plt.figure()
plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and Validation Loss vs. Epoch #')
plt.legend()
plt.show()
import tensorflow.keras.models as models
from tensorflow.keras.layers import Dense, Dropout
from tool.preparation_functions import convert_dic_to_vector
from tool.config import max_letters, language_tags
from unidecode import unidecode
import numpy as np
network = models.Sequential()
network.add(Dense(200, input_dim=26*max_letters, activation='sigmoid'))
network.add(Dense(150, activation='sigmoid'))
network.add(Dense(100, activation='sigmoid'))
network.add(Dropout(0.15))
network.add(Dense(75, activation='sigmoid'))
network.add(Dense(len(language_tags), activation='softmax'))
network.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
network.load_weights('tool/weights.h5')
# while True:
# dic = []
# valid = False
# while not valid:
# print('Let our predictor decide if the word is in Polish or English')
# word = input('Enter word to predict (maximum 10 characters, do not use special characters):\n')
# if len(word) <= max_letters:
# word = word.lower()
# valid = True
# else:
# print('Word must be less than ' + str(max_letters + 1) + ' letters long')
#
def create_cnn_model(nr_outputs,
input_shape,
optimization_method="adam",
model_name="inc_nr_filters"):
"""
Here we create the desired CNN model using the Keras API and return it
"""
K.clear_session()
model = models.Sequential()
if model_name == "inc_nr_filters":
#1+2
model.add(
layers.Conv2D(32, (3, 3),
activation='relu',
input_shape=input_shape))
model.add(layers.MaxPooling2D((2, 2)))
#3+4
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
#5+6
model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
#7+8
model.add(layers.Conv2D(256, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
#9+10
model.add(layers.Conv2D(512, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
#11+12
model.add(layers.Conv2D(1024, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
elif model_name == "same_nr_filters":
#1+2
model.add(
layers.Conv2D(256, (3, 3),
activation='relu',
input_shape=input_shape))
model.add(layers.MaxPooling2D((2, 2)))
#3+4
model.add(layers.Conv2D(256, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
#5+6
model.add(layers.Conv2D(256, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
#7+8
model.add(layers.Conv2D(256, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
#9+10
model.add(layers.Conv2D(256, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
#11+12
model.add(layers.Conv2D(256, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
# add MLP
model.add(layers.Flatten())
model.add(layers.Dense(2048, activation='relu', name="fc1"))
model.add(layers.Dense(nr_outputs, activation='softmax', name="output"))
# for all models: use the same optimizer, loss and learn rate:
model.compile(optimizer=optimization_method,
loss='categorical_crossentropy')
# return the model just built
return model
def convTower(self,
convolutions,
conv_act=None,
out_no_act=True,
conv_bias=True,
input_type="one_hot",
regularizer=None,
embedding_dim=None,
regul_dense_only=True):
"""
Make sequential DNN building block of stack
of convolutional layers
Parameters:
- convolutions -- specification of convolutional layers
as list of tuples:
either (kernel width, n filters, stride)
or (kernel width, n filters) if stride = 1
- conv_act -- activation of convolutional layers
- out_no_act -- flag of no activation on last convolutional layer
- conv_bias -- use bias for convolutional layers
- input_type -- type of input layer:
- one_hot - one hot coding of input samples
- categorical - categorical coding
decoded by preprocessing layer
- trainable - categorical coding processed
by trainable embedding layer
- regularizer -- regularizer applied to weights of dense layer
and to convolutionand embedding layers if
parameters regul_dense_only = False
- embedding_dim -- size of embedding vectors
It effects trainable embedding only,
i.e. input_type == "trainable"
If it is None self.n_tokens is used.
If it is None self.n_tokens is used
- regul_dense_only -- flag to regulirize only dense layers
Returns:
- dnn -- constructed DNN building block
"""
_conv_regul = None if regul_dense_only else regularizer
_embed_regul = None if regul_dense_only else regularizer
_dnn = models.Sequential()
if input_type == "one_hot":
_dnn.add(layers.Input(shape=(
None,
self.n_tokens,
)))
elif input_type == "categorical" or input_type == "trainable":
_trainable = (input_type == "trainable")
_dnn.add(layers.Input(shape=(None, )))
self.addEmbedding(_dnn,
trainable=_trainable,
width=embedding_dim,
regularizer=_embed_regul)
else:
sys.exit(
f"Wrong type {input_type} of input_type parameter of sequentinal convolution block"
)
self.addConv1(_dnn,
convolutions,
activation=conv_act,
bias=conv_bias,
out_no_act=out_no_act,
regularizer=_conv_regul)
return _dnn
def __init__(self, numRepsPerTest, cnnNumLayers, cnnAct, cnnChan1,
cnnChan2, cnnChan3, cnnDrop, cnnBatch, cnnReg, denNumLayers,
denNumChannels, denAct):
self.numRepsPerTest = numRepsPerTest
self.convNumLayers = cnnNumLayers
self.convActFunc = cnnAct
self.convChan1 = cnnChan1
self.convChan2 = cnnChan2
self.convChan3 = cnnChan3
self.convDrop1 = cnnDrop
self.convDrop2 = cnnDrop
self.convDrop3 = cnnDrop
self.convBatchNorm = cnnBatch
self.convReg1 = None
self.convReg2 = None
self.denseNumLayers = denNumLayers
self.denseNumChannels = denNumChannels
self.denseActFunc = denAct
self.returnData = []
self.bestAccuracy = []
self.bestEpoch = []
self.lossAtEpoch = []
self.hst = None
if cnnReg == True:
self.convReg1 = regularizer
self.convReg2 = regularizer
if self.convNumLayers > 0:
self.returnData.append("Channels: " + str(self.convChan1) +
" Dropout: " + str(self.convDrop1) +
" BatchNorm: " + str(self.convBatchNorm) +
" Reg: " + str(self.convReg1))
if self.convNumLayers > 1:
self.returnData.append("Channels: " + str(self.convChan2) +
" Dropout: " + str(self.convDrop2) +
" BatchNorm: " + str(self.convBatchNorm) +
" Reg: " + str(self.convReg2))
if self.convNumLayers > 2:
self.returnData.append("Channels: " + str(self.convChan3) +
" Dropout: " + str(self.convDrop3) +
" BatchNorm: " + str(self.convBatchNorm))
#Create Neural Network Model
#testRepetitions = [models.Sequential()] * self.numRepsPerTest
self.nn = [None] * self.numRepsPerTest
for x in range(self.numRepsPerTest):
#Create Neural Network Model
self.nn[x] = models.Sequential()
#Build CNN Layers
self.nn[x].add(
layers.Conv2D(self.convChan1, (3, 3),
activation=self.convActFunc,
kernel_regularizer=self.convReg1,
bias_regularizer=self.convReg1,
input_shape=(28, 28, 1)))
if self.convBatchNorm == True: self.nn[x].add(BatchNormalization())
self.nn[x].add(layers.MaxPooling2D((2, 2)))
self.nn[x].add(layers.Dropout(self.convDrop1))
if self.convNumLayers > 1:
self.nn[x].add(
layers.Conv2D(self.convChan2, (3, 3),
activation=self.convActFunc,
kernel_regularizer=self.convReg2,
bias_regularizer=self.convReg2))
if self.convBatchNorm == True:
self.nn[x].add(BatchNormalization())
self.nn[x].add(layers.MaxPooling2D((2, 2)))
self.nn[x].add(layers.Dropout(self.convDrop2))
if self.convNumLayers > 2:
self.nn[x].add(
layers.Conv2D(self.convChan3, (3, 3),
activation=self.convActFunc))
if self.convBatchNorm == True:
self.nn[x].add(BatchNormalization())
self.nn[x].add(layers.Dropout(self.convDrop3))
self.nn[x].add(layers.Flatten())
#Build Dense Layers
for i in range(self.denseNumLayers):
self.nn[x].add(
layers.Dense(self.denseNumChannels,
activation=self.denseActFunc))
#Build Final Dense Softmax Layer
self.nn[x].add(layers.Dense(10, activation='softmax'))
training_outputs = np.array([])
# -------------------------------preprocessing-------------------------------
max_acceleration = 100
max_velocity = 1000
max_height = 100000
max_fuel = 10
max_time = 1000
# -------------------------------model architecture-------------------------------
# try making the shape (None, 6) so I can put the entire rocket
# state/output history as 1 data point instead of putting many
model = models.Sequential([
layers.Flatten(input_shape=(6,)),
layers.Dense(20, activation='relu'),
layers.Dense(1, activation='sigmoid')
# softmax is for classification of many different things
# (like classifying between 10 different animals)
# sigmoid compresses the value between 0 and 1 but not for classification
])
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
# -------------------------------checkpoints-------------------------------
checkpoint_dir = '../../rocket_v1_checkpoints'
checkpoint_prefix = os.path.join(checkpoint_dir, 'ckpt_{epoch}')
checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_prefix,
save_weights_only=True
)
args.data_dir + "/train/", # this is the target directory
target_size=(img_width, img_height),
batch_size=args.batch_size,
class_mode='binary')
valid_datagen = kpi.ImageDataGenerator(rescale=1. / 255)
validation_generator = valid_datagen.flow_from_directory(
args.data_dir + "/validation/",
target_size=(img_width, img_height),
batch_size=args.batch_size,
class_mode='binary')
## Definition du modEle
model_conv = km.Sequential()
model_conv.add(
kl.Conv2D(32, (3, 3),
input_shape=(img_width, img_height, 3),
data_format="channels_last"))
model_conv.add(kl.Activation('relu'))
model_conv.add(kl.MaxPooling2D(pool_size=(2, 2)))
model_conv.add(kl.Conv2D(32, (3, 3)))
model_conv.add(kl.Activation('relu'))
model_conv.add(kl.MaxPooling2D(pool_size=(2, 2)))
model_conv.add(kl.Conv2D(64, (3, 3)))
model_conv.add(kl.Activation('relu'))
model_conv.add(kl.MaxPooling2D(pool_size=(2, 2)))
def train_pneumonia_binary_classification_cnn(train_data,
test_data,
path,
overwrite=False,
epochs=20,
val_data=None,
data_augmentation=False,
img_size=64):
(test_images, test_labels) = test_data
if val_data is None:
(x, y) = train_data
train_images, val_images, train_labels, val_labels = train_test_split(
x, y, test_size=0.167, train_size=0.833)
else:
(train_images, train_labels) = train_data
(val_images, val_labels) = val_data
# With data augmentation to prevent overfitting and handling the imbalance in dataset
if data_augmentation:
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=
False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=
30, # randomly rotate images in the range (degrees, 0 to 180)
zoom_range=0.2, # Randomly zoom image
width_shift_range=
0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=
0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=False) # randomly flip images
datagen.fit(train_images)
# Let's start building a model
if not os.path.exists(path) or overwrite:
if os.path.exists(path):
shutil.rmtree(path)
print(
"TRAIN ANYWAY option enabled, create and train a new one ...")
else:
print("Model not found, create and train a new one ...")
model = models.Sequential()
model.add(
layers.Conv2D(32, (3, 3),
strides=1,
padding='same',
activation='relu',
input_shape=(img_size, img_size, 1)))
model.add(layers.BatchNormalization())
model.add(layers.MaxPool2D((2, 2), strides=2, padding='same'))
model.add(
layers.Conv2D(64, (3, 3),
strides=1,
padding='same',
activation='relu'))
model.add(layers.Dropout(0.1))
model.add(layers.BatchNormalization())
model.add(layers.MaxPool2D((2, 2), strides=2, padding='same'))
model.add(
layers.Conv2D(64, (3, 3),
strides=1,
padding='same',
activation='relu'))
model.add(layers.BatchNormalization())
model.add(layers.MaxPool2D((2, 2), strides=2, padding='same'))
model.add(
layers.Conv2D(128, (3, 3),
strides=1,
padding='same',
activation='relu'))
model.add(layers.Dropout(0.2))
model.add(layers.BatchNormalization())
model.add(layers.MaxPool2D((2, 2), strides=2, padding='same'))
model.add(
layers.Conv2D(256, (3, 3),
strides=1,
padding='same',
activation='relu'))
model.add(layers.Dropout(0.2))
model.add(layers.BatchNormalization())
model.add(layers.MaxPool2D((2, 2), strides=2, padding='same'))
model.add(layers.Flatten())
model.add(layers.Dense(units=128, activation='relu'))
model.add(layers.Dense(units=1, activation='sigmoid'))
model.compile(optimizer="rmsprop",
loss='binary_crossentropy',
metrics=['accuracy'])
print(model.summary())
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy',
patience=2,
verbose=1,
factor=0.3,
min_lr=0.000001)
training_history = model.fit(datagen.flow(train_images,
train_labels,
batch_size=32),
epochs=epochs,
validation_data=datagen.flow(
val_images, val_labels),
callbacks=[learning_rate_reduction])
test_loss, test_accuracy = model.evaluate(test_images,
test_labels,
verbose=2)
print("Final Accuracy achieved is: ", test_accuracy)
model.save(path)
print("Model has been saved")
plt.plot(training_history.history['accuracy'], label="Accuracy")
plt.plot(training_history.history['val_accuracy'],
label='val_accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.ylim([0.7, 1])
plt.legend(loc='lower right')
#plt.show()
else:
print("Model found, there is no need to re-train the model ...")
(images_train, labels_train), (images_test,
labels_test) = datasets.mnist.load_data()
print('Data processing complete.')
'''
Creating the model is very easy in tensorflow.
You can add as many layers as you wish provided the input shape and the
number of output units remains the same for a specific problem.
'''
model = models.Sequential([
layers.Flatten(
input_shape=(28, 28)
), #The flatten layer to flatten the 2D(28x28 px) image into a 1D(28*28=784) array.
layers.Dense(
128, activation='elu'
), #Hidden layer 1 with elu activation for non linearity.(Dense = fully connected layer)
layers.Dense(128, activation='elu'
), #Hidden layer 2 with elu activation for non linearity.
layers.Dense(32, activation='elu'
), #Hidden layer 3 with elu activation for non linearity.
layers.Dense(10,
activation='softmax') #Output layer with softmax activation.
])
#Mention the loss function and the optimizer you want to use before fitting the model.
#You can use many other loss functions and optimizer Visit tensorflow.org docs for more information.
model.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
'''
Now the model is ready for training.
def train_cifar_9_layer_cnn(
train_data,
test_data,
path,
overwrite=False,
use_relu=False,
optimizer_config=tf.keras.optimizers.Adam(learning_rate=0.001),
epochs=20):
(train_images, train_labels) = train_data
(test_images, test_labels) = test_data
# Let's start building a model
if not os.path.exists(path) or overwrite:
if os.path.exists(path):
shutil.rmtree(path)
print(
"TRAIN ANYWAY option enabled, create and train a new one ...")
else:
print(path, " - model not found, create and train a new one ...")
model = models.Sequential()
# In the first layer, please provide the input shape (32,32,3)
model.add(
layers.Conv2D(32, (3, 3),
activation='relu',
input_shape=(32, 32,
3))) # Result will be 32 30*30 outputs
model.add(layers.MaxPooling2D(
(2, 2))) # Result will be 32 15*15 outputs
model.add(layers.Conv2D(
64, (3, 3), activation='relu')) # Result will be 64 13*13 outputs
model.add(layers.MaxPooling2D((2, 2))) # Result will be 64 6*6 outputs
model.add(layers.Conv2D(
64, (3, 3), activation='sigmoid')) # Result will be 64 4*4 outputs
model.add(layers.Flatten()
) # Result will be a vector with length 4*4*64 = 1024
if use_relu:
model.add(layers.Dense(
128, activation='relu')) # Result will 64 outputs
model.add(layers.Dense(
64, activation='relu')) # Result will 64 outputs
else:
model.add(layers.Dense(
128, activation='sigmoid')) # Result will 64 outputs
model.add(layers.Dense(
64, activation='sigmoid')) # Result will 64 outputs
model.add(layers.Dense(
10, activation='softmax')) # Result will be 10 outputs
print(model.summary())
model.compile(optimizer=optimizer_config,
loss=tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True),
metrics=['accuracy'])
training_history = model.fit(train_images,
train_labels,
epochs=epochs,
validation_data=(test_images,
test_labels))
test_loss, test_accuracy = model.evaluate(test_images,
test_labels,
verbose=2)
print("Final Accuracy achieved is: ", test_accuracy)
model.save(path)
print("Model has been saved")
plt.plot(training_history.history['accuracy'], label="Accuracy")
plt.plot(training_history.history['val_accuracy'],
label='val_accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.ylim([0.4, 1])
plt.legend(loc='lower right')
#plt.show()
else:
print("Model found, there is no need to re-train the model ...")
def train_mnist_5_layer_mlp(
train_data,
test_data,
path,
overwrite=False,
use_relu=True,
optimizer_config=tf.keras.optimizers.Adam(learning_rate=0.001),
epochs=20):
if use_relu:
print(" >> ACTIVATION BY ReLU ...")
else:
print(" >> ACTIVATION BY Sigmoid ...")
(x, y) = train_data
(test_images, test_labels) = test_data
train_images, val_images, train_labels, val_labels = train_test_split(
x, y, test_size=0.167, train_size=0.833)
# Let's start building a model
if not os.path.exists(path) or overwrite:
if os.path.exists(path):
shutil.rmtree(path)
print(
"TRAIN ANYWAY option enabled, create and train a new one ...")
else:
print("Model not found, create and train a new one ...")
model = models.Sequential()
model.add(layers.Flatten(input_shape=(28, 28, 1)))
if not use_relu:
model.add(layers.Dense(128, activation='sigmoid'))
model.add(layers.Dense(128, activation='sigmoid'))
model.add(layers.Dense(64, activation='sigmoid'))
else:
model.add(layers.Dense(128, activation='relu'))
model.add(layers.Dense(128, activation='relu'))
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(10, activation='softmax'))
print(model.summary())
model.compile(optimizer=optimizer_config,
loss=tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True),
metrics=['accuracy'])
training_history = model.fit(train_images,
train_labels,
epochs=epochs,
validation_data=(val_images, val_labels))
test_loss, test_accuracy = model.evaluate(test_images,
test_labels,
verbose=2)
print("Final Accuracy achieved is: ", test_accuracy)
model.save(path)
print("Model has been saved")
plt.plot(training_history.history['accuracy'], label="Accuracy")
plt.plot(training_history.history['val_accuracy'],
label='val_accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.ylim([0.8, 1])
plt.legend(loc='lower right')
#plt.show()
else:
print("Model found, there is no need to re-train the model ...")
def train_creditcard_3_layer_mlp(
train_data,
test_data,
path,
overwrite=False,
optimizer_config=tf.keras.optimizers.Adam(learning_rate=0.001),
epochs=100,
initial_bias=[-6.35935934]):
BATCH_SIZE = 2048
METRICS = [
tf.keras.metrics.TruePositives(name='tp'),
tf.keras.metrics.FalsePositives(name='fp'),
tf.keras.metrics.TrueNegatives(name='tn'),
tf.keras.metrics.FalseNegatives(name='fn'),
tf.keras.metrics.BinaryAccuracy(name='accuracy'),
tf.keras.metrics.Precision(name='precision'),
tf.keras.metrics.Recall(name='recall'),
tf.keras.metrics.AUC(name='auc'),
]
(x, y) = train_data
(test_features, test_labels) = test_data
train_features, val_features, train_labels, val_labels = train_test_split(
x, y, test_size=0.2, train_size=0.8)
# Let's start building a model
if not os.path.exists(path) or overwrite:
if os.path.exists(path):
shutil.rmtree(path)
print(
"TRAIN ANYWAY option enabled, create and train a new one ...")
else:
print("Model not found, create and train a new one ...")
early_stopping = tf.keras.callbacks.EarlyStopping(
monitor='val_auc',
verbose=1,
patience=10,
mode='max',
restore_best_weights=True)
model = models.Sequential()
model.add(
layers.Dense(64,
activation='relu',
input_shape=(train_features.shape[-1], )))
#model.add(layers.Dropout(0.5))
model.add(layers.Dense(64, activation='relu'))
#model.add(layers.Dropout(0.5))
model.add(
layers.Dense(
1,
activation='sigmoid',
bias_initializer=tf.keras.initializers.Constant(initial_bias)))
print(model.summary())
model.compile(optimizer=optimizer_config,
loss=tf.keras.losses.BinaryCrossentropy(),
metrics=METRICS)
training_history = model.fit(train_features,
train_labels,
batch_size=BATCH_SIZE,
epochs=epochs,
callbacks=[early_stopping],
validation_data=(val_features,
val_labels))
baseline_results = model.evaluate(test_features,
test_labels,
batch_size=BATCH_SIZE,
verbose=0)
test_loss, test_accuracy = baseline_results[0], baseline_results[-4]
test_predictions_baseline = model.predict(test_features,
batch_size=BATCH_SIZE)
cm = confusion_matrix(test_labels, test_predictions_baseline > 0.5)
plt.figure(figsize=(5, 5))
sns.heatmap(cm, annot=True, fmt="d")
plt.title('Confusion matrix @{:.2f}'.format(0.5))
plt.ylabel('Actual label')
plt.xlabel('Predicted label')
print("Final Accuracy achieved is: ", test_accuracy, "with Loss",
test_loss)
model.save(path)
print("Model has been saved")
#plt.show()
else:
print("Model found, there is no need to re-train the model ...")
for spectrogram, _ in spectrogram_ds.take(1):
input_shape = spectrogram.shape
print('Input shape:', input_shape)
num_labels = len(commands)
norm_layer = preprocessing.Normalization()
norm_layer.adapt(spectrogram_ds.map(lambda x, _: x))
model = models.Sequential([
layers.Input(shape=input_shape),
preprocessing.Resizing(32, 32),
norm_layer,
layers.Conv2D(32, 3, activation='relu'),
layers.Conv2D(64, 3, activation='relu'),
layers.MaxPooling2D(),
layers.Dropout(0.25),
layers.Flatten(),
layers.Dense(128, activation='relu'),
layers.Dropout(0.5),
layers.Dense(num_labels),
])
model.summary()
model.compile(
optimizer=tf.keras.optimizers.Adam(),
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'],
)
x_valid, x_train = x_test, x_train_all #验证集10000个
y_valid, y_train = y_test, y_train_all
#打印一张照片
# def show_single_image(img_arr):
# plt.imshow(img_arr,cmap='binary')
# plt.show()
# show_single_image(x_train[2])
# 将模型的各层堆叠起来，以搭建 tf.keras.Sequential 模型。为训练选择优化器和损失函数：
model = models.Sequential([
layers.Flatten(input_shape=(28, 28, 1)),
layers.Dense(128, activation='relu'),
layers.Dense(128, activation='relu'),
layers.Dense(128, activation='relu'),
layers.Dropout(0.5),
layers.Dense(10, activation='softmax')
])
# 编译模型
model.compile(optimizer=optimizers.Adam(lr=1e-4),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
# model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
# 打印网络参数量
model.summary()
#x_train = tf.expand_dims(x_train,axis=-1)
#print(x_train.shape)
#x_train_1 = tf.reshape(x_train,[-1,28,28,1])
#x_test_1 = tf.reshape(x_test,[-1,28,28,1])
x_train = x_train.reshape(-1, 28, 28, 1)
def main():
### Parameters
tgt_dates = (datetime(1979, 1, 1), datetime(2020, 12, 31))
tgt_dates_str = [
'{}{:02d}'.format(tgt.year, tgt.month) for tgt in tgt_dates
]
nyr, mon_per_yr = tgt_dates[1].year - tgt_dates[0].year + 1, 12
nmons = nyr * mon_per_yr
indir = './Data/'
### Read OLR
infn_olr = indir + 'olr-monthly_v02r07_{}_{}.nc'.format(*tgt_dates_str)
vars = ['time', 'lat', 'lon', 'olr']
olr_data = fns.read_nc_data(infn_olr, vars)
## Check temporal dimension
t0 = olr_data['time'][0]
if tgt_dates[0].year != t0.year or tgt_dates[
0].month != t0.month or nmons != olr_data['time'].shape[0]:
print('Temporal dimension is inconsistent')
sys.exit()
else:
print('OLR data is read')
## OLR data is degraded for box mean, from 15S to 15N, every 30deg longitude
lon0, dlon, nlon = olr_data['lon'][
0], olr_data['lon'][1] - olr_data['lon'][0], olr_data['lon'].shape[0]
lat0, dlat, nlat = olr_data['lat'][
0], olr_data['lat'][1] - olr_data['lat'][0], olr_data['lat'].shape[0]
latidx = [fns.lat_deg2y(lat, lat0, dlat) for lat in [-15, 15]]
nlat2 = latidx[1] - latidx[0]
nlon_30deg = int(np.rint(30 / (dlon)))
olr = olr_data['olr'][:, latidx[0]:latidx[1], :]
olr = olr.reshape([nmons, nlat2, nlon // nlon_30deg, nlon_30deg
]).mean(axis=(1, 3)).reshape([nyr, mon_per_yr, -1])
print('After box-averaging, ', olr.shape)
### Read Nino3.4 values
infn_nn34 = indir + 'nino3.4.txt'
nn34 = fns.read_nn34_text(infn_nn34, tgt_dates)
if nn34.shape != (nyr, mon_per_yr):
print('Temporal dimension is inconsistent')
sys.exit()
else:
print('Nino3.4 data is read', nn34.shape)
### Remove seanal cycle
olr_mm = olr.mean(axis=0)
olr = olr - olr_mm[None, :]
olr = olr.reshape([-1, olr.shape[-1]])
nn34_mm = nn34.mean(axis=0)
nn34 = nn34 - nn34_mm[None, :]
nn34 = nn34.reshape(-1)
print('Seasonal cycle is removed')
#from scipy import stats
#print(stats.describe(olr))
#print(stats.describe(nn34))
### Matching time for 3-mon prediciton of nino3.4
olr, nn34 = olr[:-3, :], nn34[3:]
###------- ML part -------###
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.model_selection import train_test_split
#from sklearn.neural_network import MLPClassifier
from sklearn.metrics import classification_report, confusion_matrix
import tensorflow as tf
from tensorflow.keras import layers, models
from tensorflow.keras.callbacks import EarlyStopping
tf.compat.v1.enable_eager_execution()
### Input data scaling
#scaler= StandardScaler() #MinMaxScaler(feature_range=(-1,1)) #
#X = scaler.fit_transform(olr)
X = olr / np.std(olr, ddof=1)
### Label encoding
one_std = 0.6 #np.std(nn34,ddof=1)
y = np.digitize(nn34, [-one_std, one_std]) #-1
nn_label = ['La Ni\u00F1a', 'Neutral', 'El Ni\u00F1o']
print("\nX data range: {:.3f} to {:.3f}".format(X.min(), X.max()))
print("y data distribution")
for lab, val in zip(nn_label, np.unique(y)):
print("{:3d}={:10s}: {:5d}".format(val, lab, (y == val).sum()))
### Train-Test split: 5 years for test, and no shuffle for testing recent years
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=60 /
y.shape[0],
shuffle=False)
### Shuffle train data before start
#rg = np.random.default_rng(seed=1) ## Numpy version>=1.17
rg = np.random.RandomState(seed=1) ## Numpy version<1.17
shf_idx = np.arange(y_train.shape[0], dtype=int)
rg.shuffle(shf_idx)
X_train, y_train = X_train[shf_idx, :], y_train[shf_idx]
print('Shuffling of train data is done.')
### MLP in Tensorflow+Keras
#model= tf.keras.Sequential([
# layers.Dense(15, activation='relu'),
# layers.Dense(len(nn_label), activation='relu') ])
model = models.Sequential()
model.add(layers.Dense(10, activation='relu'))
model.add(layers.Dense(6, activation='relu'))
model.add(layers.Dropout(0.2))
model.add(layers.Dense(len(nn_label), activation='relu'))
model.compile(
optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['sparse_categorical_accuracy'])
usualCallback = EarlyStopping(
monitor='loss', patience=25) #monitor='sparse_categorical_accuracy')
#overfitCallback = EarlyStopping(monitor='loss', min_delta=0, patience = 25)
history = model.fit(X_train,
y_train,
epochs=1200,
callbacks=[
usualCallback,
])
model.summary()
fns.plot_loss(history.history)
plt.show()
#for key in history.history.keys():
# print(key,history.history[key])
## Prediction with softmax
prob_model = models.Sequential([model, layers.Softmax()])
y_pred0 = np.argmax(prob_model(X_train), axis=1)
y_pred = np.argmax(prob_model(X_test), axis=1)
## Training score
print("\nTraining Score")
print(classification_report(y_train, y_pred0))
print("Confusion matrix, without normalization")
print(confusion_matrix(y_train, y_pred0).T)
## Test Score
print("\nTest Score")
print(classification_report(y_test, y_pred))
cm = confusion_matrix(y_test, y_pred).T
fns.plot_confusion_matrix(cm, labels=nn_label)
plt.show()
return
def pre_trained_embedding_with_imdb():
"""
Similar to the last function, this function demonstrates the use of embeddings with the IMDB dataset, however
this time we're using a pre-trained embedding layer: the GloVe embedding created at Stanford.
:return: None
"""
# Get the directory of the IMDB dataset
imdb_dir = 'C:\\Datasets\\IMDB\\aclImdb'
train_dir = os.path.join(imdb_dir, 'train')
# Prepare lists for the labels and the text inputs from the dataset
labels = []
texts = []
# Extract all the text inputs, negative first then positive. Store the inputs and the labels in order.
for label_type in ['neg', 'pos']:
dir_name = os.path.join(train_dir, label_type)
for fname in os.listdir(dir_name):
if fname[-4:] == '.txt':
f = open(os.path.join(dir_name, fname), encoding="utf8")
texts.append(f.read())
f.close()
if label_type == 'neg':
labels.append(0)
else:
labels.append(1)
# Settings for the system: cut reviews off after 100 words, train on 200 samples, validate on 10,000 samples,
# use only the top 10,000 words
max_len = 100
training_samples = 200
validation_samples = 10000
max_words = 10000
# Build the tokeniser
tokeniser = Tokenizer(num_words=max_words)
tokeniser.fit_on_texts(texts)
sequences = tokeniser.texts_to_sequences(texts)
word_index = tokeniser.word_index
print(f"Found {len(word_index)} unique tokens\n")
data = preprocessing.sequence.pad_sequences(sequences, maxlen=max_len)
labels = np.asarray(labels)
print(f"Shape of data tensor: {data.shape}")
print(f"Shape of labels tensor: {labels.shape}\n")
# Shuffle the data and split it into training and validation sets
indices = np.arange(data.shape[0])
np.random.shuffle(indices)
data = data[indices]
labels = labels[indices]
x_train = data[:training_samples]
y_train = labels[:training_samples]
x_val = data[training_samples:training_samples + validation_samples]
y_val = labels[training_samples:training_samples + validation_samples]
# Open the 100d glove embedding file and extract the embeddings into a dictionary
glove_dir = 'C:\\Datasets\\GloVe Embeddings'
embeddings_idx = {}
f = open(os.path.join(glove_dir, 'glove.6B.100d.txt'), encoding="utf8")
for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
embeddings_idx[word] = coefs
f.close()
print(f"Found {len(embeddings_idx)} word vectors\n")
# Now we need an Embedding matrix which must be of shape (max_words, embedding_dim), where each i contains the
# embedding_dim dimensional vector for the word of index i in the reference word index. Note that index 0 is
# only a placeholder.
embedding_dim = 100
embedding_matrix = np.zeros((max_words, embedding_dim))
for word, i in word_index.items():
if i < max_words:
embedding_vector = embeddings_idx.get(word)
if embedding_vector is not None:
# Words not encountered in the embedding index will be given all zeros
embedding_matrix[i] = embedding_vector
# Now let's build a simple model and set the weights of the embedding layer using the matrix we just generated
model = models.Sequential()
model.add(layers.Embedding(max_words, embedding_dim, input_length=max_len))
model.add(layers.Flatten())
model.add(layers.Dense(32, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))
# Freeze the embedding layer to prevent forgetting the pre-learned embedding space
model.layers[0].set_weights([embedding_matrix])
model.layers[0].trainable = False
# Compile and train the model
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
model.summary()
history = model.fit(x_train, y_train, epochs=10, batch_size=32, validation_data=(x_val, y_val))
# Save the model
model.save('C:\\Datasets\\IMDB\\pretrained_glove_model.h5')
# Plot the training and validation accuracy and loss
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(acc) + 1)
plt.plot(epochs, acc, 'bo', label='Training Acc')
plt.plot(epochs, val_acc, 'b', label='Validation Acc')
plt.title('Training and Validation Accuracy vs. Epochs')
plt.legend()
plt.figure()
plt.plot(epochs, loss, 'bo', label='Training Loss')
plt.plot(epochs, val_loss, 'b', label='Validation Loss')
plt.title('Training and Validation Loss vs. Epochs')
plt.legend()
plt.show()
# Now for testing purposes, lets compile the test data from the dataset and evaluate the model
test_dir = os.path.join(imdb_dir, 'test')
labels = []
texts = []
for label_type in ['neg', 'pos']:
dir_name = os.path.join(test_dir, label_type)
for fname in sorted(os.listdir(dir_name)):
if fname[-4:] == '.txt':
f = open(os.path.join(dir_name, fname), encoding="utf8")
texts.append(f.read())
f.close()
if label_type == 'neg':
labels.append(0)
else:
labels.append(1)
sequences = tokeniser.texts_to_sequences(texts)
x_test = preprocessing.sequence.pad_sequences(sequences, max_len)
y_test = np.asarray(labels)
# Evaluate the model
model.evaluate(x_test, y_test)
#/*************************************************************************************/
import random
import gym
import numpy as np
from tensorflow.keras import models, layers
env = gym.make("CartPole-v0") # 加载游戏环境
STATE_DIM, ACTION_DIM = 4, 2 # State 维度 4, Action 维度 2
model = models.Sequential([
layers.Dense(64, input_dim=STATE_DIM, activation='relu'),
layers.Dense(40, activation='relu'),
layers.Dense(ACTION_DIM, activation='linear')
])
model.summary() # 打印神经网络信息
#/*************************************************************************************/
def generate_data_one_episode():
x,y,score=[],[],0
state = env.reset()
while True:
action = random.randrange(0,2)
x.append(state)
y.append([1,0] if action ==0 else [0,1])
state ,reward ,done,_ = env.step(action)
score+=reward
if done:
break
return x,y,score
#/*************************************************************************************/
def generate_training_data(expected_score=100):
def rnn_example():
"""
A function which briefly goes through how to implement simple Recurrent Neural Networks using the IMDB dataset
as an example
:return: None
"""
# We'll build a very simple model first using an embedding layer and an RNN layer, just to show how to import
# these layers from Keras.layers. Pay close attention to the output shape of the RNN layer.
model = models.Sequential()
model.add(layers.Embedding(10000, 32))
model.add(layers.SimpleRNN(32))
model.summary()
model = None
# Now we'll show an example where there are multiple RNN layers in sequence. Note first that it is now required
# to return the full output sequence for each hidden RNN layer, however the final output layer must only return
# it's last output value.
model = models.Sequential()
model.add(layers.Embedding(10000, 32))
model.add(layers.SimpleRNN(32, return_sequences=True))
model.add(layers.SimpleRNN(32, return_sequences=True))
model.add(layers.SimpleRNN(32, return_sequences=True))
model.add(layers.SimpleRNN(32, return_sequences=True))
model.add(layers.SimpleRNN(32))
model.summary()
model = None
# Now let's try processing the IMDB dataset using this simple network. Let's load the data and pad the input
# sequences to the size required
max_features = 10000
max_len = 500
batch_size = 128
print("\nLoading data...")
(input_train, output_train), (input_test, output_test) = imdb.load_data(num_words=max_features)
print(f"{len(input_train)} training sequences")
print(f"{len(input_test)} testing sequences")
print("Pad sequences (samples x time)")
input_train = preprocessing.sequence.pad_sequences(input_train, max_len)
input_test = preprocessing.sequence.pad_sequences(input_test, max_len)
print(f"input_train shape: {input_train.shape}")
print(f"input_test shape: {input_test.shape}")
# Now we'll build a small and simple model using the SimpleRNN layer to classify the IMDB reviews
model = models.Sequential()
model.add(layers.Embedding(max_features, 32))
model.add(layers.SimpleRNN(32))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
model.summary()
# Now train the model and plot its performance
history = model.fit(input_train, output_train, epochs=10, batch_size=batch_size, validation_split=0.2)
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(acc) + 1)
plt.plot(epochs, acc, 'bo', label='Training Acc')
plt.plot(epochs, val_acc, 'b', label='Validation Acc')
plt.title('Training and Validation Accuracy vs. Epochs')
plt.legend()
plt.figure()
plt.plot(epochs, loss, 'bo', label='Training Loss')
plt.plot(epochs, val_loss, 'b', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.legend()
plt.show()
img = cv2.imread("PokemonDataset/" + row[3], cv2.IMREAD_GRAYSCALE)
img = cv2.resize(img, (32, 32))
x_train_l.append(img.reshape((32, 32, 1)))
y_new = np.zeros(493)
y_new[row[4] - 1] = 1
y_train_l.append(y_new)
x_train = np.array(x_train_l)
y_train = np.array(y_train_l)
model = models.Sequential([
layers.Conv2D(256, (3, 3),
input_shape=(32, 32, 1),
padding="same",
activation="relu"),
layers.MaxPool2D((2, 2), padding="same"),
layers.Conv2D(128, (3, 3), padding="same", activation="relu"),
layers.MaxPool2D((2, 2), padding="same"),
layers.Conv2D(64, (3, 3), padding="same", activation="relu"),
layers.MaxPool2D((2, 2), padding="same"),
layers.Conv2D(32, (3, 3), padding="same", activation="relu"),
layers.Flatten(),
layers.Dense(200),
layers.Dense(493, activation="softmax")
])
model.compile(loss=losses.CategoricalCrossentropy(), metrics=["acc"])
history = model.fit(np.divide(x_train, 255), y_train, epochs=50)
model.save("best_classifier_493.h5", save_format="h5")
def temperature_forecasting_example():
"""
This function will look at a more advanced application for Recurrent Neural Networks: forecasting the
temperature based on previous data. More specifically, the goal will be to create a model which uses the
weather data from the past to predict the temperature 24hrs from now.
:return: None
"""
# First let's set up the path to the data
data_dir = 'C:\\Datasets\\jena_climate'
fname = os.path.join(data_dir, 'jena_climate_2009_2016.csv')
# Let's open the file, read the data, close the file and then pull the data apart nto more useful structures.
f = open(fname)
data_from_file = f.read()
f.close()
lines = data_from_file.split('\n')
header = lines[0].split(',')
lines = lines[1:]
print(f"Column headers in the dataset: ")
for name in header:
print(name)
print(f"\nThe data is of shape: ({len(lines)}, {len(header)})")
# Now let's convert all 420,551 lines of data into a Numpy array. For this dataset measurements were taken every
# 10 minutes.
float_data = np.zeros((len(lines), len(header) - 1))
for i, line in enumerate(lines):
values = [float(x) for x in line.split(',')[1:]]
float_data[i, :] = values
temp = float_data[:, 1]
plt.plot(range(len(temp)), temp)
plt.title('Temperature Measurements across Time')
plt.xlabel('Sample #')
plt.ylabel('Temp (deg C)')
plt.show()
plt.plot(range(1440), temp[:1440])
plt.title('Temperature Measurements across 1st 10 Days')
plt.xlabel('Sample #')
plt.ylabel('Temp (deg C)')
plt.show()
# Now let's prepare the data for presentation to a Neural Network. We'll use the first 200,000 samples for
# training, so only pre-process those inputs
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=RuntimeWarning)
mean = np.mean(float_data[:200000], axis=0)
float_data -= mean
std = np.std(float_data[:200000], axis=0)
float_data /= std
look_back = 1440
step_size = 6
delay_size = 144
batch = 128
# Now we'll make a generator that takes the current array of float data and yields batches of data from the
# recent past, along with a target temperature in the future. Because the dataset is largely redundant (sample N
# and sample N+1 will have most of their timestamps in common), it would be wasteful to explicitly allocate
# every sample
def generator(data_input, lookback: int, delay: int, min_index: int, max_index: int = None,
shuffle: bool = False, batch_size: int = 128, step: int = 6):
if max_index is None:
max_index = len(data_input) - delay - 1
assert min_index < max_index
idx = max_index + lookback
while True:
if shuffle:
rows = np.random.randint(min_index, max_index, size=batch_size)
else:
if idx + batch_size >= max_index:
idx = min_index + lookback
rows = np.arange(i, min(i + batch_size, max_index))
idx += len(rows)
samples = np.zeros((len(rows), lookback // step, data_input.shape[-1]))
targets = np.zeros((len(rows),))
for idx2, row in enumerate(rows):
slice_begin = max(0, rows[idx2] - lookback)
if slice_begin == 0:
slice_end = lookback
else:
slice_end = rows[idx2]
indices = slice(slice_begin, slice_end, step)
samples[idx2] = data_input[indices]
targets[idx2] = data_input[rows[idx2] + delay][1]
yield samples, targets
train_gen = generator(float_data, look_back, delay_size, min_index=0, max_index=200000, shuffle=True,
step=step_size, batch_size=batch)
val_gen = generator(float_data, look_back, delay_size, min_index=200001, max_index=300000, shuffle=False,
step=step_size, batch_size=batch)
test_gen = generator(float_data, look_back, delay_size, min_index=300001, max_index=None, shuffle=False,
step=step_size, batch_size=batch)
val_steps = (300000 - 200001 - look_back)
test_steps = (len(float_data) - 300001 - look_back)
# For the sake of comparison it's often quite valuable to create a deterministic baseline against which to
# compare the ML model. In this case of predicting temperature, we can assume that the temperature tomorrow
# would be very similar to the temperature today, so using the Mean Absolute Error (MAE) metric we'd expect the
# ML model to have a lower MAE than a model which simply states that the temperature tomorrow is the same as the
# temperature today.
def evaluation_naive_method():
batch_maes = []
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=RuntimeWarning)
for step in range(val_steps):
samples, targets = next(val_gen)
preds = samples[:, -1, -1]
mae = np.nanmean(np.abs(preds - targets))
batch_maes.append(mae)
print(np.mean(batch_maes))
evaluation_naive_method()
# In the same way that using a non-ML baseline is useful, it's also quite useful to attempt a simple network
# first to establish an ML baseline. This will mean that any further complexity thrown at the problem will be
# justified.
model = models.Sequential()
model.add(layers.Flatten(input_shape=(look_back // step_size, float_data.shape[-1])))
model.add(layers.Dense(32, activation='relu'))
model.add(layers.Dense(1))
model.compile(optimizer=RMSprop(), loss='mae')
# This is not working at all. Validation is simply failing constantly
history = model.fit(train_gen, steps_per_epoch=500, epochs=20, validation_data=val_gen, validation_steps=500)
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)
plt.plot(epochs, loss, 'bo', label='Training Loss')
plt.plot(epochs, val_loss, 'b', label='Validation Loss')
plt.title('Training and Validation Loss vs. Epochs')
plt.legend()
plt.show()
model = None
# Now let's try a Recurrent network. Rather than an LSTM, let's try a Gated Recurrent Unit (GRU), which work
# using the same principals as LSTM's but are somewhat streamlined and thus cheaper to run.
model = models.Sequential()
model.add(layers.GRU(32, input_shape=(None, float_data.shape[-1])))
model.add(layers.Dense(1))
model.compile(optimizer=RMSprop(), loss='mae')
# This is not working at all. Validation is simply failing constantly, and it takes a year to complete.
history = model.fit(train_gen, steps_per_epoch=500, epochs=20, validation_data=val_gen,
validation_steps=val_steps)
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)
plt.plot(epochs, loss, 'bo', label='Training Loss')
plt.plot(epochs, val_loss, 'b', label='Validation Loss')
plt.title('Training and Validation Loss vs. Epochs')
plt.legend()
plt.show()
# Given that these networks do not appear to be training as expected, I will now simply list the remaining
# network topographies that can be used for this problem and give a few words to why they work.
# We're already familiar with the idea of dropout for deep neural networks. However, applying a random dropout
# mask to the recurrent branch of the network will greatly disrupt the signal on the feedback loop and hinder
# training. The correct approach is to apply a temporally constant dropout mask to the feedback loop, allowing
# the network to train with the presence of the error signal and avoid overfitting. Hence there are 2 dropout
# values: one for the input and one for the feedback loop.
model = models.Sequential()
model.add(layers.GRU(32, dropout=0.2, recurrent_dropout=0.2, input_shape=(None, float_data.shape[-1])))
model.add(layers.Dense(1))
model.compile(optimizer=RMSprop(), loss='mae')
# Depending on the overfitting performance of the previous designs, the next tactic is to increase the capacity
# of the network, achieved by adding more units to layers and more layers to the network. Note that when
# stacking recurrent layers you must ensure that intermediate layers return their entire sequence output, rather
# than just the last output
model = models.Sequential()
model.add(layers.GRU(32, dropout=0.2, recurrent_dropout=0.2, return_sequences=True,
input_shape=(None, float_data.shape[-1])))
model.add(layers.GRU(64, dropout=0.1, recurrent_dropout=0.5))
model.add(layers.Dense(1))
model.compile(optimizer=RMSprop(), loss='mae')
# Now we could try increasing the complexity of the network design. Here we'll attempt the use of a
# bi-directional RNN. This layout (having 2 RNN's working together, one processing the data in chronological
# order and one in antichronological order) works incredibly well on time-sensitive or order-sensitive data, and
# as such they are the go-to for natural language processing problems. By viewing the input sequence both ways
# the system can learn to detect patterns that may go overlooked in unidirectional processing. However they do
# run into problems on sequences data where the recent past is much more informative than the beginning of the
# sequence.
model = models.Sequential()
model.add(layers.Bidirectional(layers.GRU(32), input_shape=(None, float_data.shape[-1])))
model.add(layers.Dense(1))
model.compile(optimizer=RMSprop(), loss='mae')
"pitch_type_SI", "pitch_type_FT", "pitch_type_FC", "bat_score",
"post_bat_score", "pitch_type_CU", "was_3_1", "pitch_type_FS",
"was_1_1", "was_0_1", "was_2_1", "was_1_2", "was_2_2", "was_0_2",
"description", "zone"
]
for item in to_pop:
df.pop(item)
X = df.values
y = np.load('nn_y.npy')
nn_model = models.Sequential([
layers.Flatten(input_shape=(26, )),
layers.Dense(64, activation='relu'),
layers.Dense(64, activation='relu'),
layers.Dense(64, activation='relu'),
layers.Dense(64, activation='relu'),
layers.Dense(64, activation='relu'),
layers.Dense(64, activation='relu'),
layers.Dense(3, activation='softmax')
])
nn_model.load_weights('nn_686.py.wts')
cnn_model = models.Sequential([
layers.Conv2D(9, (2, 2), activation='relu', input_shape=(13, 2, 1)),
layers.Flatten(),
layers.Dense(64, activation='relu'),
layers.Dense(64, activation='relu'),
layers.Dense(64, activation='relu'),
layers.Dense(64, activation='relu'),
layers.Dense(64, activation='relu'),
layers.Dense(64, activation='relu'),
def one_dim_convenet_example():
"""
This function will revisit some previous examples and demonstrate the usefulness and applications of 1D convnets
for text and sequence learning.
:return: None
"""
max_features = 10000
max_len = 500
# Extract the data
print("Loading Data...")
(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
# Turns the lists of integers into 2D integer tensors of shape (samples, max_len)
print("Pad Sequences: (samples x time)")
x_train = preprocessing.sequence.pad_sequences(x_train, max_len)
x_test = preprocessing.sequence.pad_sequences(x_test, max_len)
print(f"x_train shape: {x_train.shape}")
print(f"x_test shape: {x_test.shape}")
model = models.Sequential()
model.add(layers.Embedding(max_features, 128, input_length=max_len))
model.add(layers.Conv1D(32, 7, activation='relu'))
model.add(layers.MaxPooling1D(5))
model.add(layers.Conv1D(32, 7, activation='relu'))
model.add(layers.GlobalMaxPool1D())
model.add(layers.Dense(1))
model.compile(optimizer=RMSprop(lr=1e-4), loss='binary_crossentropy', metrics=['acc'])
model.summary()
history = model.fit(x_train, y_train, epochs=8, batch_size=128, validation_split=0.2)
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)
plt.plot(epochs, acc, 'bo', label='Training Accuracy')
plt.plot(epochs, val_acc, 'b', label='Validation Accuracy')
plt.title('Training and Validation Accuracy vs. Epochs')
plt.legend()
plt.figure()
plt.plot(epochs, loss, 'bo', label='Training Loss')
plt.plot(epochs, val_loss, 'b', label='Validation Loss')
plt.title('Training and Validation Loss vs. Epochs')
plt.legend()
plt.show()
evaluations = model.evaluate(x_test, y_test)
print(f"Loss: {evaluations[0]}\tAccuracy: {evaluations[1] * 100}%")
# The previous example shows how 1D convnets can be used to replace simple networks that Flatten the input
# sequence, providing a cheaper alternative for competitive performance. The following examples work on the jena
# climate data from the previous example. Given the amount of issues faced in that function, this function will
# instead simply list the model topographies used and provide their summaries
float_data = np.zeros((1, 1))
# This first approach takes the simplest road possible, providing a suitable benchmark. This model, given it's
# low representational space, will likely overfit very strongly.
model = models.Sequential()
model.add(layers.Conv1D(32, 5, activation='relu', input_shape=(None, float_data.shape[-1])))
model.add(layers.MaxPooling1D(3))
model.add(layers.Conv1D(32, 5, activation='relu'))
model.add(layers.MaxPooling1D(3))
model.add(layers.Conv1D(32, 5, activation='relu'))
model.add(layers.GlobalMaxPool1D())
model.add(layers.Dense(1))
model.summary()
# This second approach utilises 1D convnets to simplify the sequence data coming into the network before passing
# the learned features on to the RNN section of the network. This should mean that the RNN is now learning to
# predict future data based data with higher representational power.
model = models.Sequential()
model.add(layers.Conv1D(32, 5, activation='relu', input_shape=(None, float_data.shape[-1])))
model.add(layers.MaxPooling1D(3))
model.add(layers.Conv1D(32, 5, activation='relu'))
model.add(layers.GRU(32, dropout=0.1, recurrent_dropout=0.5))
model.add(layers.Dense(1))
model.summary()
def convBlock(self,
convolutions,
dense=[],
pool='max',
conv_act=None,
conv_bias=True,
dense_bias=True,
input_type="one_hot",
regularizer=None,
embedding_dim=None,
dropout_rate=0,
regul_dense_only=True):
"""
Make sequential convolutional DNN building block
Parameters:
- convolutions -- specification of convolutional layers
as list of tuples:
either (kernel width, n filters, stride)
or (kernel width, n filters) if stride = 1
- dense -- specification of dense layers as list of
their width except the last one
- pool -- pooling operation either:
'max' for maximum or
'aver' for average
- conv_act -- activation of convolutional layers
- conv_bias -- use bias for convolutional layers
- dense_bias -- use bias for dense layers
- input_type -- type of input layer:
- one_hot - one hot coding of input samples
- categorical - categorical coding
decoded by preprocessing layer
- trainable - categorical coding processed
by trainable embedding layer
- regularizer -- regularizer applied to weights of dense layer
and to convolutionand embedding layers if
parameters regul_dense_only = False
- embedding_dim -- size of embedding vectors
It effects trainable embedding only,
i.e. input_type == "trainable"
If it is None self.n_tokens is used.
If it is None self.n_tokens is used
- dropout_rate -- dropout rate for inserted dropout layer
If it is 0 or None no droput layer is inserted
- regul_dense_only -- flag to regulirize only dense layers
Returns:
- dnn -- constructed DNN building block
"""
_conv_regul = None if regul_dense_only else regularizer
_embed_regul = None if regul_dense_only else regularizer
_dnn = models.Sequential()
if input_type == "one_hot":
_dnn.add(layers.Input(shape=(
None,
self.n_tokens,
)))
elif input_type == "categorical" or input_type == "trainable":
_trainable = (input_type == "trainable")
_dnn.add(layers.Input(shape=(None, )))
self.addEmbedding(_dnn,
trainable=_trainable,
width=embedding_dim,
regularizer=_embed_regul)
else:
sys.exit(
f"Wrong type {input_type} of input_type parameter of functional convolution block"
)
self.addConv1(_dnn,
convolutions,
activation=conv_act,
bias=conv_bias,
out_no_act=(pool == 'max'),
regularizer=_conv_regul)
self.addPooling(_dnn, pool)
self.addDense(_dnn, dense, bias=dense_bias, regularizer=regularizer)
self.addDropout(_dnn, dropout_rate)
return _dnn
train_images, train_labels = train_images[:
split_value], train_labels[:
split_value]
# dataset with 10 different classes
class_names = [
'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight',
'nine'
]
# keras API for creating the CNN architecture
model = models.Sequential([
layers.Conv2D(28, (3, 3), activation='relu', input_shape=(28, 28, 1)),
layers.MaxPooling2D((2, 2)),
layers.Conv2D(64, (3, 3), activation='relu'),
layers.MaxPooling2D((2, 2)),
layers.Conv2D(64, (3, 3), activation='relu'),
layers.Flatten(),
layers.Dense(64, activation='relu'),
layers.Dense(10),
])
# compile and train the model
model.compile(
optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
history = model.fit(train_images,
train_labels,
epochs=10,
validation_data=(validation_images, validation_labels))
from tensorflow.keras import datasets, layers, models, optimizers, callbacks
from keras.callbacks import EarlyStopping, ModelCheckpoint
import matplotlib.pyplot as plt
# !pip install --upgrade tensorflow
"""# Building ConvNet Configuration Types
https://arxiv.org/pdf/1409.1556.pdf (Page 3, Tables 1 & 2)
"""
mnist_shape = (28, 28, 1)
mnist_classes = 10
"""## Configuration: A (11 weight layers)"""
model_A = models.Sequential()
model_A.add(
layers.Conv2D(64, (3, 3),
padding='same',
activation='relu',
input_shape=mnist_shape))
model_A.add(layers.MaxPool2D((2, 2), strides=2, padding='same'))
model_A.add(layers.Conv2D(128, (3, 3), padding='same', activation='relu'))
model_A.add(layers.MaxPool2D((2, 2), strides=2, padding='same'))
model_A.add(layers.Conv2D(256, (3, 3), padding='same', activation='relu'))
model_A.add(layers.Conv2D(256, (3, 3), padding='same', activation='relu'))
model_A.add(layers.MaxPool2D((2, 2), strides=2, padding='same'))
#keras_mobilenet_v2.trainable = False
#estimator_model = tf.keras.Sequential([
# keras_mobilenet_v2,
# tf.keras.layers.GlobalAveragePooling2D(),
# tf.keras.layers.Dense(1)
#])
estimator_model = models.Sequential([
Conv2D(16,
3,
padding='same',
activation='softmax',
input_shape=(160, 160, 3)),
MaxPooling2D(),
Conv2D(16, 3, padding='same', activation='relu'),
MaxPooling2D(),
Conv2D(16, 3, padding='same', activation='relu'),
MaxPooling2D(),
Flatten(),
Dense(512, activation='relu'),
Dense(1)
])
# Compile the model
estimator_model.compile(
optimizer='adam',
loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
metrics=['accuracy'])
model_dir = "/tmp/fruits/"
est_model = tf.keras.estimator.model_to_estimator(keras_model=estimator_model,
if X_test_A1 is not None:
y_pred_test_A1 = clf_A1.predict(X_test_A1)
print(colored("TASK A1 - TEST SET METRICS", "cyan"))
print(sklearn.metrics.classification_report(Y_test_A1, y_pred_test_A1))
## ======================================================================================================================
# Task A2 solution - CNN
nb_filters = 16
nb_pool = 2
nb_conv = 3
nb_classes = 2
activation = "relu"
clf_A2 = models.Sequential()
clf_A2.add(
layers.Conv2D(nb_filters, (nb_conv, nb_conv),
activation=activation,
input_shape=X_train_A2.shape[1:]))
clf_A2.add(layers.Conv2D(nb_filters, (nb_conv, nb_conv),
activation=activation))
clf_A2.add(layers.MaxPooling2D(pool_size=(nb_pool, nb_pool)))
clf_A2.add(layers.Dropout(0.25))
clf_A2.add(layers.Flatten())
clf_A2.add(layers.Dense(64, activation=activation))
clf_A2.add(layers.Dropout(0.5))
clf_A2.add(layers.Dense(nb_classes))
clf_A2.compile(
train_labels = train_labels.to_numpy()
test_labels = test_labels.to_numpy()
##### Modelling
### Festlegen der Anzahl Klassen
number_of_classes = train_labels.shape[1]
# speichern der vorbereiteten Daten
#with open("small_data.pickle","wb") as f:
# pickle.dump([number_of_classes, channels, height, width, train_images, train_labels, test_images, test_labels, class_labels],f)
# Setzen eines Seeds
np.random.seed(45)
# Beginn des neuronalen Netzes
cnn = models.Sequential()
# Füge einen Covolutional-Layer mit 32 Filtern und zero-padding einem 3x3 Kernel hinzu. Die Aktivierungsfunktion dieses Layers ist die ReLU-Funktion.
cnn.add(
Conv2D(filters=32,
kernel_size=(3, 3),
input_shape=(channels, width, height),
activation="relu",
data_format="channels_first",
padding="same"))
# Füge einen weiteren Covolutional-Layer mit 32 Filtern und einem 3x3 Kernel hinzu. Die Aktivierungsfunktion dieses Layers ist die ReLU-Funktion.
cnn.add(
Conv2D(filters=32,
kernel_size=(3, 3),
data_format="channels_first",
testY = utils.to_categorical(testY)
trainX = trainX.astype('float32')
testX = testX.astype('float32')
trainX = trainX / 255.0
testX = testX / 255.0
# Create validation data set
validX = trainX[0:5000]
validY = trainY[0:5000]
trainX = trainX[5000:]
trainY = trainY[5000:]
''' Create the NN-model '''
model = models.Sequential()
model.add(
layers.Conv2D(32, (3, 3),
activation='relu',
kernel_initializer='he_uniform',
padding='same',
input_shape=(32, 32, 3)))
model.add(
layers.Conv2D(32, (3, 3),
activation='relu',
kernel_initializer='he_uniform',
padding='same'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Dropout(0.1))
def mnist_conv_net():
print("Let's first take a look at what a typical convnet looks like:")
model = models.Sequential()
model.add(
layers.Conv2D(32, (3, 3),
activation='relu',
input_shape=(28, 28, 1)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.Flatten())
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(10, activation='softmax'))
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
print(
"This particular model is actually quite useful for the mnist dataset we encountered in Chapter 2. As before,"
"we'll normalise the inputs and categorise the outputs.")
# Load data and print useful information
(train_images, train_labels), (test_images,
test_labels) = mnist.load_data()
print(
f"There are {len(train_images)} training images, each of shape {train_images[0].shape}: dtype = "
f"{train_images[0].dtype},")
print(
f"and {len(test_images)} training images, each of shape {test_images[0].shape}: dtype = "
f"{test_images[0].dtype}\n")
# Reshape and normalise the inputs for use in the neural network
print('Reshaping and normalising inputs:')
train_images = train_images.reshape((len(train_images), 28, 28, 1))
train_images = train_images.astype('float32') / 255
test_images = test_images.reshape((len(test_images), 28, 28, 1))
test_images = test_images.astype('float32') / 255
print(
f"Now there are {len(train_images)} training images, each of shape {train_images[0].shape}: dtype = "
f"{train_images[0].dtype},")
print(
f"and {len(test_images)} training images, each of shape {test_images[0].shape}: dtype = "
f"{test_images[0].dtype}\n")
# Convert the outputs to categorical labels which are more useful for training
print('Converting the outputs to categorical labels:')
train_labels = to_categorical(train_labels)
test_labels = to_categorical(test_labels)
# Train the model and evaluate the performance
print(
"\nNow let's train and evaluate the model to see how it performs on the 1st try:"
)
model.fit(train_images, train_labels, epochs=5, batch_size=64)
test_loss, test_acc = model.evaluate(test_images, test_labels)
print(
f"Test Accuracy = {test_acc * 100}%\nNote the immediate rise in test accuracy. This fairly naive network "
f"can easily outperform simple Dense networks due to convnets' inherent resistance to translational and spatial "
f"variations. But what more can convnets do?")
