def google_net(size=256, kernel=3):
model = Sequential()
model.add(Conv2D(32, (kernel, kernel),
activation='relu',
input_shape=(size, size, 3),
strides=2,
kernel_regularizer=regularizers.l2(0.01),
name='cv1'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (kernel, kernel),
activation='relu',
strides=2,
kernel_regularizer=regularizers.l2(0.01),
name='cv2'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(128, (kernel, kernel), activation='relu', strides=2, name='cv3.3'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(256, kernel_regularizer=regularizers.l2(0.01), name='features'))
model.add(Activation('relu'))
model.add(Dense(3, activation='softmax', name='denseout'))
print(model.summary())
model.compile(
loss='categorical_crossentropy',
optimizer=RMSprop(lr=1e-4, decay=0.1e-6),
metrics=['accuracy'])
return model
def _get_simple_sequential_model(compile_metrics):
model = Sequential()
model.add(
layers.Dense(
3, activation='relu', input_dim=4, kernel_initializer='ones'))
model.add(layers.Dense(1, activation='sigmoid', kernel_initializer='ones'))
model.compile(
loss='mae',
metrics=compile_metrics,
optimizer=RMSPropOptimizer(learning_rate=0.001))
return model
def get_model_1(x,y,Vocab_size,maxlen):
model = Sequential()
model.add(Embedding(Vocab_size, 30)) #input_length=maxlen))
model.add(LSTM(60, return_sequences=True))
model.add(Dense(60, activation='relu'))
model.add(Dense(Vocab_size))
model.add(Activation("softmax"))
print(model.summary())
optimizer = RMSprop(lr=0.01)
model.compile(loss="sparse_categorical_crossentropy", optimizer='adam')
return model
def get_model_1(x,y,Vocab_size,maxlen):
model = Sequential()
model.add(Embedding(Vocab_size, 50, input_length=maxlen))
model.add(LSTM(128, return_sequences=True))
model.add(LSTM(128))
model.add(Dense(128, activation='relu'))
model.add(Dense(Vocab_size))
model.add(Activation("softmax"))
print(model.summary())
model.compile(loss="sparse_categorical_crossentropy", optimizer='adam', metrics=['accuracy'])
return model
def keras_model_fn(hyperparameters):
"""keras_model_fn receives hyperparameters from the training job and returns a compiled keras model.
The model will be transformed into a TensorFlow Estimator before training and it will be saved in a
TensorFlow Serving SavedModel at the end of training.
Args:
hyperparameters: The hyperparameters passed to the SageMaker TrainingJob that runs your TensorFlow
training script.
Returns: A compiled Keras model
"""
model = Sequential()
model.add(Conv2D(32, (3, 3), padding='same', name='inputs', input_shape=(HEIGHT, WIDTH, DEPTH)))
model.add(Activation('relu'))
model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(64, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(NUM_CLASSES))
model.add(Activation('softmax'))
opt = RMSPropOptimizer(learning_rate=hyperparameters['learning_rate'], decay=hyperparameters['decay'])
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
return model
class Train:
TRAIN_DATA = './data/train'
VALIDATION_DATA = './data/validation'
EPOCHS = 5
WIDTH = 150
HEIGHT = 150
BATCH_SIZE = 32
STEPS = 500
VALIDATION_STEPS = 300
FILTERS_FIRST_CONV = 32
FILTERS_SECOND_CONV = 64
SIZE_FIRST_FILTER = (3, 3)
SIZE_SECOND_FILTER = (2, 2)
POOL_SIZE = (2, 2)
CLASSES = 2
LR = 0.0004
@classmethod
def preprocess_images(self):
self.training_datagen = ImageDataGenerator(rescale=1. / 255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
self.test_datagen = ImageDataGenerator(rescale=1. / 255)
self.training_generator = self.training_datagen.flow_from_directory(
os.path.join(sys.path[0], Train.TRAIN_DATA),
target_size=(Train.HEIGHT, Train.WIDTH),
batch_size=Train.BATCH_SIZE,
class_mode='categorical')
self.validation_generator = self.test_datagen.flow_from_directory(
os.path.join(sys.path[0], Train.VALIDATION_DATA),
target_size=(Train.HEIGHT, Train.WIDTH),
batch_size=Train.BATCH_SIZE,
class_mode='categorical')
return (self.training_generator, self.validation_generator)
@classmethod
def train_cnn(self):
(self.training_generator,
self.validation_generator) = Train.preprocess_images()
K.clear_session()
self.cnn = Sequential()
self.cnn.add(
Convolution2D(Train.FILTERS_FIRST_CONV,
Train.SIZE_FIRST_FILTER,
padding="same",
input_shape=(Train.HEIGHT, Train.WIDTH, 3),
activation='relu'))
self.cnn.add(MaxPooling2D(pool_size=Train.POOL_SIZE))
self.cnn.add(
Convolution2D(Train.FILTERS_SECOND_CONV,
Train.SIZE_SECOND_FILTER,
padding="same"))
self.cnn.add(MaxPooling2D(pool_size=Train.POOL_SIZE))
self.cnn.add(Flatten())
self.cnn.add(Dense(256, activation='relu'))
self.cnn.add(Dropout(0.5))
self.cnn.add(Dense(Train.CLASSES, activation='softmax'))
self.cnn.compile(loss='categorical_crossentropy',
optimizer=optimizers.Adam(lr=Train.LR),
metrics=['accuracy'])
self.cnn.fit_generator(self.training_generator,
steps_per_epoch=Train.STEPS,
epochs=Train.EPOCHS,
validation_data=self.validation_generator,
validation_steps=Train.VALIDATION_STEPS)
return self.cnn
@classmethod
def create_model(self):
self.cnn = Train.train_cnn()
target_dir = './model/'
if not os.path.exists(target_dir):
os.mkdir(target_dir)
self.cnn.save('./model/model.h5')
self.cnn.save_weights('./model/weights.h5')
def evaluate_model(trainX, trainy, testX, testy, testy_norm):
"""
Create, fit and evaluate a model
:param trainX: (array)
:param trainy: (array)
:param testX: (array)
:param testy: (array)
:param testy_norm: (array)
:return:
accurancy (float)
loss (float)
"""
verbose, epochs, batch_size = 1, 60, 64 # 16
trainX, testX = scale_data(trainX, testX)
# trainX, testX = Magnitude(trainX,testX)
# trainX, testX = AutoCorallation(trainX, testX)
n_timesteps, n_features, n_outputs = trainX.shape[1], trainX.shape[2], trainy.shape[1]
print(testX.shape)
print(testy.shape)
model = Sequential()
# Small structure
# model.add(Conv1D(32, 5, activation='tanh', padding='same', input_shape=(n_timesteps, n_features),kernel_regularizer=regularizers.l1(0.01))) #,kernel_regularizer=regularizers.l1(0.01))
# model.add(MaxPooling1D(pool_size=2))
# model.add(Conv1D(64, 5, activation='tanh', padding='same'))
# model.add(SpatialDropout1D(0.5))
# model.add(MaxPooling1D(pool_size=2))
# model.add(Conv1D(128, 5, activation='tanh', padding='same'))
# model.add(SpatialDropout1D(0.5))
# BatchNormalization()
# model.add(MaxPooling1D(pool_size= 2))
# model.add(Flatten())
# model.add(Dense(128, activation='tanh'))
# model.add(Dropout(0.5))
# model.add(Dense(64, activation='tanh'))
# model.add(Dropout(0.5))
#Big kernel
model.add(Conv1D(filters=32, kernel_size=32, activation='relu', input_shape=(n_timesteps, n_features),
padding='same'))
model.add(SpatialDropout1D(0.5))
model.add(MaxPooling1D(pool_size=3))
model.add(Conv1D(filters=32, kernel_size=16, activation='relu', padding='same'))
BatchNormalization()
model.add(Conv1D(filters=64, kernel_size=8, activation='relu', padding='same'))
model.add(SpatialDropout1D(0.5))
model.add(MaxPooling1D(pool_size=3))
model.add(Conv1D(filters=128, kernel_size=4, activation='relu', padding='same'))
BatchNormalization()
model.add(Conv1D(filters=128, kernel_size=3, activation='relu', padding='same'))
model.add(SpatialDropout1D(0.5))
model.add(MaxPooling1D(pool_size=3))
model.add(Conv1D(filters=128, kernel_size=2, activation='relu', padding='same'))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
BatchNormalization()
model.add(Dense(64, activation='relu'))
model.add(Dense(32, activation='relu'))
BatchNormalization()
model.add(Dense(16, activation='relu'))
model.add(Dense(6, activation='relu')) #kernel_regularizer=regularizers.l2(0.01)
model.add(Dense(n_outputs, activation='softmax'))
model.summary()
plot_model(model, 'model_info.png', show_shapes=True)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
# fit network
tensorboard = TensorBoard(log_dir="logs_big_stand_bez_reg/{}".format(time()),histogram_freq=1,write_images=True)
history = model.fit(trainX, trainy, epochs=epochs, batch_size=batch_size, verbose=verbose, validation_split=0.15,shuffle=True,callbacks=[tensorboard])
# evaluate model
loss, accuracy = model.evaluate(testX, testy, batch_size=batch_size, verbose=0)
export_model(model)
predictions = model.predict_classes(testX)
print(metrics.classification_report(testy_norm, predictions))
confusion_matrix = metrics.confusion_matrix(y_true = testy_norm, y_pred = predictions)
print(confusion_matrix)
normalised_confusion_matrix = np.array(confusion_matrix, dtype=np.float32) / np.sum(confusion_matrix) * 100
print("")
print("Confusion matrix (normalised to % of total test data):")
print(normalised_confusion_matrix)
width = 12
height = 12
# fig, ax = plt.subplots()
plt.figure(figsize=(width, height))
plt.imshow(
normalised_confusion_matrix,
interpolation='nearest',
cmap=plt.cm.rainbow
)
plt.title("Confusion matrix \n(normalized to the entire test set [%])")
plt.colorbar()
tick_marks = np.arange(6)
LABELS= ["Walk", "Walk up", "Walk down", "Sitting", "Standing", "Laying"]
plt.xticks(tick_marks, LABELS, rotation=90)
plt.yticks(tick_marks, LABELS)
plt.tight_layout()
plt.ylabel('Real value')
plt.xlabel('Prediction value')
plt.figure()
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training', 'Validation'], loc='upper left')
plt.figure()
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('Model accurancy')
plt.ylabel('Accurancy')
plt.xlabel('Epoch')
plt.legend(['Training', 'Validation'], loc='upper left')
plt.show()
return accuracy, loss
x_column = table_col - y_column
x = data[:, 0:x_column]
y = data[:, x_column]
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense
model = Sequential()
model.add(Dense(units=30, activation='relu'))
model.add(Dense(units=12, activation='relu'))
model.add(Dense(units=8, activation='relu'))
# tanh
model.add(Dense(units=y_column, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
import os
model_dir = './model/' # 모델을 저장할 폴더
if not os.path.exists(model_dir):
os.mkdir(model_dir)
# epoch: 에포크
# val_loss: 검증용 데이터 셋의 손실오차
model_name = model_dir + '{epoch:02d}-{val_loss:.4f}.hdf5'
from tensorflow.python.keras.callbacks import ModelCheckpoint
from tensorflow.python.keras.callbacks import EarlyStopping
# filepath: 파일을 저장할 경로, monitor: 모니터링할 대상
from tensorflow.python.keras.datasets import imdb from tensorflow.python.keras import preprocessing from tensorflow.python.keras.models import Sequential from tensorflow.python.keras import layers max_features = 10000 max_len = 20 (x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features) x_train = preprocessing.sequence.pad_sequences(x_train, maxlen=max_len) x_test = preprocessing.sequence.pad_sequences(x_test, maxlen=max_len) mdl = Sequential() mdl.add(layers.Embedding(10000, 8, input_length=max_len)) mdl.add(layers.Flatten()) mdl.add(layers.Dense(1, activation='sigmoid')) mdl.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc']) mdl.summary() hist = mdl.fit(x_train, y_train, epochs=10, batch_size=32, validation_split=.2)
scaler_y = MinMaxScaler()
xtrain_scaled = scaler_x.fit_transform(x_train)
ytrain_scaled = scaler_y.fit_transform(y_train)
xval_scaled = scaler_x.fit_transform(x_val)
# BUILDING THE NEURAL NETWORK
model = Sequential()
model.add(Dense(2, input_dim = 1, activation = 'relu', kernel_initializer='normal'))
model.add(Dense(70, activation = 'relu'))
model.add(Dense(70, activation = 'relu'))
model.add(Dense(70, activation = 'relu'))
model.add(Dense(1, activation = 'linear'))
# TRAINING THE DATA
model.compile(loss = 'mse', optimizer = 'adam', metrics = ['mse', 'mae', 'accuracy'])
history = model.fit(xtrain_scaled, ytrain_scaled, epochs = 100, batch_size = 70, validation_split = 0.1, verbose = 1)
print("\n\n ----- Model is trained successfully ! ----- \n\n")
# VISUALISING THE MODEL LOSS
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('MODEL LOSS')
plt.ylabel('Loss')
plt.xlabel('Epochs')
plt.legend(['Train', 'Validation'], loc='upper right')
plt.show()
# ACCURACY OF THE MODEL
ypred_scaled = model.predict(xval_scaled)
y_pred = scaler_y.inverse_transform(ypred_scaled)
# Add another max pooling layer
classifier.add(MaxPooling2D(pool_size=(4, 4)))
# Step 3: Flattening
classifier.add(Flatten())
# Step 4: Full connection
# classifier.add(Dense(output_dim=128, activation='relu'))
# classifier.add(Dense(output_dim=1, activation='sigmoid'))
classifier.add(Dense(units=units, activation='softmax'))
classifier.summary()
# Compiling the CNN
classifier.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
print('Fitting the CNN to the images')
# Fitting the CNN to the images
from keras.preprocessing.image import ImageDataGenerator
train_datagen = ImageDataGenerator(rescale=1. / 255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
print('Train set loaded')
train_set = train_datagen.flow_from_directory(
'C:/Users/Deeathex/PycharmProjects/TestSignLang1/TrainData',
target_size=(50, 50),
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from tensorflow.python import keras
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense, Flatten, Conv2D
dataset = pd.read_csv('train.csv')
Ytr = keras.utils.to_categorical(dataset.label, 10)
Xtr = dataset.values[:, 1:]
nrows = Xtr.shape[0]
Xtr = Xtr.reshape(nrows, 28, 28, 1)
Xtr = Xtr / 255
model = Sequential()
model.add(
Conv2D(20, kernel_size=(3, 3), activation='relu', input_shape=(28, 28, 1)))
model.add(Conv2D(20, kernel_size=(3, 3), activation='relu'))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dense(10, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer='adam',
metrics=['accuracy'])
model.fit(Xtr, Ytr, batch_size=128, epochs=2, validation_split=0.2)
def train_top_model_vgg():
# VGG16 generators
# Set up generators
train_batches = ImageDataGenerator(
preprocessing_function= \
applications.vgg16.preprocess_input).flow_from_directory(
train_path,
target_size=(image_size, image_size),
batch_size=train_batch_size)
valid_batches = ImageDataGenerator(
preprocessing_function= \
applications.vgg16.preprocess_input).flow_from_directory(
valid_path,
target_size=(image_size, image_size),
batch_size=val_batch_size)
test_batches = ImageDataGenerator(
preprocessing_function= \
applications.vgg16.preprocess_input).flow_from_directory(
test_path,
target_size=(image_size, image_size),
batch_size=test_batch_size,
shuffle=False)
my_model = applications.VGG16(include_top=True, weights='imagenet')
initial_model = Sequential()
for layer in my_model.layers[:22]:
initial_model.add(layer)
initial_model.layers.pop()
initial_model.add(Dense(1024, activation='relu'))
initial_model.add(Dropout(0.5))
initial_model.add(Dense(7, activation="softmax"))
print(initial_model.summary())
# Define Top2 and Top3 Accuracy
def top_3_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3)
def top_2_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=2)
initial_model.compile(optimizer=optimizers.SGD(lr=0.001, momentum=0.9, decay=0.0, nesterov=True),
loss="categorical_crossentropy",
metrics=[categorical_accuracy, top_2_accuracy, top_3_accuracy, "accuracy"])
filepath = "modelVGG.h5"
# Declare a checkpoint to save the best version of the model
checkpoint = ModelCheckpoint(filepath, monitor='val_categorical_accuracy', verbose=1,
save_best_only=True, mode='max')
# Reduce the learning rate as the learning stagnates
reduce_lr = ReduceLROnPlateau(monitor='val_categorical_accuracy', factor=0.5, patience=2,
verbose=1, mode='max', min_lr=0.00001)
early_stopping = EarlyStopping(monitor='val_categorical_accuracy', patience=5,
verbose=1, mode='max')
callbacks_list = [checkpoint, reduce_lr, early_stopping]
history = initial_model.fit_generator(
train_batches,
steps_per_epoch=train_steps,
epochs=epochs,
validation_data=valid_batches,
validation_steps=val_steps,
callbacks=callbacks_list)
try:
# summarize history for accuracy
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.savefig("VGG16_accuracy_training_plot.png")
# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.savefig("VGG16_loss_training_plot.png")
except:
print("Error")
# model.add(Dense(num_y_signals, activation='sigmoid'))
#linear statt sigmoid
model.add(Dense(num_y_signals,
activation='linear',
kernel_initializer=init))
warmup_steps = 500
#optimizer
optimizer = RMSprop(lr=1e-3)
model.compile(loss=loss_mse_warmup, optimizer=optimizer)
print(model.summary())
#tensorboard setup
path_checkpoint = 'checkpoint.keras'
#validation loss abspeichern
callback_checkpoint = ModelCheckpoint(filepath=path_checkpoint,
monitor='val_loss',
verbose=1,
save_weights_only=True,
save_best_only=True)
callback_early_stopping = EarlyStopping(monitor='val_loss',
patience=5, verbose=1)
def main_fun(args, ctx):
IMAGE_PIXELS = 28
num_classes = 10
# use Keras API to load data
from tensorflow.python.keras.datasets import mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(60000, 784)
x_test = x_test.reshape(10000, 784)
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
# setup a Keras model
model = Sequential()
model.add(Dense(512, activation='relu', input_shape=(784, )))
model.add(Dropout(0.2))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=tf.train.RMSPropOptimizer(learning_rate=0.001),
metrics=['accuracy'])
model.summary()
print("model.inputs: {}".format(model.inputs))
print("model.outputs: {}".format(model.outputs))
# convert Keras model to tf.estimator
estimator = tf.keras.estimator.model_to_estimator(model,
model_dir=args.model_dir)
# setup train_input_fn for InputMode.TENSORFLOW or InputMode.SPARK
if args.input_mode == 'tf':
# For InputMode.TENSORFLOW, just use data in memory
train_input_fn = tf.estimator.inputs.numpy_input_fn(
x={"dense_input": x_train},
y=y_train,
batch_size=128,
num_epochs=args.epochs,
shuffle=True)
hooks = []
else: # 'spark'
# For InputMode.SPARK, read data from RDD
tf_feed = TFNode.DataFeed(ctx.mgr)
def rdd_generator():
while not tf_feed.should_stop():
batch = tf_feed.next_batch(1)
if len(batch) > 0:
record = batch[0]
image = numpy.array(record[0]).astype(
numpy.float32) / 255.0
label = numpy.array(record[1]).astype(numpy.float32)
yield (image, label)
else:
return
def train_input_fn():
ds = tf.data.Dataset.from_generator(
rdd_generator, (tf.float32, tf.float32), (tf.TensorShape(
[IMAGE_PIXELS * IMAGE_PIXELS]), tf.TensorShape([10])))
ds = ds.batch(args.batch_size)
return ds
# add a hook to terminate the RDD data feed when the session ends
hooks = [StopFeedHook(tf_feed)]
# eval_input_fn ALWAYS uses data loaded in memory, since InputMode.SPARK can only feed one RDD at a time
eval_input_fn = tf.estimator.inputs.numpy_input_fn(
x={"dense_input": x_test}, y=y_test, num_epochs=1, shuffle=False)
# setup tf.estimator.train_and_evaluate() w/ FinalExporter
feature_spec = {
'dense_input': tf.placeholder(tf.float32, shape=[None, 784])
}
exporter = tf.estimator.FinalExporter(
"serving",
serving_input_receiver_fn=tf.estimator.export.
build_raw_serving_input_receiver_fn(feature_spec))
train_spec = tf.estimator.TrainSpec(input_fn=train_input_fn,
max_steps=args.steps,
hooks=hooks)
eval_spec = tf.estimator.EvalSpec(input_fn=eval_input_fn,
exporters=exporter)
# train and export model
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
# WORKAROUND FOR https://github.com/tensorflow/tensorflow/issues/21745
# wait for all other nodes to complete (via done files)
done_dir = "{}/done".format(ctx.absolute_path(args.model_dir))
print("Writing done file to: {}".format(done_dir))
tf.gfile.MakeDirs(done_dir)
with tf.gfile.GFile("{}/{}".format(done_dir, ctx.task_index),
'w') as done_file:
done_file.write("done")
for i in range(60):
if len(tf.gfile.ListDirectory(done_dir)) < len(
ctx.cluster_spec['worker']):
print("{} Waiting for other nodes {}".format(
datetime.now().isoformat(), i))
time.sleep(1)
else:
print("{} All nodes done".format(datetime.now().isoformat()))
break
plt.show()
# In[59]:
model = Sequential()
model.add(
Conv2D(input_shape=input_shape,
filters=32,
kernel_size=[3, 3],
padding="same"))
model.add(Conv2D(filters=8, kernel_size=[3, 3], activation='relu'))
model.add(Dense(10, activation='softmax'))
model.compile(loss=categorical_crossentropy,
optimizer=Adadelta(),
metrics=['accuracy'])
# In[63]:
num_classes = 10
model = Sequential()
model.add(
Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
x_column = table_col - 1 # 4 - 1 = 3
# 입력과 출력 데이터 분리
x = data[:, 0:x_column]
y = data[:, x_column:]
# 훈련용 데이터 셋과 테스트용 데이터셋 분리
seed = 0 # 같은 결과를 얻고 싶을 때 지정(값은 임의)
x_train, x_test, y_train, y_test = \
train_test_split(x, y, test_size=0.3, random_state=seed)
# 모델 생성
model = Sequential()
model.add(Dense(units=y_column, input_dim=x_column, activation='linear'))
model.compile(loss='mean_squared_error', optimizer='adam')
model.fit(x_train, y_train, epochs=5000, batch_size=10, verbose=0)
print(model.get_weights())
prediction = model.predict(x_test)
for idx in range(len(y_test)):
label = y_test[idx]
pred = prediction[idx]
print(f'real: {label}, prediction: {pred}')
print('finished')
activation='relu',
input_shape=input_shape))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Conv2D(32, (5, 5), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Conv2D(32, (5, 5), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Conv2D(64, (5, 5), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(1000, activation='relu'))
model.add(Dense(num_classes, activation='softmax'))
model.summary()
model.compile(loss=tensorflow.keras.losses.binary_crossentropy,
optimizer=tensorflow.keras.optimizers.Adam(),
metrics=['accuracy'])
class AccuracyHistory(tensorflow.keras.callbacks.Callback):
def on_train_begin(self, logs={}):
self.acc = []
def on_epoch_end(self, batch, logs={}):
self.acc.append(logs.get('acc'))
history = AccuracyHistory()
model.fit(x_train,
y_train,
class RNNGRU(object):
"""Process data for ingestion."""
def __init__(
self, data, periods=288, batch_size=64, sequence_length=20,
warmup_steps=50, epochs=20, display=False):
"""Instantiate the class.
Args:
data: Dict of values keyed by timestamp
periods: Number of timestamp data points per vector
batch_size: Size of batch
sequence_length: Length of vectors for for each target
warmup_steps:
Returns:
None
"""
# Initialize key variables
self.periods = periods
self.target_names = ['value']
self.warmup_steps = warmup_steps
self.epochs = epochs
self.batch_size = batch_size
self.display = display
###################################
# TensorFlow wizardry
config = tf.ConfigProto()
# Don't pre-allocate memory; allocate as-needed
config.gpu_options.allow_growth = True
# Only allow a total of half the GPU memory to be allocated
config.gpu_options.per_process_gpu_memory_fraction = 0.95
# Crash with DeadlineExceeded instead of hanging forever when your
# queues get full/empty
config.operation_timeout_in_ms = 60000
# Create a session with the above options specified.
backend.tensorflow_backend.set_session(tf.Session(config=config))
###################################
# Get data
(x_data, y_data) = convert_data(data, periods, self.target_names)
print('\n> Numpy Data Type: {}'.format(type(x_data)))
print("> Numpy Data Shape: {}".format(x_data.shape))
print("> Numpy Data Row[0]: {}".format(x_data[0]))
print('> Numpy Targets Type: {}'.format(type(y_data)))
print("> Numpy Targets Shape: {}".format(y_data.shape))
'''
This is the number of observations (aka. data-points or samples) in
the data-set:
'''
num_data = len(x_data)
'''
This is the fraction of the data-set that will be used for the
training-set:
'''
train_split = 0.9
'''
This is the number of observations in the training-set:
'''
self.num_train = int(train_split * num_data)
'''
This is the number of observations in the test-set:
'''
num_test = num_data - self.num_train
print('> Number of Samples: {}'.format(num_data))
print("> Number of Training Samples: {}".format(self.num_train))
print("> Number of Test Samples: {}".format(num_test))
# Create test and training data
x_train = x_data[0:self.num_train]
x_test = x_data[self.num_train:]
self.y_train = y_data[0:self.num_train]
self.y_test = y_data[self.num_train:]
self.num_x_signals = x_data.shape[1]
self.num_y_signals = y_data.shape[1]
print("> Training Minimum Value:", np.min(x_train))
print("> Training Maximum Value:", np.max(x_train))
'''
steps_per_epoch is the number of batch iterations before a training
epoch is considered finished.
'''
self.steps_per_epoch = int(self.num_train / batch_size) + 1
print("> Epochs:", epochs)
print("> Batch Size:", batch_size)
print("> Steps:", self.steps_per_epoch)
'''
Calculate the estimated memory footprint.
'''
print("> Data size: {:.2f} Bytes".format(x_data.nbytes))
'''
if memory_footprint > 7:
print('\n\n{}\n\n'.format(
'> Estimated GPU memory usage too large. Use new parameters '
'to reduce the footprint.'))
sys.exit(0)
'''
'''
The neural network works best on values roughly between -1 and 1, so we
need to scale the data before it is being input to the neural network.
We can use scikit-learn for this.
We first create a scaler-object for the input-signals.
Then we detect the range of values from the training-data and scale
the training-data.
'''
x_scaler = MinMaxScaler()
self.x_train_scaled = x_scaler.fit_transform(x_train)
print('> Scaled Training Minimum Value: {}'.format(
np.min(self.x_train_scaled)))
print('> Scaled Training Maximum Value: {}'.format(
np.max(self.x_train_scaled)))
self.x_test_scaled = x_scaler.transform(x_test)
'''
The target-data comes from the same data-set as the input-signals,
because it is the weather-data for one of the cities that is merely
time-shifted. But the target-data could be from a different source with
different value-ranges, so we create a separate scaler-object for the
target-data.
'''
self.y_scaler = MinMaxScaler()
self.y_train_scaled = self.y_scaler.fit_transform(self.y_train)
y_test_scaled = self.y_scaler.transform(self.y_test)
# Data Generator
'''
The data-set has now been prepared as 2-dimensional numpy arrays. The
training-data has almost 300k observations, consisting of 20
input-signals and 3 output-signals.
These are the array-shapes of the input and output data:
'''
print('> Scaled Training Data Shape: {}'.format(
self.x_train_scaled.shape))
print('> Scaled Training Targets Shape: {}'.format(
self.y_train_scaled.shape))
# We then create the batch-generator.
generator = self.batch_generator(batch_size, sequence_length)
# Validation Set
'''
The neural network trains quickly so we can easily run many training
epochs. But then there is a risk of overfitting the model to the
training-set so it does not generalize well to unseen data. We will
therefore monitor the model's performance on the test-set after each
epoch and only save the model's weights if the performance is improved
on the test-set.
The batch-generator randomly selects a batch of short sequences from
the training-data and uses that during training. But for the
validation-data we will instead run through the entire sequence from
the test-set and measure the prediction accuracy on that entire
sequence.
'''
validation_data = (np.expand_dims(self.x_test_scaled, axis=0),
np.expand_dims(y_test_scaled, axis=0))
# Create the Recurrent Neural Network
self.model = Sequential()
'''
We can now add a Gated Recurrent Unit (GRU) to the network. This will
have 512 outputs for each time-step in the sequence.
Note that because this is the first layer in the model, Keras needs to
know the shape of its input, which is a batch of sequences of arbitrary
length (indicated by None), where each observation has a number of
input-signals (num_x_signals).
'''
self.model.add(GRU(
units=512,
return_sequences=True,
input_shape=(None, self.num_x_signals,)))
'''
The GRU outputs a batch of sequences of 512 values. We want to predict
3 output-signals, so we add a fully-connected (or dense) layer which
maps 512 values down to only 3 values.
The output-signals in the data-set have been limited to be between 0
and 1 using a scaler-object. So we also limit the output of the neural
network using the Sigmoid activation function, which squashes the
output to be between 0 and 1.'''
self.model.add(Dense(self.num_y_signals, activation='sigmoid'))
'''
A problem with using the Sigmoid activation function, is that we can
now only output values in the same range as the training-data.
For example, if the training-data only has temperatures between -20
and +30 degrees, then the scaler-object will map -20 to 0 and +30 to 1.
So if we limit the output of the neural network to be between 0 and 1
using the Sigmoid function, this can only be mapped back to temperature
values between -20 and +30.
We can use a linear activation function on the output instead. This
allows for the output to take on arbitrary values. It might work with
the standard initialization for a simple network architecture, but for
more complicated network architectures e.g. with more layers, it might
be necessary to initialize the weights with smaller values to avoid
NaN values during training. You may need to experiment with this to
get it working.
'''
if False:
# Maybe use lower init-ranges.
# init = RandomUniform(minval=-0.05, maxval=0.05)
init = RandomUniform(minval=-0.05, maxval=0.05)
self.model.add(Dense(
self.num_y_signals,
activation='linear',
kernel_initializer=init))
# Compile Model
'''
This is the optimizer and the beginning learning-rate that we will use.
We then compile the Keras model so it is ready for training.
'''
optimizer = RMSprop(lr=1e-3)
self.model.compile(loss=self.loss_mse_warmup, optimizer=optimizer)
'''
This is a very small model with only two layers. The output shape of
(None, None, 3) means that the model will output a batch with an
arbitrary number of sequences, each of which has an arbitrary number of
observations, and each observation has 3 signals. This corresponds to
the 3 target signals we want to predict.
'''
print('> Model Summary:\n')
print(self.model.summary())
# Callback Functions
'''
During training we want to save checkpoints and log the progress to
TensorBoard so we create the appropriate callbacks for Keras.
This is the callback for writing checkpoints during training.
'''
path_checkpoint = '/tmp/23_checkpoint.keras'
callback_checkpoint = ModelCheckpoint(filepath=path_checkpoint,
monitor='val_loss',
verbose=1,
save_weights_only=True,
save_best_only=True)
'''
This is the callback for stopping the optimization when performance
worsens on the validation-set.
'''
callback_early_stopping = EarlyStopping(monitor='val_loss',
patience=5, verbose=1)
'''
This is the callback for writing the TensorBoard log during training.
'''
callback_tensorboard = TensorBoard(log_dir='/tmp/23_logs/',
histogram_freq=0,
write_graph=False)
'''
This callback reduces the learning-rate for the optimizer if the
validation-loss has not improved since the last epoch
(as indicated by patience=0). The learning-rate will be reduced by
multiplying it with the given factor. We set a start learning-rate of
1e-3 above, so multiplying it by 0.1 gives a learning-rate of 1e-4.
We don't want the learning-rate to go any lower than this.
'''
callback_reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
min_lr=1e-4,
patience=0,
verbose=1)
callbacks = [callback_early_stopping,
callback_checkpoint,
callback_tensorboard,
callback_reduce_lr]
# Train the Recurrent Neural Network
'''We can now train the neural network.
Note that a single "epoch" does not correspond to a single processing
of the training-set, because of how the batch-generator randomly
selects sub-sequences from the training-set. Instead we have selected
steps_per_epoch so that one "epoch" is processed in a few minutes.
With these settings, each "epoch" took about 2.5 minutes to process on
a GTX 1070. After 14 "epochs" the optimization was stopped because the
validation-loss had not decreased for 5 "epochs". This optimization
took about 35 minutes to finish.
Also note that the loss sometimes becomes NaN (not-a-number). This is
often resolved by restarting and running the Notebook again. But it may
also be caused by your neural network architecture, learning-rate,
batch-size, sequence-length, etc. in which case you may have to modify
those settings.
'''
print('\n> Starting data training\n')
try:
self.model.fit_generator(
generator=generator,
epochs=self.epochs,
steps_per_epoch=self.steps_per_epoch,
validation_data=validation_data,
callbacks=callbacks)
except Exception as error:
print('\n>{}\n'.format(error))
traceback.print_exc()
sys.exit(0)
# Load Checkpoint
'''
Because we use early-stopping when training the model, it is possible
that the model's performance has worsened on the test-set for several
epochs before training was stopped. We therefore reload the last saved
checkpoint, which should have the best performance on the test-set.
'''
print('> Loading model weights')
try:
self.model.load_weights(path_checkpoint)
except Exception as error:
print('\n> Error trying to load checkpoint.\n\n{}'.format(error))
traceback.print_exc()
sys.exit(0)
# Performance on Test-Set
'''
We can now evaluate the model's performance on the test-set. This
function expects a batch of data, but we will just use one long
time-series for the test-set, so we just expand the
array-dimensionality to create a batch with that one sequence.
'''
result = self.model.evaluate(
x=np.expand_dims(self.x_test_scaled, axis=0),
y=np.expand_dims(y_test_scaled, axis=0))
print('> Loss (test-set): {}'.format(result))
# If you have several metrics you can use this instead.
if False:
for res, metric in zip(result, self.model.metrics_names):
print('{0}: {1:.3e}'.format(metric, res))
def batch_generator(self, batch_size, sequence_length):
"""Create generator function to create random batches of training-data.
Args:
batch_size: Size of batch
sequence_length: Length of sequence
Returns:
(x_batch, y_batch)
"""
# Infinite loop.
while True:
# Allocate a new array for the batch of input-signals.
x_shape = (batch_size, sequence_length, self.num_x_signals)
x_batch = np.zeros(shape=x_shape, dtype=np.float16)
# Allocate a new array for the batch of output-signals.
y_shape = (batch_size, sequence_length, self.num_y_signals)
y_batch = np.zeros(shape=y_shape, dtype=np.float16)
# Fill the batch with random sequences of data.
for i in range(batch_size):
# Get a random start-index.
# This points somewhere into the training-data.
idx = np.random.randint(self.num_train - sequence_length)
# Copy the sequences of data starting at this index.
x_batch[i] = self.x_train_scaled[idx:idx+sequence_length]
y_batch[i] = self.y_train_scaled[idx:idx+sequence_length]
yield (x_batch, y_batch)
def loss_mse_warmup(self, y_true, y_pred):
"""Calculate the Mean Squared Errror.
Calculate the Mean Squared Error between y_true and y_pred,
but ignore the beginning "warmup" part of the sequences.
We will use Mean Squared Error (MSE) as the loss-function that will be
minimized. This measures how closely the model's output matches the
true output signals.
However, at the beginning of a sequence, the model has only seen
input-signals for a few time-steps, so its generated output may be very
inaccurate. Using the loss-value for the early time-steps may cause the
model to distort its later output. We therefore give the model a
"warmup-period" of 50 time-steps where we don't use its accuracy in the
loss-function, in hope of improving the accuracy for later time-steps
Args:
y_true: Desired output.
y_pred: Model's output.
Returns:
loss_mean: Mean Squared Error
"""
warmup_steps = self.warmup_steps
# The shape of both input tensors are:
# [batch_size, sequence_length, num_y_signals].
# Ignore the "warmup" parts of the sequences
# by taking slices of the tensors.
y_true_slice = y_true[:, warmup_steps:, :]
y_pred_slice = y_pred[:, warmup_steps:, :]
# These sliced tensors both have this shape:
# [batch_size, sequence_length - warmup_steps, num_y_signals]
# Calculate the MSE loss for each value in these tensors.
# This outputs a 3-rank tensor of the same shape.
loss = tf.losses.mean_squared_error(labels=y_true_slice,
predictions=y_pred_slice)
# Keras may reduce this across the first axis (the batch)
# but the semantics are unclear, so to be sure we use
# the loss across the entire tensor, we reduce it to a
# single scalar with the mean function.
loss_mean = tf.reduce_mean(loss)
return loss_mean
def plot_comparison(self, start_idx, length=100, train=True):
"""Plot the predicted and true output-signals.
Args:
start_idx: Start-index for the time-series.
length: Sequence-length to process and plot.
train: Boolean whether to use training- or test-set.
Returns:
None
"""
if train:
# Use training-data.
x_values = self.x_train_scaled
y_true = self.y_train
shim = 'Train'
else:
# Use test-data.
x_values = self.x_test_scaled
y_true = self.y_test
shim = 'Test'
# End-index for the sequences.
end_idx = start_idx + length
# Select the sequences from the given start-index and
# of the given length.
x_values = x_values[start_idx:end_idx]
y_true = y_true[start_idx:end_idx]
# Input-signals for the model.
x_values = np.expand_dims(x_values, axis=0)
# Use the model to predict the output-signals.
y_pred = self.model.predict(x_values)
# The output of the model is between 0 and 1.
# Do an inverse map to get it back to the scale
# of the original data-set.
y_pred_rescaled = self.y_scaler.inverse_transform(y_pred[0])
# For each output-signal.
for signal in range(len(self.target_names)):
# Create a filename
filename = (
'/tmp/batch_{}_epochs_{}_training_{}_{}_{}_{}.png').format(
self.batch_size, self.epochs, self.num_train, signal,
int(time.time()), shim)
# Get the output-signal predicted by the model.
signal_pred = y_pred_rescaled[:, signal]
# Get the true output-signal from the data-set.
signal_true = y_true[:, signal]
# Make the plotting-canvas bigger.
plt.figure(figsize=(15, 5))
# Plot and compare the two signals.
plt.plot(signal_true, label='true')
plt.plot(signal_pred, label='pred')
# Plot grey box for warmup-period.
_ = plt.axvspan(
0, self.warmup_steps, facecolor='black', alpha=0.15)
# Plot labels etc.
plt.ylabel(self.target_names[signal])
plt.legend()
# Show and save the image
if self.display is True:
plt.savefig(filename, bbox_inches='tight')
plt.show()
else:
plt.savefig(filename, bbox_inches='tight')
print('> Saving file: {}'.format(filename))
output_dim=embedding_size,
input_length=max_tokens,
name='layer_embedding'))
model.add(GRU(units=16, return_sequences=True))
model.add(GRU(units=8, return_sequences=True))
model.add(GRU(units=4))
# the first parameter corresponds to number of target
model.add(
Dense(4, activation='softmax')
) # changed ativation to softmax and first parameter from one to 2 vhl
optimizer = Adam(lr=1e-3)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy']) # changed loss to categorical_crossentropy
#model.fit(x_train_pad, y_train, validation_split=0.05, epochs=3, batch_size=32)
model.fit(x_train_pad,
one_hot_labels,
validation_split=0.05,
epochs=3,
batch_size=32) # replaced y_train with one_hot_labels
result = model.evaluate(
x_test_pad, np.array(two_hot_labels)) # replaced y_test with two_hot_abels
print("Accuracy: {0:.2%}".format(result[1]))
print("-----------------------------------------------------")
y_pred = model.predict(x=x_test_pad[0:1000])
class RNNGRU(object):
"""Process data for ingestion."""
def __init__(
self, data, sequence_length=20, warmup_steps=50, dropout=0,
layers=1, patience=10, units=512, display=False):
"""Instantiate the class.
Args:
data: Tuple of (x_data, y_data, target_names)
batch_size: Size of batch
sequence_length: Length of vectors for for each target
warmup_steps:
Returns:
None
"""
# Initialize key variables
self._warmup_steps = warmup_steps
self._data = data
self.display = display
path_checkpoint = '/tmp/checkpoint.keras'
_layers = int(abs(layers))
# Delete any stale checkpoint file
if os.path.exists(path_checkpoint) is True:
os.remove(path_checkpoint)
###################################
# TensorFlow wizardry
config = tf.ConfigProto()
# Don't pre-allocate memory; allocate as-needed
config.gpu_options.allow_growth = True
# Only allow a total of half the GPU memory to be allocated
config.gpu_options.per_process_gpu_memory_fraction = 0.95
# Crash with DeadlineExceeded instead of hanging forever when your
# queues get full/empty
config.operation_timeout_in_ms = 60000
# Create a session with the above options specified.
backend.tensorflow_backend.set_session(tf.Session(config=config))
###################################
# Get data
self._y_current = self._data.close()
# Create training arrays
x_train = self._data.vectors_train()
self._y_train = self._data.classes_train()
# Create test arrays for VALIDATION and EVALUATION
xv_test = self._data.vectors_test()
self._yv_test = self._data.classes_test()
(self.training_rows, self._training_vector_count) = x_train.shape
(self.test_rows, _) = xv_test.shape
(_, self._training_class_count) = self._y_train.shape
# Print stuff
print('\n> Numpy Data Type: {}'.format(type(x_train)))
print("> Numpy Data Shape: {}".format(x_train.shape))
print("> Numpy Data Row[0]: {}".format(x_train[0]))
print("> Numpy Data Row[Last]: {}".format(x_train[-1]))
print('> Numpy Targets Type: {}'.format(type(self._y_train)))
print("> Numpy Targets Shape: {}".format(self._y_train.shape))
print('> Number of Samples: {}'.format(self._y_current.shape[0]))
print('> Number of Training Samples: {}'.format(x_train.shape[0]))
print('> Number of Training Classes: {}'.format(
self._training_class_count))
print('> Number of Test Samples: {}'.format(self.test_rows))
print("> Training Minimum Value:", np.min(x_train))
print("> Training Maximum Value:", np.max(x_train))
print('> Number X signals: {}'.format(self._training_vector_count))
print('> Number Y signals: {}'.format(self._training_class_count))
# Print epoch related data
print('> Epochs:', self._data.epochs())
print('> Batch Size:', self._data.batch_size())
print('> Steps:', self._data.epoch_steps())
# Display estimated memory footprint of training data.
print("> Data size: {:.2f} Bytes".format(x_train.nbytes))
'''
The neural network works best on values roughly between -1 and 1, so we
need to scale the data before it is being input to the neural network.
We can use scikit-learn for this.
We first create a scaler-object for the input-signals.
Then we detect the range of values from the training-data and scale
the training-data.
'''
self._x_scaler = MinMaxScaler()
self._x_train_scaled = self._x_scaler.fit_transform(x_train)
print('> Scaled Training Minimum Value: {}'.format(
np.min(self._x_train_scaled)))
print('> Scaled Training Maximum Value: {}'.format(
np.max(self._x_train_scaled)))
self._xv_test_scaled = self._x_scaler.transform(xv_test)
'''
The target-data comes from the same data-set as the input-signals,
because it is the weather-data for one of the cities that is merely
time-shifted. But the target-data could be from a different source with
different value-ranges, so we create a separate scaler-object for the
target-data.
'''
self._y_scaler = MinMaxScaler()
self._y_train_scaled = self._y_scaler.fit_transform(self._y_train)
yv_test_scaled = self._y_scaler.transform(self._yv_test)
# Data Generator
'''
The data-set has now been prepared as 2-dimensional numpy arrays. The
training-data has almost 300k observations, consisting of 20
input-signals and 3 output-signals.
These are the array-shapes of the input and output data:
'''
print('> Scaled Training Data Shape: {}'.format(
self._x_train_scaled.shape))
print('> Scaled Training Targets Shape: {}'.format(
self._y_train_scaled.shape))
# We then create the batch-generator.
generator = self._batch_generator(
self._data.batch_size(), sequence_length)
# Validation Set
'''
The neural network trains quickly so we can easily run many training
epochs. But then there is a risk of overfitting the model to the
training-set so it does not generalize well to unseen data. We will
therefore monitor the model's performance on the test-set after each
epoch and only save the model's weights if the performance is improved
on the test-set.
The batch-generator randomly selects a batch of short sequences from
the training-data and uses that during training. But for the
validation-data we will instead run through the entire sequence from
the test-set and measure the prediction accuracy on that entire
sequence.
'''
validation_data = (np.expand_dims(self._xv_test_scaled, axis=0),
np.expand_dims(yv_test_scaled, axis=0))
# Create the Recurrent Neural Network
self._model = Sequential()
'''
We can now add a Gated Recurrent Unit (GRU) to the network. This will
have 512 outputs for each time-step in the sequence.
Note that because this is the first layer in the model, Keras needs to
know the shape of its input, which is a batch of sequences of arbitrary
length (indicated by None), where each observation has a number of
input-signals (num_x_signals).
'''
self._model.add(GRU(
units=units,
return_sequences=True,
recurrent_dropout=dropout,
input_shape=(None, self._training_vector_count,)))
for _ in range(0, _layers):
self._model.add(GRU(
units=units,
recurrent_dropout=dropout,
return_sequences=True))
'''
The GRU outputs a batch of sequences of 512 values. We want to predict
3 output-signals, so we add a fully-connected (or dense) layer which
maps 512 values down to only 3 values.
The output-signals in the data-set have been limited to be between 0
and 1 using a scaler-object. So we also limit the output of the neural
network using the Sigmoid activation function, which squashes the
output to be between 0 and 1.'''
self._model.add(
Dense(self._training_class_count, activation='sigmoid'))
'''
A problem with using the Sigmoid activation function, is that we can
now only output values in the same range as the training-data.
For example, if the training-data only has temperatures between -20
and +30 degrees, then the scaler-object will map -20 to 0 and +30 to 1.
So if we limit the output of the neural network to be between 0 and 1
using the Sigmoid function, this can only be mapped back to temperature
values between -20 and +30.
We can use a linear activation function on the output instead. This
allows for the output to take on arbitrary values. It might work with
the standard initialization for a simple network architecture, but for
more complicated network architectures e.g. with more layers, it might
be necessary to initialize the weights with smaller values to avoid
NaN values during training. You may need to experiment with this to
get it working.
'''
if False:
# Maybe use lower init-ranges.
# init = RandomUniform(minval=-0.05, maxval=0.05)
init = RandomUniform(minval=-0.05, maxval=0.05)
self._model.add(Dense(
self._training_class_count,
activation='linear',
kernel_initializer=init))
# Compile Model
'''
This is the optimizer and the beginning learning-rate that we will use.
We then compile the Keras model so it is ready for training.
'''
optimizer = RMSprop(lr=1e-3)
self._model.compile(
loss=self._loss_mse_warmup,
optimizer=optimizer,
metrics=['accuracy'])
'''
This is a very small model with only two layers. The output shape of
(None, None, 3) means that the model will output a batch with an
arbitrary number of sequences, each of which has an arbitrary number of
observations, and each observation has 3 signals. This corresponds to
the 3 target signals we want to predict.
'''
print('> Model Summary:\n')
print(self._model.summary())
# Callback Functions
'''
During training we want to save checkpoints and log the progress to
TensorBoard so we create the appropriate callbacks for Keras.
This is the callback for writing checkpoints during training.
'''
callback_checkpoint = ModelCheckpoint(filepath=path_checkpoint,
monitor='val_loss',
verbose=1,
save_weights_only=True,
save_best_only=True)
'''
This is the callback for stopping the optimization when performance
worsens on the validation-set.
'''
callback_early_stopping = EarlyStopping(monitor='val_loss',
patience=patience, verbose=1)
'''
This is the callback for writing the TensorBoard log during training.
'''
callback_tensorboard = TensorBoard(log_dir='/tmp/23_logs/',
histogram_freq=0,
write_graph=False)
'''
This callback reduces the learning-rate for the optimizer if the
validation-loss has not improved since the last epoch
(as indicated by patience=0). The learning-rate will be reduced by
multiplying it with the given factor. We set a start learning-rate of
1e-3 above, so multiplying it by 0.1 gives a learning-rate of 1e-4.
We don't want the learning-rate to go any lower than this.
'''
callback_reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
min_lr=1e-4,
patience=0,
verbose=1)
callbacks = [callback_early_stopping,
callback_checkpoint,
callback_tensorboard,
callback_reduce_lr]
# Train the Recurrent Neural Network
'''We can now train the neural network.
Note that a single "epoch" does not correspond to a single processing
of the training-set, because of how the batch-generator randomly
selects sub-sequences from the training-set. Instead we have selected
steps_per_epoch so that one "epoch" is processed in a few minutes.
With these settings, each "epoch" took about 2.5 minutes to process on
a GTX 1070. After 14 "epochs" the optimization was stopped because the
validation-loss had not decreased for 5 "epochs". This optimization
took about 35 minutes to finish.
Also note that the loss sometimes becomes NaN (not-a-number). This is
often resolved by restarting and running the Notebook again. But it may
also be caused by your neural network architecture, learning-rate,
batch-size, sequence-length, etc. in which case you may have to modify
those settings.
'''
print('\n> Starting data training\n')
self._history = self._model.fit_generator(
generator=generator,
epochs=self._data.epochs(),
steps_per_epoch=self._data.epoch_steps(),
validation_data=validation_data,
callbacks=callbacks)
# Load Checkpoint
'''
Because we use early-stopping when training the model, it is possible
that the model's performance has worsened on the test-set for several
epochs before training was stopped. We therefore reload the last saved
checkpoint, which should have the best performance on the test-set.
'''
print('> Loading model weights')
if os.path.exists(path_checkpoint):
self._model.load_weights(path_checkpoint)
# Performance on Test-Set
'''
We can now evaluate the model's performance on the test-set. This
function expects a batch of data, but we will just use one long
time-series for the test-set, so we just expand the
array-dimensionality to create a batch with that one sequence.
'''
result = self._model.evaluate(
x=np.expand_dims(self._xv_test_scaled, axis=0),
y=np.expand_dims(yv_test_scaled, axis=0))
print('> Loss (test-set): {}'.format(result))
# If you have several metrics you can use this instead.
if False:
for res, metric in zip(result, self._model.metrics_names):
print('{0}: {1:.3e}'.format(metric, res))
def _batch_generator(self, batch_size, sequence_length):
"""Create generator function to create random batches of training-data.
Args:
batch_size: Size of batch
sequence_length: Length of sequence
Returns:
(x_batch, y_batch)
"""
# Infinite loop.
while True:
# Allocate a new array for the batch of input-signals.
x_shape = (
batch_size, sequence_length, self._training_vector_count)
x_batch = np.zeros(shape=x_shape, dtype=np.float16)
# Allocate a new array for the batch of output-signals.
y_shape = (batch_size, sequence_length, self._training_class_count)
y_batch = np.zeros(shape=y_shape, dtype=np.float16)
# Fill the batch with random sequences of data.
for i in range(batch_size):
# Get a random start-index.
# This points somewhere into the training-data.
idx = np.random.randint(
self.training_rows - sequence_length)
# Copy the sequences of data starting at this index.
x_batch[i] = self._x_train_scaled[idx:idx+sequence_length]
y_batch[i] = self._y_train_scaled[idx:idx+sequence_length]
yield (x_batch, y_batch)
def _loss_mse_warmup(self, y_true, y_pred):
"""Calculate the Mean Squared Errror.
Calculate the Mean Squared Error between y_true and y_pred,
but ignore the beginning "warmup" part of the sequences.
We will use Mean Squared Error (MSE) as the loss-function that will be
minimized. This measures how closely the model's output matches the
true output signals.
However, at the beginning of a sequence, the model has only seen
input-signals for a few time-steps, so its generated output may be very
inaccurate. Using the loss-value for the early time-steps may cause the
model to distort its later output. We therefore give the model a
"warmup-period" of 50 time-steps where we don't use its accuracy in the
loss-function, in hope of improving the accuracy for later time-steps
Args:
y_true: Desired output.
y_pred: Model's output.
Returns:
loss_mean: Mean Squared Error
"""
warmup_steps = self._warmup_steps
# The shape of both input tensors are:
# [batch_size, sequence_length, num_y_signals].
# Ignore the "warmup" parts of the sequences
# by taking slices of the tensors.
y_true_slice = y_true[:, warmup_steps:, :]
y_pred_slice = y_pred[:, warmup_steps:, :]
# These sliced tensors both have this shape:
# [batch_size, sequence_length - warmup_steps, num_y_signals]
# Calculate the MSE loss for each value in these tensors.
# This outputs a 3-rank tensor of the same shape.
loss = tf.losses.mean_squared_error(labels=y_true_slice,
predictions=y_pred_slice)
# Keras may reduce this across the first axis (the batch)
# but the semantics are unclear, so to be sure we use
# the loss across the entire tensor, we reduce it to a
# single scalar with the mean function.
loss_mean = tf.reduce_mean(loss)
return loss_mean
def plot_train(self, start_idx, length=100):
"""Plot the predicted and true output-signals.
Args:
start_idx: Start-index for the time-series.
length: Sequence-length to process and plot.
Returns:
None
"""
# Plot
self._plot_comparison(start_idx, length=length, train=True)
def plot_test(self, start_idx, length=100):
"""Plot the predicted and true output-signals.
Args:
start_idx: Start-index for the time-series.
length: Sequence-length to process and plot.
Returns:
None
"""
# Plot
self._plot_comparison(start_idx, length=length, train=False)
def _plot_comparison(self, start_idx, length=100, train=True):
"""Plot the predicted and true output-signals.
Args:
start_idx: Start-index for the time-series.
length: Sequence-length to process and plot.
train: Boolean whether to use training- or test-set.
Returns:
None
"""
# Initialize key variables
datetimes = {}
num_train = self.training_rows
# End-index for the sequences.
end_idx = start_idx + length
# Variables for date formatting
days = mdates.DayLocator() # Every day
months = mdates.MonthLocator() # Every month
months_format = mdates.DateFormatter('%b %Y')
days_format = mdates.DateFormatter('%d')
# Assign other variables dependent on the type of data we are plotting
if train is True:
# Use training-data.
x_values = self._x_train_scaled[start_idx:end_idx]
y_true = self._y_train[start_idx:end_idx]
shim = 'Train'
# Datetimes to use for training
datetimes[shim] = self._data.datetime()[
:num_train][start_idx:end_idx]
else:
# Scale the data
x_test_scaled = self._x_scaler.transform(
self._data.vectors_test_all())
# Use test-data.
x_values = x_test_scaled[start_idx:end_idx]
y_true = self._yv_test[start_idx:end_idx]
shim = 'Test'
# Datetimes to use for testing
datetimes[shim] = self._data.datetime()[
num_train:][start_idx:end_idx]
# Input-signals for the model.
x_values = np.expand_dims(x_values, axis=0)
# Use the model to predict the output-signals.
y_pred = self._model.predict(x_values)
# The output of the model is between 0 and 1.
# Do an inverse map to get it back to the scale
# of the original data-set.
y_pred_rescaled = self._y_scaler.inverse_transform(y_pred[0])
# For each output-signal.
for signal in range(len(self._data.labels())):
# Assign other variables dependent on the type of data plot
if train is True:
# Only get current values that are a part of the training data
current = self._y_current[:num_train][start_idx:end_idx]
# The number of datetimes for the 'actual' plot must match
# that of current values
datetimes['actual'] = self._data.datetime()[
:num_train][start_idx:end_idx]
else:
# Only get current values that are a part of the test data.
current = self._y_current[
num_train:][start_idx:]
# The number of datetimes for the 'actual' plot must match
# that of current values
datetimes['actual'] = self._data.datetime()[
num_train:][start_idx:]
# Create a filename
filename = (
'/tmp/batch_{}_epochs_{}_training_{}_{}_{}_{}.png').format(
self._data.batch_size(),
self._data.epochs(),
num_train,
signal,
int(time.time()),
shim)
# Get the output-signal predicted by the model.
signal_pred = y_pred_rescaled[:, signal]
# Get the true output-signal from the data-set.
signal_true = y_true[:, signal]
# Create a new chart
(fig, axis) = plt.subplots(figsize=(15, 5))
# Plot and compare the two signals.
axis.plot(
datetimes[shim][:len(signal_true)],
signal_true,
label='Current +{}'.format(self._data.labels()[signal]))
axis.plot(
datetimes[shim][:len(signal_pred)],
signal_pred,
label='Prediction')
axis.plot(datetimes['actual'], current, label='Current')
# Set plot labels and titles
axis.set_title('{1}ing Forecast ({0} Future Intervals)'.format(
self._data.labels()[signal], shim))
axis.set_ylabel('Values')
axis.legend(
bbox_to_anchor=(1.04, 0.5),
loc='center left', borderaxespad=0)
# Add gridlines and ticks
ax = plt.gca()
ax.grid(True)
# Add major gridlines
ax.xaxis.grid(which='major', color='black', alpha=0.2)
ax.yaxis.grid(which='major', color='black', alpha=0.2)
# Add minor ticks (They must be turned on first)
ax.minorticks_on()
ax.xaxis.grid(which='minor', color='black', alpha=0.1)
ax.yaxis.grid(which='minor', color='black', alpha=0.1)
# Format the tick labels
ax.xaxis.set_major_locator(months)
ax.xaxis.set_major_formatter(months_format)
ax.xaxis.set_minor_locator(days)
# Remove tick marks
ax.tick_params(axis='both', which='both', length=0)
# Print day numbers on xaxis for Test data only
if train is False:
ax.xaxis.set_minor_formatter(days_format)
plt.setp(ax.xaxis.get_minorticklabels(), rotation=90)
# Rotates and right aligns the x labels, and moves the bottom of
# the axes up to make room for them
fig.autofmt_xdate()
# Plot grey box for warmup-period if we are working with training
# data and the start is within the warmup-period
if (0 < start_idx < self._warmup_steps):
if train is True:
plt.axvspan(
datetimes[shim][start_idx],
datetimes[shim][self._warmup_steps],
facecolor='black', alpha=0.15)
# Show and save the image
if self.display is True:
fig.savefig(filename, bbox_inches='tight')
plt.show()
else:
fig.savefig(filename, bbox_inches='tight')
print('> Saving file: {}'.format(filename))
# Close figure
plt.close(fig=fig)
def plot_accuracy(self):
"""Plot the predicted and true output-signals.
Args:
None
Returns:
None
"""
# Summarize history for accuracy
plt.figure(figsize=(15, 5))
plt.plot(self._history.history['acc'])
plt.plot(self._history.history['val_acc'])
plt.title('Model Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
# Summarize history for loss
plt.figure(figsize=(15, 5))
plt.plot(self._history.history['loss'])
plt.plot(self._history.history['val_loss'])
plt.title('Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper left')
plt.show()
print(CNN_dataframe.columns) num_col = len(CNN_dataframe.columns) X = np.asarray(CNN_dataframe.drop(labels = 'c',axis=1)) y = np.asarray(CNN_dataframe['c']) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3) neural_network = Sequential() neural_network.add(Dense(activation = 'relu', input_dim = num_col - 1, units=6)) neural_network.add(Dense(activation = 'relu', units=6)) neural_network.add(Dense(activation = 'relu', units=6)) neural_network.add(Dense(activation = 'relu', units=6)) neural_network.add(Dense(activation = 'relu', units=6)) neural_network.add(Dense(activation = 'relu', units=6)) neural_network.add(Dense(activation = 'sigmoid', units=1)) neural_network.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy']) neural_network.fit(X_train, y_train, batch_size = 5, epochs = 5) y_pred = neural_network.predict(X_test) y_pred = (y_pred > 0.5) print(neural_network.get_config()) cm = confusion_matrix(y_test, y_pred) #Random Forest df3 = "/Users/Vartika_Bisht/Desktop/df_CNN.csv" RF_dataframe = pd.read_csv(df3) print(RF_dataframe) print(RF_dataframe.columns) num_col = len(RF_dataframe.columns) X = RF_dataframe.drop(labels=['c', 't2'], axis=1)
model2.add(Conv2D(64, (3, 3), activation='relu',
input_shape=(150, 150, 3)))
model2.add(MaxPool2D(pool_size=(2, 2)))
model2.add(tf.keras.layers.BatchNormalization())
model2.add(Conv2D(64, (3, 3), activation='relu'))
model2.add(MaxPool2D(pool_size=(2, 2)))
model2.add(tf.keras.layers.BatchNormalization())
model2.add(Conv2D(64, (3, 3), activation='relu'))
model2.add(MaxPool2D(pool_size=(2, 2)))
# model2.add(tf.keras.layers.BatchNormalization())
model2.add(Flatten())
model2.add(Dropout(0.5))
model2.add(Dense(64, activation='relu'))
model2.add(Dense(1, activation='sigmoid'))
model2.compile(optimizer='RMSprop',
loss='binary_crossentropy',
metrics=['accuracy'])
history = model2.fit_generator(train_generator,
steps_per_epoch=187,
epochs=30,
validation_data=validation_generator,
validation_steps=62)
test_loss, test_acc = model2.evaluate(reshaped_x_test, test_y)
print(test_acc)
acc = history.history['acc']
val_acc = history.history['val_acc']
plt.figure()
plt.plot(acc)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.20,
random_state=42)
model = Sequential()
model.add(Embedding(len(word_index) + 1, 100, weights=[embedding_matrix]))
model.add(
Bidirectional(LSTM(128, return_sequences=True, recurrent_dropout=0.3)))
model.add(Dropout(0.3))
model.add(
Bidirectional(LSTM(128, return_sequences=True, recurrent_dropout=0.1)))
model.add(Dropout(0.5))
model.add(TimeDistributed(Dense(2, activation="softmax")))
model.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
model.fit(X_train,
np.array(y_train),
batch_size=32,
epochs=5,
validation_split=0.1)
model_json = model.to_json()
with open("model.json", "w") as json_file:
json_file.write(model_json)
model.save_weights("model.h5")
pickle.dump(tokenizer, open("tokenizer.pickle", "wb"))
print("Saved model to disk")
test_output = model.predict(X_test)
def ffnthree(xtrain,
ytrain,
xtest,
ytest,
input_shape,
num_classes,
batch_size,
epochs,
callbacks,
ismodelsaved=False,
tl=False):
if ismodelsaved == False:
# model definition
ffn3 = Sequential()
ffn3.add(
Dense(100,
input_dim=input_shape,
kernel_initializer="lecun_uniform",
activation="relu"))
ffn3.add(BatchNormalization())
ffn3.add(Dense(50, activation="relu", kernel_initializer="uniform"))
ffn3.add(Dropout(0.5))
ffn3.add(Dense(10, activation="relu", kernel_initializer="uniform"))
ffn3.add(Dense(num_classes, activation='softmax'))
#
ffn3.compile(loss=tfkeras.losses.binary_crossentropy,
optimizer=tf.keras.optimizers.RMSprop(0.001, rho=0.9),
metrics=['accuracy'])
#
historyffn3 = ffn3.fit(xtrain,
ytrain,
batch_size=batch_size,
epochs=epochs,
verbose=0,
validation_data=(xtest, ytest),
callbacks=callbacks)
score = ffn3.evaluate(xtest, ytest, verbose=0)
p('Test loss:', score[0])
p('Test accuracy:', score[1])
#
# display learning curves
if True:
plt.figure()
plt.plot(historyffn3.history['loss'], label='train loss')
plt.plot(historyffn3.history['val_loss'], label='test loss')
plt.title('Learning Curves')
plt.xlabel('epochs')
plt.ylabel('loss')
plt.legend()
plt.show()
else:
if input_shape == 92:
ffn3 = tf.keras.models.load_model(flpath +
'saved_model_4x23/ffn3_4x23')
else:
if tl:
ffn3 = tf.keras.models.load_model(
flpath + 'saved_model_guideseq_8x23/ffn3_8x23')
else:
ffn3 = tf.keras.models.load_model(
flpath + 'saved_model_crispr_8x23/ffn3crispr_8x23')
p("FFN3: Done")
return ffn3
def neuralnet(no_model,dnafull,dna0,dna1,dna2,dna3,dna4,dna5,dna6):
"""
dna_temp[0] hid_layer_num INT 1~5
dna_temp[1] hid_layer_node INT 16~128
dna_temp[2] epoch INT 100~500
dna_temp[3] dropout FLOAT 0.00~0.20
dna_temp[4] maxlen INT 9~19
dna_temp[5] time_bias INT 1~9
dna_temp[6] layer INT 1~3
"""
"""
パラメーター設定
"""
#入力層次元
n_in = 20
#中間層次元、層数
n_hiddens = list()
for i in range(dna0):
n_hiddens.append(dna1)
n_centors = dna0
#出力層次元、層数
n_out = 5
#活性化関数
activation = 'relu'
#ドロップアウト率
p_keep = dna3
#計算回数
epochs = dna2
#EarlyStoppingするか
isEs= False
#EarlyStoppingをするまでの回数
es_patience= 60
#ミニバッチ処理のサイズ
batch_size = 1000
#最適化アルゴリズム
opt='rmsprop'
#学習率(本プログラムでは未使用でデフォルト値を使用）
# learning_rate=0.001
#Adamのパラメータ(最適化アルゴリズムがAdamの時のみ使用・本プログラムでは未使用)
# beta_1=0.9
# beta_2=0.999
#reccrentの参照数
maxlen= dna4
#Yを何秒ずらすか（=0だと過去maxlen秒参照、=maxlen/2だと前後maxlen/2秒参照、=maxlenだと未来maxlen秒参照になる)
time_bias= dna5
#RNNの種類(SimpleRNN,LSTM,GRU)
layer_int = dna6
#双方向性を使用するか
BiDir= False
#RNNの偶数層を逆向きにするか
back= False
#乱数の固定シード
# ranseed= 12345
# weight1 = 1
# weight2 = 1
#
print('No_%d' % no_model)
print(dna0,dna1,dna2,dna3,dna4,dna5,dna6)
#乱数固定
import os
os.environ['PYTHONHASHSEED']='0'
# np.random.seed(ranseed)
# rn.seed(ranseed)
#スレッド数等を１に固定(再現性に必要)
session_conf = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
from tensorflow.python.keras import backend as K
# tf.compat.v1.set_random_seed()
sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph(),config=session_conf)
K.set_session(sess)
#重み初期化
init=initializers.TruncatedNormal()
#ファイルの名前
name = 'linear_data_FIR8_comAngle&AveStd_coor_s1_ntd2_26'
# number = '-2'
#ファイル読み込み
csv_input = pd.read_csv(filepath_or_buffer= name+".csv", encoding="ms932", sep=",")
array = csv_input.values
#仮の入出力値を読み取り
X=array[:,1:n_in+1].astype(np.float32)
Y=array[:,n_in+1].astype(np.int)
#タイムスタンプを読み取り
TIME=array[:,0]
leng = len(Y)
data = []
target = []
i = 0
for i in range(maxlen, leng):
#入力データを参照秒数ごとにまとめる
data.append(X[i-maxlen+1:i+1,:])
#出力データをN秒ごとの作業に変換
target.append(Y[i-time_bias])
#入出力データのshapeの調整
X = np.array(data).reshape(len(data), maxlen, n_in)
Y = np.array(target)
#タイムスタンプを入出力データと同期
TIME=TIME[maxlen-time_bias:leng-time_bias]
#学習データとテストデータの分割
x_train, x_test, y_train0, y_test0,time_train,time_test = train_test_split(X, Y,TIME, train_size=0.85,shuffle=False)
#学習データをtrainとvalidationに分割
x_train, x_validation, y_train0, y_validation0 = train_test_split(x_train, y_train0,train_size=0.9,shuffle=False)
#yを1ofKデータに変換(train,val,test)
ntr=y_train0.size
y_train=np.zeros(n_out*ntr).reshape(ntr,n_out).astype(np.float32)
for i in range(ntr):
y_train[i,y_train0[i]]=1.0
nte=y_test0.size
y_test=np.zeros(n_out*nte).reshape(nte,n_out).astype(np.float32)
for i in range(nte):
y_test[i,y_test0[i]]=1.0
y_validation=np.eye(n_out)[(y_validation0.reshape(y_validation0.size))]
# nrow=y_test0.size
# モデル設定
model = Sequential()
for i in range(n_centors):
if(i==n_centors-1):
retSeq=False
else:
retSeq=True
if(i%2==1 and back):
gBack=True
else:
gBack=False
if(i==0):
in_dir=n_in
else:
in_dir=n_hiddens[i-1]
if (layer_int==1):
if(BiDir):
model.add(Bidirectional(SimpleRNN(n_hiddens[i],activation=activation,kernel_initializer=init,recurrent_initializer=init,dropout=p_keep,recurrent_dropout=p_keep, return_sequences=retSeq,go_backwards=gBack, input_shape=(maxlen, in_dir) )))
else:
# model.add(SimpleRNN(n_hiddens[i],activation=activation,kernel_initializer=init,recurrent_initializer=init, return_sequences=retSeq,go_backwards=gBack, input_shape=(maxlen, in_dir) ))
model.add(SimpleRNN(n_hiddens[i],activation=activation,kernel_initializer=init,recurrent_initializer=init,dropout=p_keep,recurrent_dropout=p_keep, return_sequences=retSeq,go_backwards=gBack, input_shape=(maxlen, in_dir) ))
elif(layer_int==2):
if(BiDir):
model.add(Bidirectional(LSTM(n_hiddens[0],activation=activation,kernel_initializer=init,recurrent_initializer=init,dropout=p_keep,recurrent_dropout=p_keep, return_sequences=retSeq,go_backwards=gBack, input_shape=(maxlen, in_dir) )))
else:
model.add(LSTM(n_hiddens[0],activation=activation,kernel_initializer=init,recurrent_initializer=init,dropout=p_keep,recurrent_dropout=p_keep, return_sequences=retSeq,go_backwards=gBack, input_shape=(maxlen, in_dir) ))
elif(layer_int==3):
if(BiDir):
model.add(Bidirectional(GRU(n_hiddens[0],activation=activation,kernel_initializer=init,recurrent_initializer=init,dropout=p_keep,recurrent_dropout=p_keep, return_sequences=retSeq,go_backwards=gBack, input_shape=(maxlen, in_dir) )))
else:
model.add(GRU(n_hiddens[0],activation=activation,kernel_initializer=init,recurrent_initializer=init,dropout=p_keep,recurrent_dropout=p_keep, return_sequences=retSeq,go_backwards=gBack, input_shape=(maxlen, in_dir) ))
model.add(Dense(n_out,kernel_initializer=init))
model.add(Activation('softmax'))
model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])
early_stopping =EarlyStopping(monitor='val_loss', patience=es_patience, verbose=1)
# now = datetime.now().strftime('%Y%m%d%H%M')
# flog = name+number+'.log1.csv'
#
# csv_logger=CSVLogger(flog)
if (isEs):
caBacks=[early_stopping]#,csv_logger]
else:
caBacks=[]#csv_logger]
#モデル学習
# start = time.time()
model.fit(x_train,y_train, epochs=epochs, batch_size=batch_size,validation_data=(x_validation,y_validation),callbacks=caBacks)
#hist = model.fit(x_train,y_train, epochs=epochs, batch_size=batch_size,callbacks=caBacks)
#,callbacks=[early_stopping]
# slapsed_time=time.time() - start
#
#
# val_acc = hist.history['val_acc']
# acc = hist.history['acc']
# val_loss = hist.history['val_loss']
# loss = hist.history['loss']
#
#now = datetime.now().strftime('%Y%m%d%H%M')
#
#plt.rc('font',family='serif')
#fig = plt.figure()
#plt.plot(range(len(loss)), loss, label='loss', color='r')
#plt.plot(range(len(val_loss)), val_loss, label='val_loss', color='b')
#plt.xlabel('epochs')
#plt.legend()
#plt.show()
#plt.savefig(name+number+'.loss.png')
#
##plt.rc('font',family='serif')
##fig = plt.figure()
##plt.plot(range(len(val_acc)), val_acc, label='acc', color='b')
##plt.xlabel('epochs')
##plt.show()
##plt.savefig(name+number+'.val_acc.png')
classes = model.predict_classes(x_test, batch_size=1)
#prob = model.predict_proba(x_test, batch_size=1)
#重みの出力
#L1 = model.get_weights()
#W1 = np.dot(L1[0],L1[1])+L1[2]
#W2 = np.dot(W1,L1[3])
#W3 = np.dot(W2,L1[4])
#W4 = W3+L1[5]
#weight1 = np.dot(W4,L1[6])+L1[7]
#weight = weight1.transpose()
#結果を出力
im = [[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,0,0,0]]
ip = [0,0,0,0]
it = [0,0,0,0]
f = [0,0,0,0]
ia = [0,0,0,0]
ib = [0,0,0,0]
j = 0
for i in range(4):
for j in range(y_test0.size):
if y_test0[j]==i+1:
it[i] += 1
if classes[j] == 1:
im[i][0] += 1
if classes[j] == 2:
im[i][1] += 1
if classes[j] == 3:
im[i][2] += 1
if classes[j] == 4:
im[i][3] += 1
else:
pass
for i in range(4):
for k in range(y_test0.size):
if classes[k]==i+1:
ip[i]+=1
else:
pass
#再現率を導出
for i in range(4):
if it[i]==0:
ia[i] = 0
else:
ia[i] = im[i][i]/it[i]
#適合率を導出
for i in range(4):
if ip[i]==0:
ib[i] = 0
else:
ib[i] = im[i][i]/ip[i]
#F値を導出
for i in range(4):
if ia[i]+ib[i]==0:
f[i] = 0
else:
f[i] = 2*ia[i]*ib[i]/(ia[i]+ib[i])
# it_sum = sum(it)
# ip_sum = sum(ip)
# ii = im[0][0]+im[1][1]+im[2][2]+im[3][3]#+i5
if_ave = sum(f)/4
model.save(name+'_'+str(no_model)+".h5")
# model.save("kanno_"+str(no_model)+".model")
# =============================================================================
backend.clear_session()
# =============================================================================
return if_ave
MaxPooling2D(),
Conv2D(512, 3, padding='same', activation='relu'),
Conv2D(512, 3, padding='same', activation='relu'),
Conv2D(512, 1, padding='same', activation='relu'),
MaxPooling2D(),
Conv2D(512, 3, padding='same', activation='relu'),
Conv2D(512, 3, padding='same', activation='relu'),
Conv2D(512, 1, padding='same', activation='relu'),
MaxPooling2D(),
Flatten(),
Dense(512, activation='relu'),
Dense(1, activation="sigmoid")
])
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=['accuracy'])
model.summary()
history = model.fit(x, y, batch_size=32, epochs=epochs, validation_split=0.1)
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs_range = range(epochs)
plt.figure(figsize=(8, 8))
plt.subplot(1, 2, 1)
plt.plot(epochs_range, acc, label='training accuracy')
#############
#sequentiel : permet la creation de la neseau de neurone
model = Sequential()
# units : nombre de noeud en sortie dans notre cas =1
#input_shape: on 2 noeuds en entrée
#activation: la fonction d'activation en sortie est sigmoid =(1/(1+exp(-x)))
#Dense : permer la liaison entre les noeuds
model.add(Dense(units=1, input_shape=(2, ), activation='sigmoid'))
adam = Adam(lr=0.1)
##################
#loss: erreur binary_crossentropy (seule noeud en sortie)
#metrics: permet de choisir les parametres qu'on veut verifier/afficher dans notre cas en prend accuracy
model.compile(adam, loss='binary_crossentropy', metrics=['accuracy'])
#model.fit :Permets l'entrainement de reseau
# x= : inputed_data / y= c'est le target / verbose=1 : afficher les détailles d'entrainement
#epochs=500: nombre de fois toute la donné (inputed_data) va étre traiter par le réseau
#batch_size = c'est la nombre d'exemplaire de données afin de mesurer l'erreur et faire la correction des parametres de réseau
#dans notre cas on fait 20 iteration pour atteindre un epoch
#shuffle=true: à la fin de chaque epoch on fait le mixage de données d'entrée
h = model.fit(x=X, y=y, verbose=1, epochs=500, batch_size=50, shuffle='true')
################
plt.plot(h.history['acc'])
plt.title('accuracy')
plt.xlabel('epoch')
plt.legend(['accuracy'])
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense, Flatten, GlobalAveragePooling2D
num_classes = 2
resnet_weights_path = '../input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5'
my_new_model = Sequential()
my_new_model.add(
ResNet50(include_top=False, pooling='avg', weights=resnet_weights_path))
my_new_model.add(Dense(num_classes, activation='softmax'))
# Say not to train first layer (ResNet) model. It is already trained
my_new_model.layers[0].trainable = False
my_new_model.compile(optimizer='sgd',
loss='categorical_crossentropy',
metrics=['accuracy'])
### ls ../input/urban-and-rural-photos/rural_and_urban_photos
from tensorflow.python.keras.applications.resnet50 import preprocess_input
from tensorflow.python.keras.preprocessing.image import ImageDataGenerator
image_size = 224
data_generator_with_aug = ImageDataGenerator(
preprocessing_function=preprocess_input,
horizontal_flip=True,
width_shift_range=0.2,
height_shift_range=0.2)
train_generator = data_generator_with_aug.flow_from_directory(
'../input/urban-and-rural-photos/rural_and_urban_photos/train',
class KerasCNN(object):
"""Support vector machine class."""
# Convolutional Layer 1.
filter_size1 = 5 # Convolution filters are 5 x 5 pixels.
num_filters1 = 16 # There are 16 of these filters.
# Convolutional Layer 2.
filter_size2 = 5 # Convolution filters are 5 x 5 pixels.
num_filters2 = 36 # There are 36 of these filters.
# Fully-connected layer.
fc_size = 128 # Number of neurons in fully-connected laye
# Get data from files
data = MNIST(data_dir='/tmp/data/MNIST/')
# The number of pixels in each dimension of an image.
img_size = data.img_size
# The images are stored in one-dimensional arrays of this length.
img_size_flat = data.img_size_flat
# Tuple with height and width of images used to reshape arrays.
img_shape = data.img_shape
# Tuple with height, width and depth used to reshape arrays.
# This is used for reshaping in Keras.
img_shape_full = data.img_shape_full
# Number of classes, one class for each of 10 digits.
num_classes = data.num_classes
# Number of colour channels for the images: 1 channel for gray-scale.
num_channels = data.num_channels
def __init__(self):
"""Instantiate the class.
Args:
train_batch_size: Training batch size
Returns:
None
"""
# Initialize variables
epochs = 2
"""
print('{0: <{1}} {2}'.format('Encoded X image:', fill, self.x_image))
"""
# Start construction of the Keras Sequential model.
self.model = Sequential()
# Add an input layer which is similar to a feed_dict in TensorFlow.
# Note that the input-shape must be a tuple containing the image-size.
self.model.add(InputLayer(input_shape=(self.img_size_flat,)))
# The input is a flattened array with 784 elements,
# but the convolutional layers expect images with shape (28, 28, 1)
self.model.add(Reshape(self.img_shape_full))
# First convolutional layer with ReLU-activation and max-pooling.
self.model.add(
Conv2D(kernel_size=5, strides=1, filters=16, padding='same',
activation='relu', name='layer_conv1'))
self.model.add(MaxPooling2D(pool_size=2, strides=2))
# Second convolutional layer with ReLU-activation and max-pooling.
self.model.add(
Conv2D(kernel_size=5, strides=1, filters=36, padding='same',
activation='relu', name='layer_conv2'))
self.model.add(MaxPooling2D(pool_size=2, strides=2))
# Flatten the 4-rank output of the convolutional layers
# to 2-rank that can be input to a fully-connected / dense layer.
self.model.add(Flatten())
# First fully-connected / dense layer with ReLU-activation.
self.model.add(Dense(128, activation='relu'))
# Last fully-connected / dense layer with softmax-activation
# for use in classification.
self.model.add(Dense(self.num_classes, activation='softmax'))
# Model Compilation
'''
The Neural Network has now been defined and must be finalized by adding
a loss-function, optimizer and performance metrics. This is called
model "compilation" in Keras.
We can either define the optimizer using a string, or if we want more
control of its parameters then we need to instantiate an object. For
example, we can set the learning-rate.
'''
optimizer = Adam(lr=1e-3)
'''
For a classification-problem such as MNIST which has 10 possible
classes, we need to use the loss-function called
categorical_crossentropy. The performance metric we are interested in
is the classification accuracy.
'''
self.model.compile(
optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
# Training
'''
Now that the model has been fully defined with loss-function and
optimizer, we can train it. This function takes numpy-arrays and
performs the given number of training epochs using the given
batch-size. An epoch is one full use of the entire training-set. So for
10 epochs we would iterate randomly over the entire training-set 10
times.
'''
self.model.fit(x=self.data.x_train,
y=self.data.y_train,
epochs=epochs, batch_size=128)
# Evaluation
'''
Now that the model has been trained we can test its performance on the
test-set. This also uses numpy-arrays as input.
'''
result = self.model.evaluate(x=self.data.x_test, y=self.data.y_test)
'''
Print actual versus predicted values
'''
print('\nActual vs Predicted X values')
start = 0
stop = 300
predictions = self.model.predict(self.data.x_test[start:stop])
for pointer in range(start, stop):
predicted = np.argmax(predictions[pointer])
actual = np.argmax(self.data.y_test[pointer])
print(
'{}: Actual: {}\tPredicted: {}\tMatch: {}'.format(
str(pointer).zfill(3),
predicted, actual, predicted == actual))
'''
We can print all the performance metrics for the test-set.
'''
print('\nPerfomance metrics')
for name, value in zip(self.model.metrics_names, result):
print('{} {}'.format(name, value))
'''
Print the model summary
'''
print('\n\nModel Summary\n\n{}'.format(self.model.summary()))
def plot_example_errors(self, cls_pred):
"""Plot 9 images in a 3x3 grid.
Function used to plot 9 images in a 3x3 grid, and writing the true and
predicted classes below each image.
Args:
cls_pred: Array of the predicted class-number for all images in the
test-set.
Returns:
None
"""
# Boolean array whether the predicted class is incorrect.
incorrect = (cls_pred != self.data.y_test_cls)
# Get the images from the test-set that have been
# incorrectly classified.
images = self.data.x_test[incorrect]
# Get the predicted classes for those images.
cls_pred = cls_pred[incorrect]
# Get the true classes for those images.
cls_true = self.data.y_test_cls[incorrect]
# Plot the first 9 images.
plot_images(
images[0:9], self.img_shape, cls_true[0:9], cls_pred=cls_pred[0:9])
model = Sequential()
# RNN cell
model.add(
LSTM(CELL_SIZE,
batch_input_shape=(None, TIME_STEPS, INPUT_SIZE),
dropout=0.2)) # LSTM 层
model.add(Dense(OUTPUT_SIZE)) # 二分类层
model.add(Activation('softmax')) # Sigmoid
# optimizer
adam = Adam(LR)
model.compile(optimizer=adam,
loss='categorical_crossentropy',
metrics=['accuracy'])
# training
for step in range(4001):
# data shape = (batch_num, steps, inputs/outputs)
X_batch = X_train[BATCH_INDEX:BATCH_INDEX + BATCH_SIZE, :, :]
Y_batch = y_train[BATCH_INDEX:BATCH_INDEX + BATCH_SIZE, :]
cost = model.train_on_batch(X_batch, Y_batch)
BATCH_INDEX += BATCH_SIZE
BATCH_INDEX = 0 if BATCH_INDEX >= X_train.shape[0] else BATCH_INDEX
if step % 500 == 0:
cost, accuracy = model.evaluate(X_test,
y_test,
batch_size=y_test.shape[0],
def main():
if sys.platform == 'darwin':
# mac
matplotlib.use('TkAgg')
print('Using Mac OS')
# get expert_data
parser = argparse.ArgumentParser()
parser.add_argument('expert_data', type=str)
parser.add_argument('envname', type=str)
parser.add_argument('--batch_size',
dest='batch_size',
type=int,
default=32)
parser.add_argument('--epoch', dest='epoch', type=int, default=100)
parser.add_argument('--lr', dest='lr', type=float, default=1e-3)
parser.add_argument('--render', action='store_true')
parser.add_argument("--max_timesteps", type=int)
parser.add_argument('--num_rollouts',
type=int,
default=5,
help='Number of expert roll outs')
args = parser.parse_args()
with open(args.expert_data, 'rb') as f:
expert_data = pickle.loads(f.read())
obs = expert_data['observations']
acts = expert_data['actions']
acts = np.squeeze(acts, axis=[1])
num_exmple = obs.shape[0]
print('number of training examples: ', num_exmple)
print('dimension of observation: ', obs[0].shape)
print('dimension of action: ', acts[0].shape)
# shuffle_list = np.arange(num_exmple)
# np.random.shuffle(shuffle_list)
# obs, acts = obs[shuffle_list], acts[shuffle_list]
obs, acts = shuffle(obs, acts, random_state=0)
split = int(0.8 * num_exmple)
obs_train, acts_train = obs[:split], acts[:split]
obs_val, acts_val = obs[split:], acts[split:]
model = Sequential()
model.add(Dense(128, activation='relu', input_shape=(obs.shape[1], )))
model.add(Dense(128, activation='relu'))
# model.add(Dense(512, activation='relu'))
# model.add(Dense(256, activation='relu'))
model.add(Dense(128, activation='relu'))
model.add(Dense(acts.shape[1], activation='linear'))
model.compile(loss='mse', optimizer='adam', metrics=['accuracy'])
model.fit(obs_train,
acts_train,
batch_size=args.batch_size,
epochs=args.epoch,
verbose=1)
score = model.evaluate(obs_val, acts_val, verbose=1)
model.save('output/' + args.envname + '_bc.h5')
# set up session
tfconfig = tf.ConfigProto()
tfconfig.gpu_options.allow_growth = True
num_train = obs_train.shape[0]
shuffle_list = np.arange(num_train)
losses = []
with tf.Session(config=tfconfig) as sess:
# model = Behavioral_clone(obs.shape[1], acts.shape[1])
# model.build_net([128, 256, 512, 256, 128], lr=args.lr)
# sess.run(tf.global_variables_initializer())
tf_util.initialize()
env = gym.make(args.envname)
max_steps = args.max_timesteps or env.spec.timestep_limit
returns = []
observations = []
actions = []
model = load_model('output/' + args.envname + '_bc.h5')
for i in range(args.num_rollouts):
print('iter', i)
obs = env.reset()
done = False
totalr = 0.
steps = 0
while not done:
# action = model.action(sess, obs[None, :])
obs = obs.reshape(1, len(obs))
action = (model.predict(obs, batch_size=64, verbose=0))
observations.append(obs)
actions.append(action)
obs, r, done, _ = env.step(action)
totalr += r
steps += 1
if args.render:
env.render()
if steps % 100 == 0:
print("%i/%i" % (steps, max_steps))
if steps >= max_steps:
break
returns.append(totalr)
print('returns', returns)
print('mean return', np.mean(returns))
print('std of return', np.std(returns))
model.add(BatchNormalization())
model.add(
LSTM(units=32, dropout=0.3, return_sequences=True, recurrent_dropout=0.3))
model.add(BatchNormalization())
model.add(LSTM(units=32, dropout=0.3, recurrent_dropout=0.3))
model.add(BatchNormalization())
model.add(Dense(512, activation='relu'))
model.add(BatchNormalization())
model.add(Dense(256, activation='relu'))
model.add(BatchNormalization())
model.add(Dense(64, activation='relu'))
model.add(BatchNormalization())
model.add(Dense(1, activation='sigmoid'))
adam = Adam(lr=0.0003)
model.compile(loss='mean_squared_error', optimizer=adam, metrics=['accuracy'])
print('Summary of the built model : ')
print(model.summary())
"""
Training the model
"""
model.fit(X_train_pad,
y_train,
batch_size=256,
epochs=10,
validation_data=(X_val_pad, y_val),
verbose=1)
"""
Validating the model
"""
n_cols = X_train.shape[1] #nbr of columns in predictors
# Set up the model: model
model = Sequential()
# Add the first layer
model.add(Dense(50, activation='relu', input_shape=(n_cols, )))
# Add the second layer
model.add(Dense(100, activation='relu'))
# Add the output layer
model.add(Dense(2))
# Compile the model
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
# Fit the model
r = model.fit(X_train, Y_train, validation_split=0.3, epochs=40)
# Compile the model
model.compile(optimizer='adam', loss='mean_squared_error')
# Print the loss
print("Loss function: " + model.loss)
# Fit the model
r = model.fit(X_train, Y_train, epochs=40, validation_data=(X_test, Y_test))
predictions = model.predict(X_test)
model.add(Dense(15, activation='relu'))
model.add(Dense(out_dim, activation='linear')) # TODO:: improve architecture!
#opt = keras.optimizers.SGD(lr=0.01, momentum=0.9, decay=0.01)
#opt = keras.optimizers.Adam(learning_rate=0.01)
opt = keras.optimizers.Adam(learning_rate=0.01, decay=1e-3/100)
model.summary()
keras2ascii(model)
# mse: loss = square(y_true - y_pred)
# mae: loss = abs(y_true - y_pred)
# mape: loss = 100 * abs(y_true - y_pred) / y_true
# msle: loss = square(log(y_true + 1.) - log(y_pred + 1.))
model.compile(loss='mse', metrics=['mse', 'mae', 'mape', 'msle'], optimizer=opt)
#monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, patience=5, verbose=1, mode='auto', restore_best_weights=True)
print("[INFO] training model...")
#history = model.fit(x_train, y_train, epochs=100, batch_size=15, verbose=2, validation_data=(x_test, y_test), callbacks=[PlotLossesKeras()])
history = model.fit(x_train, y_train, epochs=10, batch_size=64, verbose=2, validation_data=(x_test, y_test))
#loss_history = np.array(history)
#np.savetxt("loss_history.txt", loss_history, delimiter=",")
# Plot metrics
print(history.history.keys())
# "Loss"
plt.figure()
def onBeginTraining(self):
ue.log("starting mnist keras cnn training")
model_file_name = "mnistKerasCNN"
model_directory = ue.get_content_dir() + "/Scripts/"
model_sess_path = model_directory + model_file_name + ".tfsess"
model_json_path = model_directory + model_file_name + ".json"
my_file = Path(model_json_path)
#reset the session each time we get training calls
K.clear_session()
#let's train
batch_size = 128
num_classes = 10
epochs = 5 # lower default for simple testing
self.batch_size = batch_size # so that it can be obtained inside keras callbacks
# input image dimensions
img_rows, img_cols = 28, 28
# the data, shuffled and split between train and test sets
(x_train, y_train), (x_test, y_test) = mnist.load_data()
if K.image_data_format() == 'channels_first':
x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)
input_shape = (1, img_rows, img_cols)
else:
x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
input_shape = (img_rows, img_cols, 1)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
ue.log('x_train shape:' + str(x_train.shape))
ue.log(str(x_train.shape[0]) + 'train samples')
ue.log(str(x_test.shape[0]) + 'test samples')
#pre-fill our callEvent data to optimize callbacks
jsonPixels = {}
size = {'x':28, 'y':28}
jsonPixels['size'] = size
self.jsonPixels = jsonPixels
self.x_train = x_train
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
model = Sequential()
model.add(Conv2D(64, kernel_size=(3, 3),
activation='relu',
input_shape=input_shape))
# model.add(Dropout(0.2))
# model.add(Flatten())
# model.add(Dense(512, activation='relu'))
# model.add(Dropout(0.2))
# model.add(Dense(num_classes, activation='softmax'))
#model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
model.fit(x_train, y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test),
callbacks=[self.stopcallback])
score = model.evaluate(x_test, y_test, verbose=0)
ue.log("mnist keras cnn training complete.")
ue.log('Test loss:' + str(score[0]))
ue.log('Test accuracy:' + str(score[1]))
self.session = K.get_session()
self.model = model
stored = {'model':model, 'session': self.session}
#run a test evaluation
ue.log(x_test.shape)
result_test = model.predict(np.reshape(x_test[500],(1,28,28,1)))
ue.log(result_test)
#flush the architecture model data to disk
#with open(model_json_path, "w") as json_file:
# json_file.write(model.to_json())
#flush the whole model and weights to disk
#saver = tf.train.Saver()
#save_path = saver.save(K.get_session(), model_sess_path)
#model.save(model_path)
return stored
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
y_train = tf.keras.utils.to_categorical(y_train, num_classes)
y_test = tf.keras.utils.to_categorical(y_test, num_classes)
model = Sequential()
model.add(Dense(512, activation='relu', input_shape=(784,)))
model.add(Dropout(0.2))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(num_classes, activation='softmax'))
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(),
metrics=['accuracy'])
history = model.fit(x_train, y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test))
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
assert score[1] > 0.95
class FaceRecogPredictor:
def __init__(self, labels, from_json=False):
self.from_json = from_json
self.labels = labels
if from_json:
self.load()
else:
self.amt_labels = len(self.labels)
self.dirs = [PHOTO_DIR + label + "/" for label in self.labels]
self.model = Sequential()
self.model.add(
Conv2D(64,
kernel_size=1,
activation="relu",
input_shape=(1, PHOTO_ROWS, PHOTO_COLS)))
self.model.add(Conv2D(32, kernel_size=1, activation="relu"))
self.model.add(Flatten())
self.model.add(Dense(16, activation="relu"))
self.model.add(Dense(self.amt_labels, activation="softmax"))
self.model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=["accuracy"])
def fetch_data(self):
xs = []
ys = []
for ix, direc in enumerate(self.dirs):
y = np.array([0 for i in range(self.amt_labels)])
y[ix] = 1
photos = [direc + fname for fname in os.listdir(direc)]
for p in photos:
photo = Image.open(p)
resized = photo.resize((PHOTO_ROWS, PHOTO_COLS))
gray = resized.convert("L")
xs.append(np.array([np.array(gray)]))
ys.append(y)
return (xs, ys)
def get_train_test(self, xs, ys, split=0.7):
sh_xs, sh_ys = shuffle(xs, ys)
len_xs = len(xs)
train_size = int(len_xs * split)
x_train, y_train = sh_xs[:train_size], sh_ys[:train_size]
x_test, y_test = sh_xs[train_size + 1:len_xs], sh_ys[train_size +
1:len_xs]
return (x_train, x_test, y_train, y_test)
def train(self):
xs, ys = self.fetch_data()
x_train, x_test, y_train, y_test = self.get_train_test(xs, ys)
self.model.fit(np.array(x_train),
np.array(y_train),
validation_data=(np.array(x_test), np.array(y_test)),
epochs=100)
def predict(self, x):
prediction = self.model.predict(x)
ix = np.argmax(prediction)
return self.labels[ix]
def save(self):
with open(MODEL_DIR + "model.json", "w") as json_file:
json_file.write(self.model.to_json())
self.model.save_weights("model.h5")
def load(self):
inside = os.listdir(MODEL_DIR)
if "model.json" not in inside or "model.h5" not in inside:
raise ValueError(
"Make sure both model.json and model.h5 are inside \'models/\'"
)
with open(MODEL_DIR + "model.json", "w") as json_file:
self.model = model_from_json(json_file.read())
self.model.load_weights(MODEL_DIR + "model.h5")
def model(self, params=None):
"""Create the Recurrent Neural Network.
Args:
None
Returns:
_model: RNN model
"""
# Initialize key variables
if params is None:
_hyperparameters = self.hyperparameters
else:
_hyperparameters = params
# Calculate the steps per epoch
epoch_steps = int(
self.training_rows / _hyperparameters['batch_size']) + 1
# Create the model object
_model = Sequential()
'''
We can now add a Gated Recurrent Unit (GRU) to the network. This will
have 512 outputs for each time-step in the sequence.
Note that because this is the first layer in the model, Keras needs to
know the shape of its input, which is a batch of sequences of arbitrary
length (indicated by None), where each observation has a number of
input-signals (num_x_signals).
'''
_model.add(GRU(
units=_hyperparameters['units'],
return_sequences=True,
recurrent_dropout=_hyperparameters['dropout'],
input_shape=(None, self._training_vector_count,)))
for _ in range(1, _hyperparameters['layers']):
_model.add(GRU(
units=_hyperparameters['units'],
recurrent_dropout=_hyperparameters['dropout'],
return_sequences=True))
'''
The GRU outputs a batch of sequences of 512 values. We want to predict
3 output-signals, so we add a fully-connected (or dense) layer which
maps 512 values down to only 3 values.
The output-signals in the data-set have been limited to be between 0
and 1 using a scaler-object. So we also limit the output of the neural
network using the Sigmoid activation function, which squashes the
output to be between 0 and 1.
'''
_model.add(
Dense(self._training_class_count, activation='sigmoid'))
'''
A problem with using the Sigmoid activation function, is that we can
now only output values in the same range as the training-data.
For example, if the training-data only has values between -20 and +30,
then the scaler-object will map -20 to 0 and +30 to 1. So if we limit
the output of the neural network to be between 0 and 1 using the
Sigmoid function, this can only be mapped back to values between
-20 and +30.
We can use a linear activation function on the output instead. This
allows for the output to take on arbitrary values. It might work with
the standard initialization for a simple network architecture, but for
more complicated network architectures e.g. with more layers, it might
be necessary to initialize the weights with smaller values to avoid
NaN values during training. You may need to experiment with this to
get it working.
'''
if False:
# Maybe use lower init-ranges.
init = RandomUniform(minval=-0.05, maxval=0.05)
_model.add(Dense(
self._training_class_count,
activation='linear',
kernel_initializer=init))
# Compile Model
'''
This is the optimizer and the beginning learning-rate that we will use.
We then compile the Keras model so it is ready for training.
'''
optimizer = RMSprop(lr=1e-3)
_model.compile(
loss=self._loss_mse_warmup,
optimizer=optimizer,
metrics=['accuracy'])
'''
This is a very small model with only two layers. The output shape of
(None, None, 3) means that the model will output a batch with an
arbitrary number of sequences, each of which has an arbitrary number of
observations, and each observation has 3 signals. This corresponds to
the 3 target signals we want to predict.
'''
print('\n> Model Summary:\n')
print(_model.summary())
# Create the batch-generator.
generator = self._batch_generator(
_hyperparameters['batch_size'],
_hyperparameters['sequence_length'])
# Validation Set
'''
The neural network trains quickly so we can easily run many training
epochs. But then there is a risk of overfitting the model to the
training-set so it does not generalize well to unseen data. We will
therefore monitor the model's performance on the test-set after each
epoch and only save the model's weights if the performance is improved
on the test-set.
The batch-generator randomly selects a batch of short sequences from
the training-data and uses that during training. But for the
validation-data we will instead run through the entire sequence from
the test-set and measure the prediction accuracy on that entire
sequence.
'''
validation_data = (np.expand_dims(self._x_validation_scaled, axis=0),
np.expand_dims(self._y_validation_scaled, axis=0))
# Callback Functions
'''
During training we want to save checkpoints and log the progress to
TensorBoard so we create the appropriate callbacks for Keras.
This is the callback for writing checkpoints during training.
'''
callback_checkpoint = ModelCheckpoint(filepath=self._path_checkpoint,
monitor='val_loss',
verbose=1,
save_weights_only=True,
save_best_only=True)
'''
This is the callback for stopping the optimization when performance
worsens on the validation-set.
'''
callback_early_stopping = EarlyStopping(
monitor='val_loss',
patience=_hyperparameters['patience'],
verbose=1)
'''
This is the callback for writing the TensorBoard log during training.
'''
callback_tensorboard = TensorBoard(log_dir='/tmp/23_logs/',
histogram_freq=0,
write_graph=False)
'''
This callback reduces the learning-rate for the optimizer if the
validation-loss has not improved since the last epoch
(as indicated by patience=0). The learning-rate will be reduced by
multiplying it with the given factor. We set a start learning-rate of
1e-3 above, so multiplying it by 0.1 gives a learning-rate of 1e-4.
We don't want the learning-rate to go any lower than this.
'''
callback_reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
min_lr=1e-4,
patience=0,
verbose=1)
callbacks = [callback_early_stopping,
callback_checkpoint,
callback_tensorboard,
callback_reduce_lr]
# Train the Recurrent Neural Network
'''We can now train the neural network.
Note that a single "epoch" does not correspond to a single processing
of the training-set, because of how the batch-generator randomly
selects sub-sequences from the training-set. Instead we have selected
steps_per_epoch so that one "epoch" is processed in a few minutes.
With these settings, each "epoch" took about 2.5 minutes to process on
a GTX 1070. After 14 "epochs" the optimization was stopped because the
validation-loss had not decreased for 5 "epochs". This optimization
took about 35 minutes to finish.
Also note that the loss sometimes becomes NaN (not-a-number). This is
often resolved by restarting and running the Notebook again. But it may
also be caused by your neural network architecture, learning-rate,
batch-size, sequence-length, etc. in which case you may have to modify
those settings.
'''
print('\n> Parameters for training\n')
pprint(_hyperparameters)
print('\n> Starting data training\n')
_model.fit_generator(
generator=generator,
epochs=_hyperparameters['epochs'],
steps_per_epoch=epoch_steps,
validation_data=validation_data,
callbacks=callbacks)
# Return
return _model
s_cont_data.append(cont_data[i + 2])
cont_data = cont_data[:-2] #remove end stop character
# add
x = np.array(cont_data)
x = np.reshape(x, (int(x.size / 2), 2))
print(x)
y = np.array(s_cont_data)
#y = to_categorical(y, num_classes=vocab_size) #no need with sparse
model = Sequential()
model.add(Embedding(vocab_size * 5, 110,
input_length=2)) #vocab size with five different eras
model.add(LSTM(40)) # 40 hidden units
model.add(Dense(vocab_size, activation='softmax')) #choose next word
model.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'
]) #memory error with categorical_crossentropy
history = model.fit(x, y, epochs=4)
sequence = generate_seq(model, tok, 'Her', 500, 0, 4)
with open(path + 'standard_output.txt', 'w') as seq_out:
seq_out.write(sequence)
#https://stackoverflow.com/questions/39283358/keras-how-to-record-validation-loss #might want a validation set
loss_history = np.array(history.history['loss'])
np.savetxt(path + 'loss_standard.txt', loss_history, delimiter=',')
#model.save(path + 'histor_model.h5')
#https://stackoverflow.com/questions/45735070/keras-text-preprocessing-saving-tokenizer-object-to-file-for-scoring?utm_medium=organic&utm_source=google_rich_qa&utm_campaign=google_rich_qa
def _model(self):
"""Create the Recurrent Neural Network.
Args:
None
Returns:
_model: RNN model
"""
# Create the model object
_model = Sequential()
'''
We can now add a Gated Recurrent Unit (GRU) to the network. This will
have 512 outputs for each time-step in the sequence.
Note that because this is the first layer in the model, Keras needs to
know the shape of its input, which is a batch of sequences of arbitrary
length (indicated by None), where each observation has a number of
input-signals (num_x_signals).
'''
_model.add(GRU(
units=self._units,
return_sequences=True,
recurrent_dropout=self._dropout,
input_shape=(None, self._training_vector_count,)))
for _ in range(0, self._layers):
_model.add(GRU(
units=self._units,
recurrent_dropout=self._dropout,
return_sequences=True))
'''
The GRU outputs a batch of sequences of 512 values. We want to predict
3 output-signals, so we add a fully-connected (or dense) layer which
maps 512 values down to only 3 values.
The output-signals in the data-set have been limited to be between 0
and 1 using a scaler-object. So we also limit the output of the neural
network using the Sigmoid activation function, which squashes the
output to be between 0 and 1.'''
_model.add(
Dense(self._training_class_count, activation='sigmoid'))
'''
A problem with using the Sigmoid activation function, is that we can
now only output values in the same range as the training-data.
For example, if the training-data only has temperatures between -20
and +30 degrees, then the scaler-object will map -20 to 0 and +30 to 1.
So if we limit the output of the neural network to be between 0 and 1
using the Sigmoid function, this can only be mapped back to temperature
values between -20 and +30.
We can use a linear activation function on the output instead. This
allows for the output to take on arbitrary values. It might work with
the standard initialization for a simple network architecture, but for
more complicated network architectures e.g. with more layers, it might
be necessary to initialize the weights with smaller values to avoid
NaN values during training. You may need to experiment with this to
get it working.
'''
if False:
# Maybe use lower init-ranges.
# init = RandomUniform(minval=-0.05, maxval=0.05)
init = RandomUniform(minval=-0.05, maxval=0.05)
_model.add(Dense(
self._training_class_count,
activation='linear',
kernel_initializer=init))
# Compile Model
'''
This is the optimizer and the beginning learning-rate that we will use.
We then compile the Keras model so it is ready for training.
'''
optimizer = RMSprop(lr=1e-3)
_model.compile(
loss=self._loss_mse_warmup,
optimizer=optimizer,
metrics=['accuracy'])
'''
This is a very small model with only two layers. The output shape of
(None, None, 3) means that the model will output a batch with an
arbitrary number of sequences, each of which has an arbitrary number of
observations, and each observation has 3 signals. This corresponds to
the 3 target signals we want to predict.
'''
print('> Model Summary:\n')
print(_model.summary())
# Return
return _model
class RNNGRU(object):
"""Support vector machine class."""
def __init__(self,
batch_size=64, sequence_length=20, warmup_steps=50,
epochs=20, display=False):
"""Instantiate the class.
Args:
batch_size: Size of batch
sequence_length: Length of vectors for for each target
save: Save charts if True
Returns:
None
"""
# Initialize key variables
self.target_names = ['Temp', 'WindSpeed', 'Pressure']
self.warmup_steps = warmup_steps
self.epochs = epochs
self.batch_size = batch_size
self.display = display
# Get data
x_data, y_data = self.data()
print('\n> Numpy Data Type: {}'.format(type(x_data)))
print("> Numpy Data Shape: {}".format(x_data.shape))
print("> Numpy Data Row[0]: {}".format(x_data[0]))
print('> Numpy Targets Type: {}'.format(type(y_data)))
print("> Numpy Targets Shape: {}".format(y_data.shape))
'''
This is the number of observations (aka. data-points or samples) in
the data-set:
'''
num_data = len(x_data)
'''
This is the fraction of the data-set that will be used for the
training-set:
'''
train_split = 0.9
'''
This is the number of observations in the training-set:
'''
self.num_train = int(train_split * num_data)
'''
This is the number of observations in the test-set:
'''
num_test = num_data - self.num_train
print('> Number of Samples: {}'.format(num_data))
print("> Number of Training Samples: {}".format(self.num_train))
print("> Number of Test Samples: {}".format(num_test))
print("> Batch Size: {}".format(batch_size))
steps_per_epoch = int(self.num_train/batch_size)
print("> Recommended Epoch Steps: {:.2f}".format(steps_per_epoch))
# Create test and training data
x_train = x_data[0:self.num_train]
x_test = x_data[self.num_train:]
self.y_train = y_data[0:self.num_train]
self.y_test = y_data[self.num_train:]
self.num_x_signals = x_data.shape[1]
self.num_y_signals = y_data.shape[1]
print("> Training Minimum Value:", np.min(x_train))
print("> Training Maximum Value:", np.max(x_train))
'''
The neural network works best on values roughly between -1 and 1, so we
need to scale the data before it is being input to the neural network.
We can use scikit-learn for this.
We first create a scaler-object for the input-signals.
Then we detect the range of values from the training-data and scale
the training-data.
'''
x_scaler = MinMaxScaler()
self.x_train_scaled = x_scaler.fit_transform(x_train)
print('> Scaled Training Minimum Value: {}'.format(
np.min(self.x_train_scaled)))
print('> Scaled Training Maximum Value: {}'.format(
np.max(self.x_train_scaled)))
self.x_test_scaled = x_scaler.transform(x_test)
'''
The target-data comes from the same data-set as the input-signals,
because it is the weather-data for one of the cities that is merely
time-shifted. But the target-data could be from a different source with
different value-ranges, so we create a separate scaler-object for the
target-data.
'''
self.y_scaler = MinMaxScaler()
self.y_train_scaled = self.y_scaler.fit_transform(self.y_train)
y_test_scaled = self.y_scaler.transform(self.y_test)
# Data Generator
'''
The data-set has now been prepared as 2-dimensional numpy arrays. The
training-data has almost 300k observations, consisting of 20
input-signals and 3 output-signals.
These are the array-shapes of the input and output data:
'''
print('> Scaled Training Data Shape: {}'.format(
self.x_train_scaled.shape))
print('> Scaled Training Targets Shape: {}'.format(
self.y_train_scaled.shape))
# We then create the batch-generator.
generator = self.batch_generator(batch_size, sequence_length)
# Validation Set
'''
The neural network trains quickly so we can easily run many training
epochs. But then there is a risk of overfitting the model to the
training-set so it does not generalize well to unseen data. We will
therefore monitor the model's performance on the test-set after each
epoch and only save the model's weights if the performance is improved
on the test-set.
The batch-generator randomly selects a batch of short sequences from
the training-data and uses that during training. But for the
validation-data we will instead run through the entire sequence from
the test-set and measure the prediction accuracy on that entire
sequence.
'''
validation_data = (np.expand_dims(self.x_test_scaled, axis=0),
np.expand_dims(y_test_scaled, axis=0))
# Create the Recurrent Neural Network
self.model = Sequential()
'''
We can now add a Gated Recurrent Unit (GRU) to the network. This will
have 512 outputs for each time-step in the sequence.
Note that because this is the first layer in the model, Keras needs to
know the shape of its input, which is a batch of sequences of arbitrary
length (indicated by None), where each observation has a number of
input-signals (num_x_signals).
'''
self.model.add(GRU(
units=512,
return_sequences=True,
input_shape=(None, self.num_x_signals,)))
'''
The GRU outputs a batch of sequences of 512 values. We want to predict
3 output-signals, so we add a fully-connected (or dense) layer which
maps 512 values down to only 3 values.
The output-signals in the data-set have been limited to be between 0
and 1 using a scaler-object. So we also limit the output of the neural
network using the Sigmoid activation function, which squashes the
output to be between 0 and 1.'''
self.model.add(Dense(self.num_y_signals, activation='sigmoid'))
'''
A problem with using the Sigmoid activation function, is that we can
now only output values in the same range as the training-data.
For example, if the training-data only has temperatures between -20
and +30 degrees, then the scaler-object will map -20 to 0 and +30 to 1.
So if we limit the output of the neural network to be between 0 and 1
using the Sigmoid function, this can only be mapped back to temperature
values between -20 and +30.
We can use a linear activation function on the output instead. This
allows for the output to take on arbitrary values. It might work with
the standard initialization for a simple network architecture, but for
more complicated network architectures e.g. with more layers, it might
be necessary to initialize the weights with smaller values to avoid
NaN values during training. You may need to experiment with this to
get it working.
'''
if False:
# Maybe use lower init-ranges.
init = RandomUniform(minval=-0.05, maxval=0.05)
self.model.add(Dense(
self.num_y_signals,
activation='linear',
kernel_initializer=init))
# Compile Model
'''
This is the optimizer and the beginning learning-rate that we will use.
We then compile the Keras model so it is ready for training.
'''
optimizer = RMSprop(lr=1e-3)
self.model.compile(loss=self.loss_mse_warmup, optimizer=optimizer)
'''
This is a very small model with only two layers. The output shape of
(None, None, 3) means that the model will output a batch with an
arbitrary number of sequences, each of which has an arbitrary number of
observations, and each observation has 3 signals. This corresponds to
the 3 target signals we want to predict.
'''
print('> Model Summary:\n')
print(self.model.summary())
# Callback Functions
'''
During training we want to save checkpoints and log the progress to
TensorBoard so we create the appropriate callbacks for Keras.
This is the callback for writing checkpoints during training.
'''
path_checkpoint = '/tmp/23_checkpoint.keras'
callback_checkpoint = ModelCheckpoint(filepath=path_checkpoint,
monitor='val_loss',
verbose=1,
save_weights_only=True,
save_best_only=True)
'''
This is the callback for stopping the optimization when performance
worsens on the validation-set.
'''
callback_early_stopping = EarlyStopping(monitor='val_loss',
patience=5, verbose=1)
'''
This is the callback for writing the TensorBoard log during training.
'''
callback_tensorboard = TensorBoard(log_dir='/tmp/23_logs/',
histogram_freq=0,
write_graph=False)
'''
This callback reduces the learning-rate for the optimizer if the
validation-loss has not improved since the last epoch
(as indicated by patience=0). The learning-rate will be reduced by
multiplying it with the given factor. We set a start learning-rate of
1e-3 above, so multiplying it by 0.1 gives a learning-rate of 1e-4.
We don't want the learning-rate to go any lower than this.
'''
callback_reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
min_lr=1e-4,
patience=0,
verbose=1)
callbacks = [callback_early_stopping,
callback_checkpoint,
callback_tensorboard,
callback_reduce_lr]
# Train the Recurrent Neural Network
'''We can now train the neural network.
Note that a single "epoch" does not correspond to a single processing
of the training-set, because of how the batch-generator randomly
selects sub-sequences from the training-set. Instead we have selected
steps_per_epoch so that one "epoch" is processed in a few minutes.
With these settings, each "epoch" took about 2.5 minutes to process on
a GTX 1070. After 14 "epochs" the optimization was stopped because the
validation-loss had not decreased for 5 "epochs". This optimization
took about 35 minutes to finish.
Also note that the loss sometimes becomes NaN (not-a-number). This is
often resolved by restarting and running the Notebook again. But it may
also be caused by your neural network architecture, learning-rate,
batch-size, sequence-length, etc. in which case you may have to modify
those settings.
'''
self.model.fit_generator(
generator=generator,
epochs=self.epochs,
steps_per_epoch=steps_per_epoch,
validation_data=validation_data,
callbacks=callbacks)
# Load Checkpoint
'''
Because we use early-stopping when training the model, it is possible
that the model's performance has worsened on the test-set for several
epochs before training was stopped. We therefore reload the last saved
checkpoint, which should have the best performance on the test-set.
'''
try:
self.model.load_weights(path_checkpoint)
except Exception as error:
print('\n> Error trying to load checkpoint.\n\n{}'.format(error))
sys.exit(0)
# Performance on Test-Set
'''
We can now evaluate the model's performance on the test-set. This
function expects a batch of data, but we will just use one long
time-series for the test-set, so we just expand the
array-dimensionality to create a batch with that one sequence.
'''
result = self.model.evaluate(
x=np.expand_dims(self.x_test_scaled, axis=0),
y=np.expand_dims(y_test_scaled, axis=0))
print('> Loss (test-set): {}'.format(result))
# If you have several metrics you can use this instead.
if False:
for res, metric in zip(result, self.model.metrics_names):
print('{0}: {1:.3e}'.format(metric, res))
def batch_generator(self, batch_size, sequence_length):
"""Generator function for creating random batches of training-data.
Args:
batch_size: Size of batch
sequence_length: Length of sequence
Returns:
(x_batch, y_batch)
"""
# Infinite loop.
while True:
# Allocate a new array for the batch of input-signals.
x_shape = (batch_size, sequence_length, self.num_x_signals)
x_batch = np.zeros(shape=x_shape, dtype=np.float16)
# Allocate a new array for the batch of output-signals.
y_shape = (batch_size, sequence_length, self.num_y_signals)
y_batch = np.zeros(shape=y_shape, dtype=np.float16)
# Fill the batch with random sequences of data.
for i in range(batch_size):
# Get a random start-index.
# This points somewhere into the training-data.
idx = np.random.randint(self.num_train - sequence_length)
# Copy the sequences of data starting at this index.
x_batch[i] = self.x_train_scaled[idx:idx+sequence_length]
y_batch[i] = self.y_train_scaled[idx:idx+sequence_length]
yield (x_batch, y_batch)
def plot_comparison(self, start_idx, length=100, train=True):
"""Plot the predicted and true output-signals.
Args:
start_idx: Start-index for the time-series.
length: Sequence-length to process and plot.
train: Boolean whether to use training- or test-set.
Returns:
None
"""
if train:
# Use training-data.
x_values = self.x_train_scaled
y_true = self.y_train
shim = 'Train'
else:
# Use test-data.
x_values = self.x_test_scaled
y_true = self.y_test
shim = 'Test'
# End-index for the sequences.
end_idx = start_idx + length
# Select the sequences from the given start-index and
# of the given length.
x_values = x_values[start_idx:end_idx]
y_true = y_true[start_idx:end_idx]
# Input-signals for the model.
x_values = np.expand_dims(x_values, axis=0)
# Use the model to predict the output-signals.
y_pred = self.model.predict(x_values)
# The output of the model is between 0 and 1.
# Do an inverse map to get it back to the scale
# of the original data-set.
y_pred_rescaled = self.y_scaler.inverse_transform(y_pred[0])
# For each output-signal.
for signal in range(len(self.target_names)):
# Create a filename
filename = (
'/tmp/batch_{}_epochs_{}_training_{}_{}_{}_{}.png').format(
self.batch_size, self.epochs, self.num_train, signal,
int(time.time()), shim)
# Get the output-signal predicted by the model.
signal_pred = y_pred_rescaled[:, signal]
# Get the true output-signal from the data-set.
signal_true = y_true[:, signal]
# Make the plotting-canvas bigger.
plt.figure(figsize=(15, 5))
# Plot and compare the two signals.
plt.plot(signal_true, label='true')
plt.plot(signal_pred, label='pred')
# Plot grey box for warmup-period.
_ = plt.axvspan(
0, self.warmup_steps, facecolor='black', alpha=0.15)
# Plot labels etc.
plt.ylabel(self.target_names[signal])
plt.legend()
# Show and save the image
if self.display is True:
plt.savefig(filename, bbox_inches='tight')
plt.show()
else:
plt.savefig(filename, bbox_inches='tight')
print('> Saving file: {}'.format(filename))
def data(self):
"""Get data to analyze.
Args:
None
Returns:
(x_data, y_data): X and Y values as numpy arrays
"""
# Download data
weather.maybe_download_and_extract()
# Import data into Pandas dataframe
pandas_df = weather.load_resampled_data()
print('\n> First Rows of Data:\n\n{}'.format(pandas_df.head(3)))
# Print the cities
cities = weather.cities
print('\n> Cities: {}'.format(cities))
# Print dataframe shape
print(
'> Dataframe shape (Original): {}'.format(pandas_df.values.shape))
# The two signals that have missing data. (Columns with Nans)
pandas_df.drop(('Esbjerg', 'Pressure'), axis=1, inplace=True)
pandas_df.drop(('Roskilde', 'Pressure'), axis=1, inplace=True)
# Print dataframe shape
print('> Dataframe shape (New): {}'.format(pandas_df.values.shape))
# Verify that the columns have been dropped
print(
'\n> First Rows of Updated Data:\n\n{}'.format(pandas_df.head(1)))
# Add Data
'''
We can add some input-signals to the data that may help our model in
making predictions.
For example, given just a temperature of 10 degrees Celcius the model
wouldn't know whether that temperature was measured during the day or
the night, or during summer or winter. The model would have to infer
this from the surrounding data-points which might not be very accurate
for determining whether it's an abnormally warm winter, or an
abnormally cold summer, or whether it's day or night. So having this
information could make a big difference in how accurately the model can
predict the next output.
Although the data-set does contain the date and time information for
each observation, it is only used in the index so as to order the data.
We will therefore add separate input-signals to the data-set for the
day-of-year (between 1 and 366) and the hour-of-day (between 0 and 23).
'''
pandas_df['Various', 'Day'] = pandas_df.index.dayofyear
pandas_df['Various', 'Hour'] = pandas_df.index.hour
# Target Data for Prediction
'''
We will try and predict the future weather-data for this city.
'''
target_city = 'Odense'
'''
We will try and predict these signals.
'''
self.target_names = ['Temp', 'WindSpeed', 'Pressure']
'''
The following is the number of time-steps that we will shift the
target-data. Our data-set is resampled to have an observation for each
hour, so there are 24 observations for 24 hours.
If we want to predict the weather 24 hours into the future, we shift
the data 24 time-steps. If we want to predict the weather 7 days into
the future, we shift the data 7 * 24 time-steps.
'''
shift_days = 1
shift_steps = shift_days * 24 # Number of hours.
# Create a new data-frame with the time-shifted data.
'''
Note the negative time-shift!
We want the future state targets to line up with the timestamp of the
last value of each sample set.
'''
df_targets = pandas_df[
target_city][self.target_names].shift(-shift_steps)
'''
WARNING! You should double-check that you have shifted the data in the
right direction! We want to predict the future, not the past!
The shifted data-frame is confusing because Pandas keeps the original
This is the first shift_steps + 5 rows of the original data-frame:
'''
explanatory_hours = shift_steps + 5
print('\n> First Rows of Updated Data ({} hours):\n\n{}'.format(
explanatory_hours,
pandas_df[target_city][self.target_names].head(explanatory_hours)))
'''
The following is the first 5 rows of the time-shifted data-frame. This
should be identical to the last 5 rows shown above from the original
data, except for the time-stamp.
'''
print('\n> First Rows of Shifted Data - Target Labels '
'(Notice 1980 Dates):\n\n{}'.format(df_targets.head(5)))
'''
The time-shifted data-frame has the same length as the original
data-frame, but the last observations are NaN (not a number) because
the data has been shifted backwards so we are trying to shift data that
does not exist in the original data-frame.
'''
print('\n> Last Rows of Shifted Data - Target Labels '
'(Notice 2018 Dates):\n\n{}'.format(df_targets.tail()))
# NumPy Arrays
'''
We now convert the Pandas data-frames to NumPy arrays that can be input
to the neural network. We also remove the last part of the numpy
arrays, because the target-data has NaN for the shifted period, and we
only want to have valid data and we need the same array-shapes for the
input- and output-data.
These are the input-signals:
'''
x_data = pandas_df.values[0:-shift_steps]
y_data = df_targets.values[:-shift_steps]
# Return
return (x_data, y_data)
def loss_mse_warmup(self, y_true, y_pred):
"""Calculate the Mean Squared Errror.
Calculate the Mean Squared Error between y_true and y_pred,
but ignore the beginning "warmup" part of the sequences.
We will use Mean Squared Error (MSE) as the loss-function that will be
minimized. This measures how closely the model's output matches the
true output signals.
However, at the beginning of a sequence, the model has only seen
input-signals for a few time-steps, so its generated output may be very
inaccurate. Using the loss-value for the early time-steps may cause the
model to distort its later output. We therefore give the model a
"warmup-period" of 50 time-steps where we don't use its accuracy in the
loss-function, in hope of improving the accuracy for later time-steps
Args:
y_true: Desired output.
y_pred: Model's output.
Returns:
loss_mean: Mean Squared Error
"""
warmup_steps = self.warmup_steps
# The shape of both input tensors are:
# [batch_size, sequence_length, num_y_signals].
# Ignore the "warmup" parts of the sequences
# by taking slices of the tensors.
y_true_slice = y_true[:, warmup_steps:, :]
y_pred_slice = y_pred[:, warmup_steps:, :]
# These sliced tensors both have this shape:
# [batch_size, sequence_length - warmup_steps, num_y_signals]
# Calculate the MSE loss for each value in these tensors.
# This outputs a 3-rank tensor of the same shape.
loss = tf.losses.mean_squared_error(labels=y_true_slice,
predictions=y_pred_slice)
# Keras may reduce this across the first axis (the batch)
# but the semantics are unclear, so to be sure we use
# the loss across the entire tensor, we reduce it to a
# single scalar with the mean function.
loss_mean = tf.reduce_mean(loss)
return loss_mean
