def train_model(train_data_generator, val_data_generator, model, initial_epoch,
s):
"""
Model training.
# Arguments
train_data_generator: Training data generated batch by batch.
val_data_generator: Validation data generated batch by batch.
model: Target image channels.
initial_epoch: Dimension of model output.
"""
# Initialize loss weights
model.alpha = tf.Variable(s['alpha_loss_weight'],
trainable=False,
name='alpha',
dtype=tf.float32)
model.beta = tf.Variable(s['beta_loss_weight'],
trainable=False,
name='beta',
dtype=tf.float32)
# Initialize number of samples for hard-mining
model.k_mse = tf.Variable(s['batch_size'],
trainable=False,
name='k_mse',
dtype=tf.int32)
model.k_entropy = tf.Variable(s['batch_size'],
trainable=False,
name='k_entropy',
dtype=tf.int32)
# Set optimizer to adadelta (worked better over different dataset sizes than adam)
optimizer = optimizers.Adadelta()
# Configure training process
model.compile(loss=[
utils.hard_mining_mse(model.k_mse),
utils.hard_mining_entropy(model.k_entropy)
],
optimizer=optimizer,
loss_weights=[model.alpha, model.beta])
# Save training and validation losses.
save_model_and_loss = utils.MyCallback(filepath=os.path.join(
FLAGS.experiment_rootdir, s['model_dir']),
logpath=FLAGS.experiment_rootdir,
period=s['log_rate'],
batch_size=s['batch_size'])
# Train model
steps_per_epoch = int(
np.ceil(train_data_generator.samples / s['batch_size']))
validation_steps = int(
np.ceil(val_data_generator.samples / s['batch_size']))
model.fit_generator(train_data_generator,
epochs=s['epochs'],
steps_per_epoch=steps_per_epoch,
callbacks=[save_model_and_loss],
validation_data=val_data_generator,
validation_steps=validation_steps,
initial_epoch=initial_epoch)
return_sequences=False)(first_ind)
fc = Dense(128, kernel_initializer='he_normal')(second_ind)
ac = Activation('relu')(fc)
output = Dropout(FLAGS.dropout)(ac)
output = Dense(num_classes, activation='softmax')(output)
model = Model(input=[inputs], output=output)
# try using different optimizers and different optimizer configs
optimizer = optimizers.SGD()
if FLAGS.optimizer == "SGD":
optimizer = optimizers.SGD()
elif FLAGS.optimizer == "Adam":
optimizer = optimizers.Adam()
elif FLAGS.optimizer == "Adadelta":
optimizer = optimizers.Adadelta()
elif FLAGS.optimizer == "Adagrad":
optimizer = optimizers.Adagrad()
elif FLAGS.optimizer == "RMSprop":
optimizer = optimizers.RMSprop()
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
model.summary()
print('Train...')
modelCheckpoint = ModelCheckpoint(
'weights/%s_%s_%s_%s.h5' %
(FLAGS.dataName, FLAGS.units, FLAGS.batch_size, FLAGS.optimizer),
nb_classes = Y_train.shape[1]
print(nb_classes, 'classes')
rate = 0.1
model = Sequential()
model.add(Dense(128, input_shape=(dims, ), init='uniform', activation='relu'))
model.add(Dropout(0.5, noise_shape=None, seed=42))
model.add(Dense(128, init='uniform', activation='relu'))
model.add(Dropout(0.5, noise_shape=None, seed=42))
model.add(Dense(1, init='uniform', activation='sigmoid'))
sgd = optimizers.SGD(lr=0.01, decay=0.005, momentum=0.5, nesterov=True)
rmsprop = optimizers.RMSprop(lr=0.01, rho=0.9, epsilon=1e-08, decay=0.001)
adagrad = optimizers.Adagrad(lr=0.01, epsilon=1e-09, decay=0.0001)
adadelta = optimizers.Adadelta(lr=0.1, rho=0.95, epsilon=1e-08, decay=0.005)
adamax = optimizers.Adamax(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.001)
adam = optimizers.Adam(lr=0.01,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.001)
model.compile(optimizer=adagrad,
loss='binary_crossentropy',
metrics=["accuracy"])
model.summary()
# print(autoencoder.summary())
#create optimizer
if optim == 'sgd':
optimi = optimizers.SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)
elif optim == 'rmsprop':
optimi = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.0)
elif optim == 'adam':
optimi = optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
elif optim == 'adadelta':
optimi = optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0)
elif optim == 'adagrad':
optimi = optimizers.Adagrad(lr=0.01, epsilon=None, decay=0.0)
else:
optimi = optimizers.Adamax(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0)
#comipile generated model
model.compile(loss="categorical_crossentropy",
optimizer=optimi,
metrics=['accuracy'])
time_start = time.clock()
# model.add(Conv2D(64,(3,3),activation='relu'))
# 第二层：
# model.add(Conv2D(32, (3, 3), activation='relu')) # model.add(Dropout(0.25))
# model.add(MaxPooling2D(pool_size=(2, 2)))
# model.add(Dropout(0.5))
# 2、全连接层和输出层：
model.add(Flatten())
# model.add(Dense(128, activation='relu'))
# model.add(Dropout(0.5))
model.add(Dense(20, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(2, activation='softmax'))
model.summary()
model.compile(loss='binary_crossentropy' ,#'categorical_crossentropy', # ,
optimizer=optimizers.Adadelta(lr=0.01, rho=0.95, epsilon=1e-06), # ,'Adadelta'
metrics=['accuracy'])
# 模型训练
model.fit(x_train, y_train, batch_size=104, epochs=50)
# 模型得分
score = model.evaluate(x_train, y_train, verbose=0)
# 识别结果
y_pred = model.predict(x_train)
# 转onehot变label
y_predict = np.argmax(y_pred,axis=1)
# 精确度
y_train = np.argmax(y_train.values,axis=1)
accuracy = accuracy_score(y_train,y_predict)
scaler.fit(X_train)
X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)
X_validation = scaler.transform(X_validation)
# print(X_train)
# print(X_test)
# print(X_validation)
s1 = X_train.shape[0]
s2 = X_test.shape[0]
s3 = X_validation.shape[0]
X_train = pca.fit_transform(X_train).reshape(s1, 24, 24, 1)
X_test = pca.fit_transform(X_test).reshape(s2, 24, 24, 1)
X_validation = pca.fit_transform(X_validation).reshape(s3, 24, 24, 1)
model = CNNmodel(num_emotions=7)
optimizer = opt.Adadelta(lr=0.01, decay=1e-6)
model.compile(
loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'],
)
model.fit(X_train,
y_train,
validation_data=(X_validation, y_validation),
epochs=50,
verbose=2,
batch_size=128)
scores = model.evaluate(X_test, y_test, verbose=0)
print(scores[1] * 100)
input_samples.addSample("ttbb" + naming, label="ttbb")
input_samples.addSample("tt2b" + naming, label="tt2b")
input_samples.addSample("ttb" + naming, label="ttb")
input_samples.addSample("ttcc" + naming, label="ttcc")
input_samples.addSample("ttlf" + naming, label="ttlf")
# custom net config
from keras import optimizers
net_config = {
"layers": [100, 100, 100, 100, 100, 100, 100, 100],
"loss_function": "categorical_crossentropy",
"Dropout": 0.40,
"L2_Norm": 5e-4,
"batch_size": 1000,
"optimizer": optimizers.Adadelta(decay=0.999),
"activation_function": "elu",
"output_activation": "Softmax",
"earlystopping_percentage": 0.1,
"batchNorm": False,
}
# path to output directory (adjust NAMING)
savepath = basedir + "/workdir/" + "ttbarReco_GenLevelTraining_v2_" + str(
JTcategory)
# initializing DNN training class
dnn = DNN.DNN(
save_path=savepath,
input_samples=input_samples,
event_category=JTcategory,
def compile_adage(self):
# Compile the autoencoder to prepare for training
adadelta = optimizers.Adadelta(lr=self.learning_rate)
self.full_model.compile(optimizer=adadelta, loss=self.loss)
#print(data_set.X_train[(data_set.X_train['LabelMass']==0)])
print(
'Number of training: {}, validation: {} and total events: {}.'.format(
num_train, num_valid, num_tot))
#Define model with given parameters
model = KerasModel(shape_train[1], args.numlayer, args.numn, args.dropout)
#Possible optimizers
sgd = optimizers.SGD(lr=args.lr,
decay=1e-6,
momentum=args.momentum,
nesterov=True)
ada = optimizers.Adadelta(lr=1, rho=0.95, epsilon=None, decay=0.01)
nadam = keras.optimizers.Nadam(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
schedule_decay=0.004)
model.compile(
loss='binary_crossentropy' #f1_loss
,
optimizer=sgd,
metrics=['accuracy']) #,f1])
model.summary()
#model.save("modelNN_initial_"+args.model+".h5")
# Save the model architecture
def run_nn(**kwargs):
"""
Run the neural network for the given parameters
Adapted from the code provided in lecture
"""
# Start the timer
start_t = timer()
# Number of input, hidden, and output nodes
input_nodes = 784
hidden_nodes = 200
output_nodes = 10
# Set parameters
learning_rate = kwargs["learning_rate"]
optimizer = kwargs["optimizer"]
batch_size = kwargs["batch_size"]
epochs = kwargs["epochs"]
# Create the Keras model
model = Sequential()
model.add(
Dense(
hidden_nodes,
activation='sigmoid',
input_shape=(input_nodes, ),
bias=False,
))
model.add(Dense(
output_nodes,
activation='sigmoid',
bias=False,
))
# Print the model summary
model.summary()
# Set the optimizer
if optimizer == "adam":
opt = optimizers.Adam(learning_rate=learning_rate)
elif optimizer == "sgd":
opt = optimizers.SGD(learning_rate=learning_rate)
elif optimizer == "rmsprop":
opt = optimizers.RMSprop(learning_rate=learning_rate)
elif optimizer == "adagrad":
opt = optimizers.Adagrad(learning_rate=learning_rate)
elif optimizer == "adadelta":
opt = optimizers.Adadelta(learning_rate=learning_rate)
elif optimizer == "adamax":
opt = optimizers.Adamax(learning_rate=learning_rate)
elif optimizer == "nadam":
opt = optimizers.Nadam(learning_rate=learning_rate)
# Default optimizer is adam
else:
opt = optimizers.Adam(learning_rate=learning_rate)
# Define the error criterion, optimizer, and an optional metric to monitor during training
model.compile(
loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'],
)
# Load the mnist training data CSV
df = pd.read_csv("mnist_csv/mnist_train.csv", header=None)
# Columns 1-784 are the input values
x_train = np.asfarray(df.loc[:, 1:input_nodes].values)
x_train /= 255.0
# Column 0 is the desired label
labels = df.loc[:, 0].values
# Convert labels to one-hot vectors
y_train = np_utils.to_categorical(labels, output_nodes)
# Train the neural network
# Train the model
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1)
# Save the model
model.save('MNIST_3layer_keras.h5')
print('model saved')
# Test the model
# Load the MNIST test data CSV file into a list
test_data_file = open('mnist_csv/mnist_test.csv', 'r')
test_data_list = test_data_file.readlines()
test_data_file.close()
# Scorecard for how well the network performs, initially empty
scorecard = []
# Go through all the data in the test data set, one by one
for record in test_data_list:
# Split the record by the commas
data_sample = record.split(',')
# Correct answer is first value
correct_label = int(data_sample[0])
# Scale and shift the inputs
inputs = np.asfarray(data_sample[1:]) / 255.0
# Make prediction
outputs = model.predict(np.reshape(inputs, (1, len(inputs))))
# The index of the highest value corresponds to the label
label = np.argmax(outputs)
# Append correct or incorrect to list
if label == correct_label:
# Network's answer matches correct answer, add 1 to scorecard
scorecard.append(1)
else:
# Netowrk's answer doesn't match correct answer, add 0 to scorecard
scorecard.append(0)
pass
pass
# Calculate the accuracy
scorecard_array = np.asarray(scorecard)
accuracy = scorecard_array.sum() / scorecard_array.size
print('accuracy = {}'.format(accuracy))
# Stop the timer
end_t = timer()
execution_time = end_t - start_t
print('elapsed time = {}'.format(execution_time))
output = {'accuracy': accuracy, 'execution_time': execution_time}
return output
def get_150():
ctr=0
image_list=[]
gt_list=[]
folder_n = '\\data\\noisy_set\\'
file_path_n = path + folder_n
folder_g = '\\data\\gt_set\\'
file_path_g = path + folder_g
optimizers_list = ['adadelta']
loss_functions_list = ['mean_squared_error']
# adagrad = optimizers.Adagrad(lr=0.01, epsilon=None, decay=0.0)
# adam = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
adadelta = optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0)
for opt,lossfn in zip(optimizers_list,loss_functions_list):
for filename_n,filename_g in zip(os.listdir(file_path_n),os.listdir(file_path_g)): #assuming png
img_rows, img_cols, ch = 64, 64, 3
# print (filename_n)
with open(file_path_n + filename_n,'rb') as infile_n:
buf = infile_n.read()
x = np.fromstring(buf, dtype = 'uint8')
img = cv2.imdecode(x,1)
img = cv2.resize(img,(img_rows,img_cols))
img = cv2.resize(img,(0,0),fx=4,fy=4)
image_list.append(img)
with open(file_path_g + filename_g,'rb') as infile_g:
buf = infile_g.read()
x = np.fromstring(buf, dtype = 'uint8')
img = cv2.imdecode(x,cv2.IMREAD_UNCHANGED)
img = cv2.resize(img,(img_rows,img_cols))
img = cv2.resize(img,(0,0),fx=4,fy=4)
gt_list.append(img)
ctr = ctr + 1
if ctr%150 == 0:
print(ctr)
images_ip = np.asarray(image_list,dtype='float32')
train_size=int(0.8*len(images_ip))
x_train = images_ip[:train_size]
x_test = images_ip[train_size:]
del(images_ip)
x_train /= 255
x_test /= 255
images_ip_gt = np.asarray(gt_list,dtype='float32')
y_train = images_ip_gt[:train_size]
y_test = images_ip_gt[train_size:]
del(images_ip_gt)
y_train /= 255
y_test /= 255
image_list=[]
gt_list=[]
img_rows, img_cols, ch = 256, 256, 3
if K.image_data_format() == 'channels_first':
x_train = x_train.reshape(x_train.shape[0], ch, img_rows, img_cols)
x_test = x_test.reshape(x_test.shape[0], ch, img_rows, img_cols)
y_train = y_train.reshape(y_train.shape[0], ch, img_rows, img_cols)
y_test = y_test.reshape(y_test.shape[0], ch, img_rows, img_cols)
input_shape = (ch, img_rows, img_cols)
else:
x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, ch)
x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, ch)
y_train = y_train.reshape(y_train.shape[0], img_rows, img_cols, ch)
y_test = y_test.reshape(y_test.shape[0], img_rows, img_cols, ch)
input_shape = (img_rows, img_cols, ch)
# if opt == 'adagrad':
# opt_spec = adagrad
# if opt == 'adam':
# opt_spec = adam
if opt == 'adadelta':
opt_spec = adadelta
if ctr == 150:
print(ctr)
autoencoder = new_model(input_shape,opt,opt_spec,lossfn)
print(opt,lossfn)
train_model(opt,opt_spec,lossfn,x_train,y_train,x_test,y_test)
kw,
padding='valid',
kernel_initializer=initializers.RandomNormal(np.sqrt(2 / kw)),
input_shape=(tam_fijo, embedding_vecor_length)))
submodel.add(advanced_activations.PReLU(initializers.Constant(value=0.25)))
submodel.add(GlobalMaxPooling1D())
submodels.append(submodel)
model = Sequential()
model.add(Merge(submodels, mode="concat"))
model.add(Dropout(dropout))
model.add(Dense(3, activation='softmax'))
# Log to tensorboard
tensorBoardCallback = TensorBoard(log_dir='./logs22', write_graph=True)
adadelta = optimizers.Adadelta(lr=alpha)
model.compile(loss='categorical_crossentropy',
optimizer=adadelta,
metrics=['accuracy', 'mse'])
model.fit([X_train] * len(size_filters),
y_train,
epochs=n_iter,
callbacks=[tensorBoardCallback],
batch_size=size_batch,
validation_data=([X_test] * len(size_filters), y_test))
# Evaluation on the test set
scores = model.evaluate([X_test] * len(size_filters), y_test, verbose=0)
print("---- TEST EVALUATION ----")
print("Accuracy: %.2f%%" % (scores[1] * 100))
def get_optimizer(opt,
decay=None,
lr=None,
momentum=0.0,
nesterov=False,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-8,
rho=None):
"""
get_optimizer is a wrapper for Keras optimizers.
Parameters
----------
beta_1 : `float`
adam optimizer parameter in range [0, 1) for updating bias first
moment estimate
beta_2 : `float`
adam optimizer parameter in range [0, 1) for updating bias second
moment estimate
decay : `None` or `float`
learning rate decay
epsilon : `float`
parameter for numerical stability
opt : `str`
Keras optimizer. Options: "sgd", "adam", "nadam", "rmsprop",
"adagrad", "adamax" and "adadelta"
lr : `None` or `float`
optimizer learning rate
momentum : `float`
accelerate the gradient descent in the direction that dampens
oscillations
nesterov : `bool`
use Nesterov Momentum
rho : `None` or `float`
gradient history
Returns
-------
optimizer : :class:`keras.optimizer`
keras optimizer object
"""
###############################
# Stochastic Gradient Descent #
###############################
if opt == 'sgd':
if lr is None:
lr = 0.01
if decay is None:
decay = 0.0
optimizer = optimizers.SGD(lr=lr,
momentum=momentum,
decay=decay,
nesterov=nesterov)
########
# Adam #
########
elif opt == 'adam':
if lr is None:
lr = 0.001
if decay is None:
decay = 0.0
optimizer = optimizers.Adam(lr=lr,
beta_1=beta_1,
beta_2=beta_2,
epsilon=epsilon,
decay=decay)
##########
# Adamax #
##########
elif opt == 'adamax':
if lr is None:
lr = 0.002
if decay is None:
decay = 0.0
optimizer = optimizers.Adam(lr=lr,
beta_1=beta_1,
beta_2=beta_2,
epsilon=epsilon,
decay=decay)
#########
# Nadam #
#########
# It is recommended to leave the parameters of this
# optimizer at their default values.
elif opt == 'nadam':
if lr is None:
lr = 0.002
if decay is None:
decay = 0.004
optimizer = optimizers.Adam(lr=lr,
beta_1=beta_1,
beta_2=beta_2,
epsilon=epsilon,
decay=decay)
###########
# RMSprop #
###########
# It is recommended to leave the parameters of this
# optimizer at their default values (except the learning
# rate, which can be freely tuned).
elif opt == 'rmsprop':
if lr is None:
lr = 0.001
if decay is None:
decay = 0.0
if rho is None:
rho = 0.9
optimizer = optimizers.RMSprop(lr=lr,
rho=rho,
epsilon=epsilon,
decay=decay)
###########
# Adagrad #
###########
# It is recommended to leave the parameters of this
# optimizer at their default values.
elif opt == 'adagrad':
if lr is None:
lr = 0.01
if decay is None:
decay = 0.0
optimizer = optimizers.Adagrad(lr=lr, decay=decay, epsilon=epsilon)
############
# Adadelta #
############
# It is recommended to leave the parameters of this
# optimizer at their default values.
elif opt == 'adadelta':
if lr is None:
lr = 1.0
if decay is None:
decay = 0.0
if rho is None:
rho = 0.95
optimizer = optimizers.Adadelta(lr=lr,
rho=rho,
epsilon=epsilon,
decay=decay)
else:
print('ERROR: Unknown optimizer')
sys.exit(1)
return optimizer
def build3CnnModel(embeddingMatrix):
"""Constructs the architecture of the model
Input:
embeddingMatrix : The embedding matrix to be loaded in the embedding layer.
Output:
model : A basic LSTM model
"""
embeddingLayer = Embedding(embeddingMatrix.shape[0],
EMBEDDING_DIM,
weights=[embeddingMatrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
# model = Sequential()
# model.add(embeddingLayer)
# model.add(LSTM(LSTM_DIM, dropout=DROPOUT))
# model.add(Dense(NUM_CLASSES, activation='sigmoid'))
#
# rmsprop = optimizers.rmsprop(lr=LEARNING_RATE)
# model.compile(loss='categorical_crossentropy',
# optimizer=rmsprop,
# metrics=['acc'])
########################################################
print('Training model.')
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedded_sequences = embeddingLayer(sequence_input)
# add first conv filter
embedded_sequences = Reshape((MAX_SEQUENCE_LENGTH, EMBEDDING_DIM, 1))(embedded_sequences)
x = Conv2D(100, (5, EMBEDDING_DIM), activation='relu')(embedded_sequences)
x = MaxPooling2D((MAX_SEQUENCE_LENGTH - 5 + 1, 1))(x)
# add second conv filter.
y = Conv2D(100, (4, EMBEDDING_DIM), activation='relu')(embedded_sequences)
y = MaxPooling2D((MAX_SEQUENCE_LENGTH - 4 + 1, 1))(y)
# add third conv filter.
z = Conv2D(100, (3, EMBEDDING_DIM), activation='relu')(embedded_sequences)
z = MaxPooling2D((MAX_SEQUENCE_LENGTH - 3 + 1, 1))(z)
# concate the conv layers
alpha = concatenate([x, y, z])
# flatted the pooled features.
alpha = Flatten()(alpha)
# dropout
alpha = Dropout(0.5)(alpha)
# predictions
preds = Dense(NUM_CLASSES, activation='softmax')(alpha)
# build model
model = Model(sequence_input, preds)
adadelta = optimizers.Adadelta()
model.compile(loss='categorical_crossentropy',
optimizer=adadelta,
metrics=['acc'])
return model
def lstm_dense_sunspots(args):
"""
Main function
"""
# %%
# IMPORTS
# code repository sub-package imports
from artificial_neural_networks.code.utils.download_monthly_sunspots import \
download_monthly_sunspots
from artificial_neural_networks.code.utils.generic_utils import save_regress_model, \
series_to_supervised, affine_transformation
from artificial_neural_networks.code.utils.vis_utils import regression_figs
# %%
if args.verbose > 0:
print(args)
# For reproducibility
if args.reproducible:
os.environ['PYTHONHASHSEED'] = '0'
np.random.seed(args.seed)
rn.seed(args.seed)
tf.set_random_seed(args.seed)
sess = tf.Session(graph=tf.get_default_graph())
K.set_session(sess)
# print(hash("keras"))
# %%
# Load the Monthly sunspots dataset
sunspots_path = download_monthly_sunspots()
sunspots = np.genfromtxt(fname=sunspots_path,
dtype=np.float32,
delimiter=",",
skip_header=1,
usecols=1)
# %%
# Train-Test split
L_series = len(sunspots)
split_ratio = 2 / 3 # between zero and one
n_split = int(L_series * split_ratio)
look_back = args.look_back
steps_ahead = args.steps_ahead
train = sunspots[:n_split + (steps_ahead - 1)]
test = sunspots[n_split - look_back:]
train_x, train_y = series_to_supervised(train, look_back, steps_ahead)
test_x, test_y = series_to_supervised(test, look_back, steps_ahead)
train_y_series = train[look_back:train.shape[0] - (steps_ahead - 1)]
test_y_series = test[look_back:]
# %%
# PREPROCESSING STEP
scaling_factor = args.scaling_factor
translation = args.translation
n_train = train_x.shape[0] # number of training examples/samples
n_test = test_x.shape[0] # number of test examples/samples
n_in = train_x.shape[1] # number of features / dimensions
n_out = train_y.shape[1] # number of steps ahead to be predicted
# Reshape training and test sets
train_x = train_x.reshape(n_train, n_in, 1)
test_x = test_x.reshape(n_test, n_in, 1)
# Apply preprocessing
train_x_ = affine_transformation(train_x, scaling_factor, translation)
train_y_ = affine_transformation(train_y, scaling_factor, translation)
test_x_ = affine_transformation(test_x, scaling_factor, translation)
test_y_ = affine_transformation(test_y, scaling_factor, translation)
train_y_series_ = affine_transformation(train_y_series, scaling_factor,
translation)
test_y_series_ = affine_transformation(test_y_series, scaling_factor,
translation)
# %%
# Model hyperparameters and ANN Architecture
stateful = args.stateful
if stateful:
x = Input(shape=(n_in, 1), batch_shape=(1, n_in, 1)) # input layer
else:
x = Input(shape=(n_in, 1)) # input layer
h = x
h = LSTM(units=args.layer_size, stateful=stateful)(h) # hidden layer
out = Dense(units=n_out, activation=None)(h) # output layer
model = Model(inputs=x, outputs=out)
if args.verbose > 0:
model.summary()
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1))
loss_function = root_mean_squared_error
metrics = ['mean_absolute_error', 'mean_absolute_percentage_error']
lr = args.lrearning_rate
epsilon = args.epsilon
optimizer_selection = {
'Adadelta':
optimizers.Adadelta(lr=lr, rho=0.95, epsilon=epsilon, decay=0.0),
'Adagrad':
optimizers.Adagrad(lr=lr, epsilon=epsilon, decay=0.0),
'Adam':
optimizers.Adam(lr=lr,
beta_1=0.9,
beta_2=0.999,
epsilon=epsilon,
decay=0.0,
amsgrad=False),
'Adamax':
optimizers.Adamax(lr=lr,
beta_1=0.9,
beta_2=0.999,
epsilon=epsilon,
decay=0.0),
'Nadam':
optimizers.Nadam(lr=lr,
beta_1=0.9,
beta_2=0.999,
epsilon=epsilon,
schedule_decay=0.004),
'RMSprop':
optimizers.RMSprop(lr=lr, rho=0.9, epsilon=epsilon, decay=0.0),
'SGD':
optimizers.SGD(lr=lr, momentum=0.0, decay=0.0, nesterov=False)
}
optimizer = optimizer_selection[args.optimizer]
model.compile(optimizer=optimizer, loss=loss_function, metrics=metrics)
# %%
# Save trained models for every epoch
models_path = r'artificial_neural_networks/trained_models/'
model_name = 'sunspots_lstm_dense'
weights_path = models_path + model_name + '_weights'
model_path = models_path + model_name + '_model'
file_suffix = '_{epoch:04d}_{val_loss:.4f}_{val_mean_absolute_error:.4f}'
if args.save_weights_only:
file_path = weights_path
else:
file_path = model_path
file_path += file_suffix
monitor = 'val_loss'
if args.save_models:
checkpoint = ModelCheckpoint(file_path + '.h5',
monitor=monitor,
verbose=args.verbose,
save_best_only=args.save_best,
mode='auto',
save_weights_only=args.save_weights_only)
callbacks = [checkpoint]
else:
callbacks = []
# %%
# TRAINING PHASE
"""
if stateful:
shuffle = False
else:
shuffle = True
"""
if args.time_training:
start = timer()
for i in range(0, args.n_epochs):
if args.verbose > 0:
print('Epoch: {0}/{1}'.format(i + 1, args.n_epochs))
model.fit(x=train_x_,
y=train_y_,
validation_data=(test_x_, test_y_),
batch_size=args.batch_size,
epochs=1,
verbose=args.verbose,
callbacks=callbacks,
shuffle=True)
if stateful:
model.reset_states()
if args.time_training:
end = timer()
duration = end - start
print('Total time for training (in seconds):')
print(duration)
# %%
def model_predict(x_, y_):
"""
Predict using the LSTM Model (Multi-step ahead Forecasting)
"""
n_y_ = y_.shape[0]
y_pred = np.zeros(n_y_)
if args.recursive: # Recursive Strategy # TODO
if args.verbose > 0:
print('Following Recursive Strategy ...')
n_x_ = x_.shape[0]
n_iter = int(np.floor(n_x_ / steps_ahead))
L_last_window = n_x_ % steps_ahead
first = 0
# Multi-step ahead Forecasting of all the full windows
for i in range(0, n_iter):
""" if args.verbose > 0:
print('Completed: {0}/{1}'.format(i + 1, n_iter + 1)) """
pred_start = i * steps_ahead
pred_end = pred_start + steps_ahead
# first time step of each window (no recursion possible)
j = pred_start
k = j - pred_start # (always zero and unused)
x_dyn = np.copy(x_[j:j + 1]) # use actual values only
y_dyn = model.predict(x_dyn)[:, first]
y_pred[j:j + 1] = y_dyn
# remaining time steps of each window (with recursion)
for j in range(pred_start + 1, pred_end):
k = j - pred_start
x_dyn = np.copy(x_[j:j +
1]) # use actual values (if possible)
x_start = np.max([0, look_back - k])
y_start = np.max([0, k - look_back]) + pred_start
# y_start = np.max([pred_start, j - look_back])
x_dyn[0, x_start:look_back,
0] = np.copy(y_pred[y_start:j]) # use pred. values
y_dyn = model.predict(x_dyn)[:, first]
# y_after = np.max([0, y_dyn]) + 0.015 * np.random.randn()
y_pred[j:j + 1] = np.max([0, y_dyn])
# y_pred[j:j + 1] = y_dyn
# Multi-step ahead Forecasting of the last window
if L_last_window > 0:
""" if args.verbose > 0:
print('Completed: {0}/{1}'.format(n_iter + 1, n_iter + 1)) """
pred_start = n_x_ - L_last_window
pred_end = n_y_
# first time step of the last window (no recursion possible)
j = pred_start
k = j - pred_start # (always zero and unused)
x_dyn = np.copy(x_[j:j + 1]) # use actual values only
y_dyn = model.predict(x_dyn)[:, first]
y_pred[j:j + 1] = y_dyn
# remaining time steps of the last window (with recursion)
for j in range(pred_start + 1, pred_end):
k = j - pred_start
x_dyn = np.roll(x_dyn, -1) # use act. values (if possible)
x_start = np.max([0, look_back - k])
y_start = np.max([0, k - look_back]) + pred_start
# y_start = np.max([pred_start, j - look_back])
x_dyn[0, x_start:look_back,
0] = np.copy(y_pred[y_start:j]) # use pred. values
y_dyn = model.predict(x_dyn)[:, first]
# y_after = np.max([0, y_dyn]) + 0.015 * np.random.randn()
y_pred[j:j + 1] = np.max([0, y_dyn])
# y_pred[j:j + 1] = y_dyn
"""
# One-step ahead Forecasting
n_x_ = x_.shape[0]
for i in range(0, n_x_):
x_dyn = x_[i:i+1]
y_dyn = model.predict(x_dyn)[0, 0]
y_pred[i] = y_dyn
for i in range(n_x_, n_y):
x_dyn[0, :, 0] = y_[i - look_back:i]
y_dyn = model.predict(x_dyn)[0, 0]
y_pred[i] = y_dyn
"""
else: # Multiple Ouptput Strategy # TODO
if args.verbose > 0:
print('Following Multiple Ouptput Strategy ...')
n_iter = int(np.floor(n_y_ / steps_ahead))
L_last_window = n_y_ % steps_ahead
y_dyn = x_[0, steps_ahead]
# Multi-step ahead Forecasting of all the full windows
for i in range(0, n_iter):
pred_start = i * steps_ahead
pred_end = pred_start + steps_ahead
x_dyn = x_[pred_start:pred_start + 1] # TODO
y_dyn = model.predict(x_dyn)[0]
y_pred[pred_start:pred_end] = y_dyn
# Multi-step ahead Forecasting of the last window
if L_last_window > 0:
pred_start = n_y_ - L_last_window
pred_end = n_y_
x_dyn[0, :, 0] = y_[pred_end - look_back:pred_end]
y_dyn = model.predict(x_dyn)[0]
y_pred[pred_start:pred_end] = y_dyn[:L_last_window]
return y_pred
# %%
# TESTING PHASE
# Predict preprocessed values
train_y_sum = [0]
test_y_sum = [0]
reps = 1
for i in range(reps):
train_y_pred_ = model_predict(train_x_, train_y_series_)
test_y_pred_ = model_predict(test_x_, test_y_series_)
train_y_sum = np.sum([train_y_sum, train_y_pred_], axis=0)
test_y_sum = np.sum([test_y_sum, test_y_pred_], axis=0)
train_y_pred_ = train_y_sum / reps
test_y_pred_ = test_y_sum / reps
# Remove preprocessing
train_y_pred = affine_transformation(train_y_pred_,
scaling_factor,
translation,
inverse=True)
test_y_pred = affine_transformation(test_y_pred_,
scaling_factor,
translation,
inverse=True)
train_rmse = sqrt(mean_squared_error(train_y_series, train_y_pred))
train_mae = mean_absolute_error(train_y_series, train_y_pred)
train_r2 = r2_score(train_y_series, train_y_pred)
test_rmse = sqrt(mean_squared_error(test_y_series, test_y_pred))
test_mae = mean_absolute_error(test_y_series, test_y_pred)
test_r2 = r2_score(test_y_series, test_y_pred)
if args.verbose > 0:
print('Train RMSE: %.4f ' % (train_rmse))
print('Train MAE: %.4f ' % (train_mae))
print('Train (1 - R_squared): %.4f ' % (1.0 - train_r2))
print('Train R_squared: %.4f ' % (train_r2))
print('')
print('Test RMSE: %.4f ' % (test_rmse))
print('Test MAE: %.4f ' % (test_mae))
print('Test (1 - R_squared): %.4f ' % (1.0 - test_r2))
print('Test R_squared: %.4f ' % (test_r2))
# %%
# Data Visualization
if args.plot:
regression_figs(train_y=train_y_series,
train_y_pred=train_y_pred,
test_y=test_y_series,
test_y_pred=test_y_pred)
# %%
# Save the architecture and the lastly trained model
save_regress_model(model, models_path, model_name, weights_path,
model_path, file_suffix, test_rmse, test_mae, args)
# %%
return model
def create_network_v8():
# Define the neural network architecture
model = kmodels.Sequential()
penalty = 0.0001
model.add(
klayers.Conv2D(12, (5, 5),
input_shape=input_shape,
activation='relu',
padding='same',
kernel_regularizer=kregularizers.l2(penalty)))
model.add(klayers.MaxPooling2D(pool_size=(2, 2)))
model.add(
klayers.Conv2D(24, (5, 5),
activation='relu',
padding='same',
kernel_regularizer=kregularizers.l2(penalty)))
model.add(klayers.MaxPooling2D(pool_size=(2, 2)))
model.add(
klayers.Conv2D(24, (3, 3),
activation='relu',
padding='same',
kernel_regularizer=kregularizers.l2(penalty)))
model.add(klayers.MaxPooling2D(pool_size=(2, 2)))
model.add(
klayers.Conv2D(24, (3, 3),
activation='relu',
padding='same',
kernel_regularizer=kregularizers.l2(penalty)))
model.add(klayers.MaxPooling2D(pool_size=(2, 2)))
model.add(
klayers.Conv2D(24, (3, 3),
activation='relu',
padding='same',
kernel_regularizer=kregularizers.l2(penalty)))
model.add(klayers.MaxPooling2D(pool_size=(2, 2)))
model.add(
klayers.Conv2D(24, (3, 3),
activation='relu',
kernel_regularizer=kregularizers.l2(penalty)))
model.add(klayers.MaxPooling2D(pool_size=(2, 2)))
#model.add(klayers.Conv2D(
# 5, (1, 1),
# activation='relu',
# kernel_regularizer=kregularizers.l2(0.01)))
#model.add(klayers.MaxPooling2D(pool_size=(2, 2)))
model.add(klayers.GlobalAveragePooling2D())
#model.add(klayers.Flatten())
model.add(klayers.Dropout(0.2))
model.add(
klayers.Dense(8,
activation='relu',
kernel_regularizer=kregularizers.l2(penalty),
bias_regularizer=kregularizers.l2(penalty)))
model.add(klayers.Dropout(0.1))
model.add(
klayers.Dense(1,
activation='sigmoid',
kernel_regularizer=kregularizers.l2(penalty),
bias_regularizer=kregularizers.l2(penalty)))
model.compile(
loss='binary_crossentropy',
optimizer=koptimizers.Adadelta(lr=0.7), #'adadelta',
metrics=['accuracy',
completeness_metric(0),
purity_metric(0)])
return model
class kerasDef:
config = Configuration()
config.readConfig(config.configFile)
runKeras = config.runKeras
alwaysRetrain = config.alwaysRetrainKeras
alwaysImprove = config.alwaysImproveKeras
# Format: [number_neurons_HL1, number_neurons_HL2, number_neurons_HL3,...]
hidden_layers = config.hidden_layersKeras
# Stock Optimizers: Adagrad (recommended), Adam, Ftrl, RMSProp, SGD
# https://www.tensorflow.org/api_guides/python/train
#optimizer = "Adagrad"
# Additional optimizers: ProximalAdagrad, AdamOpt, Adadelta,
# GradientDescent, ProximalGradientDescent,
# https://www.tensorflow.org/api_guides/python/train
optimizer = config.optimizerKeras
l2_reg_strength = config.l2_reg_strengthKeras
learning_rate = config.learning_rateKeras
learning_decay_rate = config.learning_decay_rateKeras
# activation functions: https://keras.io/activations/
# softmax, elu, relu, selu, softplus, softsign, tanh, sigmoid,
# hard_sigmoid, linear
activation_function = config.activation_functionKeras
# When not None, the probability of dropout.
dropout_perc = config.dropout_percKeras
trainingSteps = config.trainingStepsKeras # number of training steps
fullBatch = config.fullBatchKeras
batchSize = config.batchSizeKeras
# Setting them both to zero disables Histograms tracnking in tensorboard
tbHistogramFreq = config.tbHistogramFreqKeras
thresholdProbabilityPred = config.thresholdProbabilityPredKeras
plotModel = config.plotModelKeras
#*************************************************
# Setup variables and definitions- do not change.
#*************************************************
if runKeras == True:
import tensorflow as tf
import keras.optimizers as opt
from keras.layers import Activation
if optimizer == "SGD":
optimizer_tag = " SGD, learn_rate: "+str(learning_rate)
optimizer = opt.SGD(lr=learning_rate, decay=learning_decay_rate,
momentum=0.9, nesterov=True)
if optimizer == "Adagrad":
optimizer_tag = " Adagrad, learn_rate: "+str(learning_rate)
optimizer = opt.Adagrad(lr=learning_rate, epsilon=1e-08,
decay=learning_decay_rate)
if optimizer == "Adadelta":
optimizer_tag = " AdaDelta, learn_rate: "+str(learning_rate)
optimizer = opt.Adadelta(lr=learning_rate, epsilon=1e-08, rho=0.95,
decay=learning_decay_rate)
if optimizer == "Adam":
optimizer_tag = " Adam, learn_rate: "+str(learning_rate)
optimizer = opt.Adam(lr=learning_rate, beta_1=0.9,
beta_2=0.999, epsilon=1e-08,
decay=learning_decay_rate,
amsgrad=False)
if optimizer == "Adamax":
optimizer_tag = " Adamax, learn_rate: "+str(learning_rate)
optimizer = opt.Adamax(lr=learning_rate, beta_1=0.9,
beta_2=0.999, epsilon=1e-08,
decay=learning_decay_rate)
if optimizer == "RMSprop":
optimizer_tag = " RMSprop, learn_rate: "+str(learning_rate)
optimizer = opt.RMSprop(lr=learning_rate, rho=0.95,
epsilon=1e-08,
decay=learning_decay_rate)
'''
def create_adadelta(self):
if self.learning_rate is None: self.learning_rate = 1
self.momentum = None
return optimizers.Adadelta(lr=self.learning_rate, rho=0.95)
batch_size = 8 * np.random.randint(4, 10) # 64
number_of_epoch = 80
type = 2 # np.random.randint(1, 8)
optimizer_range = ['Adadelta', 'Adam', 'Adamax', 'Nadam']
optimizer_name = random.choice(optimizer_range)
dropout = np.random.rand(3) > 0.5
datageneration = 1
lr_reduce_factor = 1 / np.random.uniform(2, 10)
momentum = None
if (optimizer_name == 'Adadelta'):
learn_rate_range = [0.5, 1.5]
learn_rate = random.uniform(learn_rate_range[0], learn_rate_range[1])
optimizer = optimizers.Adadelta(lr=learn_rate)
elif (optimizer_name == 'Adam'):
learn_rate_range = [0.0005, 0.005]
learn_rate = random.uniform(learn_rate_range[0], learn_rate_range[1])
optimizer = optimizers.Adam(lr=learn_rate)
elif (optimizer_name == 'Adamax'):
learn_rate_range = [0.0005, 0.005]
learn_rate = random.uniform(learn_rate_range[0], learn_rate_range[1])
optimizer = optimizers.Adamax(lr=learn_rate)
elif (optimizer_name == 'Nadam'):
learn_rate_range = [0.0005, 0.005]
learn_rate = random.uniform(learn_rate_range[0], learn_rate_range[1])
optimizer = optimizers.Nadam(lr=learn_rate)
print 'Random parameters: batch=', batch_size, ' opt=', optimizer_name, ' learning rate=', learn_rate, ' dropout=', dropout, ' lr_reduce_factor=', lr_reduce_factor
elif algorithm == 'sgd':
optimizer = opt.SGD(lr=0.01,
momentum=0.0,
decay=0.0,
nesterov=False,
clipnorm=clipnorm,
clipvalue=clipvalue)
elif algorithm == 'adagrad':
optimizer = opt.Adagrad(lr=0.01,
epsilon=1e-06,
clipnorm=clipnorm,
clipvalue=clipvalue)
elif algorithm == 'adadelta':
optimizer = opt.Adadelta(lr=1.0,
rho=0.95,
epsilon=1e-06,
clipnorm=clipnorm,
clipvalue=clipvalue)
elif algorithm == 'adam':
optimizer = opt.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
clipnorm=clipnorm,
clipvalue=clipvalue)
elif algorithm == 'adamax':
optimizer = opt.Adamax(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
clipnorm=clipnorm,
def __call__(self): return optimizers.Adadelta()
def lstm_dense_sunspots(args):
"""
Main function
"""
# %%
# IMPORTS
# code repository sub-package imports
from artificial_neural_networks.code.utils.download_monthly_sunspots import \
download_monthly_sunspots
from artificial_neural_networks.code.utils.generic_utils import save_regress_model, \
series_to_supervised, affine_transformation
from artificial_neural_networks.code.utils.vis_utils import regression_figs
# %%
if args.verbose > 0:
print(args)
# For reproducibility
if args.reproducible:
os.environ['PYTHONHASHSEED'] = '0'
np.random.seed(args.seed)
rn.seed(args.seed)
tf.set_random_seed(args.seed)
sess = tf.Session(graph=tf.get_default_graph())
K.set_session(sess)
# print(hash("keras"))
# %%
# Load the Monthly sunspots dataset
sunspots_path = download_monthly_sunspots()
sunspots = np.genfromtxt(fname=sunspots_path,
dtype=np.float32,
delimiter=",",
skip_header=1,
usecols=1)
# %%
# Train-Test split
L_series = len(sunspots)
split_ratio = 2 / 3 # between zero and one
n_split = int(L_series * split_ratio)
look_back = args.look_back
train = sunspots[:n_split]
test = sunspots[n_split - look_back:]
train_x, train_y = series_to_supervised(train, look_back)
test_x, test_y = series_to_supervised(test, look_back)
# %%
# PREPROCESSING STEP
scaling_factor = args.scaling_factor
translation = args.translation
n_train = train_x.shape[0] # number of training examples/samples
n_test = test_x.shape[0] # number of test examples/samples
n_in = train_x.shape[1] # number of features / dimensions
n_out = train_y.shape[1] # number of steps ahead to be predicted
# Reshape training and test sets
train_x = train_x.reshape(n_train, n_in, 1)
test_x = test_x.reshape(n_test, n_in, 1)
# Apply preprocessing
train_x_ = affine_transformation(train_x, scaling_factor, translation)
train_y_ = affine_transformation(train_y, scaling_factor, translation)
test_x_ = affine_transformation(test_x, scaling_factor, translation)
test_y_ = affine_transformation(test_y, scaling_factor, translation)
# %%
# Model hyperparameters and ANN Architecture
x = Input(shape=(n_in, 1)) # input layer
h = x
h = LSTM(units=args.layer_size)(h) # hidden layer
out = Dense(units=n_out, activation=None)(h) # output layer
model = Model(inputs=x, outputs=out)
if args.verbose > 0:
model.summary()
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1))
loss_function = root_mean_squared_error
metrics = ['mean_absolute_error', 'mean_absolute_percentage_error']
lr = args.lrearning_rate
epsilon = args.epsilon
optimizer_selection = {
'Adadelta':
optimizers.Adadelta(lr=lr, rho=0.95, epsilon=epsilon, decay=0.0),
'Adagrad':
optimizers.Adagrad(lr=lr, epsilon=epsilon, decay=0.0),
'Adam':
optimizers.Adam(lr=lr,
beta_1=0.9,
beta_2=0.999,
epsilon=epsilon,
decay=0.0,
amsgrad=False),
'Adamax':
optimizers.Adamax(lr=lr,
beta_1=0.9,
beta_2=0.999,
epsilon=epsilon,
decay=0.0),
'Nadam':
optimizers.Nadam(lr=lr,
beta_1=0.9,
beta_2=0.999,
epsilon=epsilon,
schedule_decay=0.004),
'RMSprop':
optimizers.RMSprop(lr=lr, rho=0.9, epsilon=epsilon, decay=0.0),
'SGD':
optimizers.SGD(lr=lr, momentum=0.0, decay=0.0, nesterov=False)
}
optimizer = optimizer_selection[args.optimizer]
model.compile(optimizer=optimizer, loss=loss_function, metrics=metrics)
# %%
# Save trained models for every epoch
models_path = r'artificial_neural_networks/trained_models/'
model_name = 'sunspots_lstm_dense'
weights_path = models_path + model_name + '_weights'
model_path = models_path + model_name + '_model'
file_suffix = '_{epoch:04d}_{val_loss:.4f}_{val_mean_absolute_error:.4f}'
if args.save_weights_only:
file_path = weights_path
else:
file_path = model_path
file_path += file_suffix
monitor = 'val_loss'
if args.save_models:
checkpoint = ModelCheckpoint(file_path + '.h5',
monitor=monitor,
verbose=args.verbose,
save_best_only=args.save_best,
mode='auto',
save_weights_only=args.save_weights_only)
callbacks = [checkpoint]
else:
callbacks = []
# %%
# TRAINING PHASE
if args.time_training:
start = timer()
model.fit(x=train_x_,
y=train_y_,
validation_data=(test_x_, test_y_),
batch_size=args.batch_size,
epochs=args.n_epochs,
verbose=args.verbose,
callbacks=callbacks)
if args.time_training:
end = timer()
duration = end - start
print('Total time for training (in seconds):')
print(duration)
# %%
# TESTING PHASE
# Predict preprocessed values
train_y_pred_ = model.predict(train_x_)[:, 0]
test_y_pred_ = model.predict(test_x_)[:, 0]
# Remove preprocessing
train_y_pred = affine_transformation(train_y_pred_,
scaling_factor,
translation,
inverse=True)
test_y_pred = affine_transformation(test_y_pred_,
scaling_factor,
translation,
inverse=True)
train_rmse = sqrt(mean_squared_error(train_y, train_y_pred))
train_mae = mean_absolute_error(train_y, train_y_pred)
train_r2 = r2_score(train_y, train_y_pred)
test_rmse = sqrt(mean_squared_error(test_y, test_y_pred))
test_mae = mean_absolute_error(test_y, test_y_pred)
test_r2 = r2_score(test_y, test_y_pred)
if args.verbose > 0:
print('Train RMSE: %.4f ' % (train_rmse))
print('Train MAE: %.4f ' % (train_mae))
print('Train (1 - R_squared): %.4f ' % (1.0 - train_r2))
print('Train R_squared: %.4f ' % (train_r2))
print('')
print('Test RMSE: %.4f ' % (test_rmse))
print('Test MAE: %.4f ' % (test_mae))
print('Test (1 - R_squared): %.4f ' % (1.0 - test_r2))
print('Test R_squared: %.4f ' % (test_r2))
# %%
# Data Visualization
if args.plot:
regression_figs(train_y=train_y,
train_y_pred=train_y_pred,
test_y=test_y,
test_y_pred=test_y_pred)
# %%
# Save the architecture and the lastly trained model
save_regress_model(model, models_path, model_name, weights_path,
model_path, file_suffix, test_rmse, test_mae, args)
# %%
return model
activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))
# Horovod: adjust learning rate based on number of GPUs.
opt = optimizers.Adadelta(1.0 * hvd.size())
# Horovod: add Horovod Distributed Optimizer.
opt = hvd.DistributedOptimizer(opt)
model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])
# Horovod: broadcast initial variable states from rank 0 to all other processes.
# This is necessary to ensure consistent initialization of all processes when
# training is started with random weights or restored from a checkpoint.
callbacks.append(hvd.callbacks.BroadcastGlobalVariablesCallback(0))
# Horovod: print model summary only on rank 0.
if hvd.rank() == 0:
print(model.summary())
# set the first 10 layers to non-trainable
for layer in model.layers[:10]:
layer.trainable = False
# build custom model
reg = Flatten()(model.output)
reg = Dense(1024, activation='relu')(reg)
reg = Dropout(0.5)(reg)
reg = Dense(3, activation='linear')(reg)
# combine into tuned model
tuned_model = Model(input=model.input, output=reg)
tuned_model.compile(
loss='mse',
metrics=['mse'],
optimizer=optimizers.Adadelta(lr=0.1),
)
# prepare train data augmentation configuration
train_datagen = ImageDataGenerator(**IMG_AUG)
test_datagen = ImageDataGenerator(rescale=IMG_AUG['rescale'])
# read training y
labels = pd.read_csv('labels.csv').set_index('id')
y = labels.values
# read training X
train_palm_fname = ['data/ %d.jpg' % fname for fname in labels.index.tolist()]
X = np.zeros((len(train_palm_fname), IMG_WIDTH, IMG_HEIGHT, 3))
for idx, fname in enumerate(train_palm_fname):
X[idx, :, :, :] = np.array(
)
ret_array = np.concatenate([leftover_array, new_array])
new_batch = True
left_over_array = np.zeros(
shape=[0, image_height, image_width])
assert (ret_array.shape[0] == batch_size)
num_batches_generated += 1
ret_array = np.reshape(ret_array,
(len(ret_array), 1, image_height, image_width))
yield ret_array
if __name__ == '__main__':
my_opt = optimizers.Adadelta()
#get the data
try:
data_folder = os.environ["DATA"]
except KeyError:
"Please cd into the module's base folder and run set_env from there."
file_list = os.listdir(data_folder)
train_data_list = file_list[
0:
47] #choosing 63 here keeps ~80% of data for testing, rest for training and val, need to automatize this
val_data_list = file_list[
47:
63] #choosing 63 here keeps ~80% of data for testing, rest for training and val, need to automatize this
test_data_list = file_list[63:]
def test_adadelta():
_test_optimizer(optimizers.Adadelta(), target=0.6)
_test_optimizer(optimizers.Adadelta(decay=1e-3), target=0.6)
return cce
#nitialize the Reward predictor model
Qmodel = Sequential()
#model.add(Dense(num_env_variables+num_env_actions, activation='tanh', input_dim=dataX.shape[1]))
Qmodel.add(Dense(2048, activation='relu', input_dim=dataX.shape[1]))
#Qmodel.add(Dropout(0.2))
#Qmodel.add(Dense(256, activation='relu'))
#Qmodel.add(Dropout(0.5))
#Qmodel.add(Dense(128, activation='relu'))
#Qmodel.add(Dropout(0.5))
Qmodel.add(Dense(dataY.shape[1]))
#opt = optimizers.adadelta(lr=learning_rate)
opt = optimizers.Adadelta()
Qmodel.compile(loss='mse', optimizer=opt, metrics=['accuracy'])
#initialize the action predictor model
action_predictor_model = Sequential()
#model.add(Dense(num_env_variables+num_env_actions, activation='tanh', input_dim=dataX.shape[1]))
action_predictor_model.add(
Dense(2048, activation='relu', input_dim=apdataX.shape[1]))
#action_predictor_model.add(Dropout(0.5))
#action_predictor_model.add(Dense(256, activation='relu'))
#action_predictor_model.add(Dropout(0.5))
#action_predictor_model.add(Dense(128, activation='relu'))
#action_predictor_model.add(Dropout(0.5))
action_predictor_model.add(Dense(apdataY.shape[1]))
def fine_tune(train_data, train_labels, sx, sy, max_index, epochs, batch_size,
input_folder, result_path):
print(train_data.shape, train_labels.shape)
#load full model with imagenet weights
model = applications.InceptionV3(weights='imagenet',
include_top=False,
input_shape=(sx, sy, 3))
print('Model loaded')
#create top model
top_model = Sequential()
top_model.add(Flatten(input_shape=model.output_shape[1:]))
top_model.add(Dropout(0.3))
top_model.add(Dense(256, activation='relu'))
top_model.add(Dropout(0.3))
top_model.add(Dense(52, activation='softmax'))
#load top model weights
top_model.load_weights(result_path + 'bottleneck_fc_model.h5')
#join models
new_model = Sequential()
new_model.add(model)
new_model.add(top_model)
#optimizer settings
lr = 0.0001
decay = 1e-6
momentum = 0.9
#optimizer = optimizers.SGD(lr=lr, decay=decay, momentum=momentum, nesterov=True)
optimizer = optimizers.Adadelta()
loss = 'categorical_crossentropy'
new_model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])
fineture_log = result_path + 'training_' + str(
max_index) + '_finetune_log.csv'
csv_logger_finetune = callbacks.CSVLogger(fineture_log)
earlystop = EarlyStopping(monitor='val_acc',
min_delta=0.0001,
patience=5,
verbose=1,
mode='auto')
#create data generators for training model
datagen = ImageDataGenerator(featurewise_center=True,
rotation_range=90,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.1,
horizontal_flip=1,
vertical_flip=1,
shear_range=0.05)
datagen.fit(train_data)
generator = datagen.flow(train_data,
train_labels,
batch_size=batch_size,
shuffle=True)
validation_generator = datagen.flow(train_data,
train_labels,
batch_size=batch_size,
shuffle=True)
new_model.fit_generator(generator,
epochs=epochs,
steps_per_epoch=len(train_data) // batch_size,
validation_data=validation_generator,
validation_steps=len(train_data) // batch_size //
5,
callbacks=[csv_logger_finetune, earlystop],
verbose=2)
with open(fineture_log, 'a') as log:
log.write('\n')
log.write('input images: ' + input_folder + '\n')
log.write('batch_size:' + str(batch_size) + '\n')
log.write('learning rate: ' + str(lr) + '\n')
log.write('learning rate decay: ' + str(decay) + '\n')
log.write('momentum: ' + str(momentum) + '\n')
log.write('loss: ' + loss + '\n')
new_model.save(result_path + 'FinalModel.h5'
) # save the final model for future loading and prediction
else:
print('new model')
layers = []
layers.append(
Dense(1, input_dim=hogmat_size, activation='relu', use_bias=True))
layers.append(
Dense(1, activation='sigmoid',
kernel_initializer='random_uniform'))
#layers.append(Dense( 1, activation='softmax' ) )
#layers.append( Dropout(.5 , noise_shape=None, seed=None))
model = Sequential(layers)
#sgd = optimizers.SGD(lr= 1, momentum=0.1, decay=0.01, nesterov=False)
#sgd = optimizers.Adagrad(lr=0.01, epsilon=.01, decay=0.01)
sgd = optimizers.Adadelta(lr=.10, rho=0.095, epsilon=None, decay=0.0)
#sgd = optimizers.RMSprop(lr=1, rho=0.9, epsilon=None, decay=0.0)
model.compile(loss='binary_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
tstart = t.time()
print('loading dataset')
with open(traintest, 'rb') as traintestin:
xtotalmat, ytotalmat, xtesttotalmat, ytesttotalmat = pickle.loads(
traintestin.read())
print('done')
print('training')
metrics = model.fit(x=xtotalmat,
y=ytotalmat,
batch_size=500,
def testKeras(examples, labels, subsetPercent = 0.2, desiredError = 0.001, timeLimit = 30):
# Test each algorithm on a smaller dataset.
exampleSubset = examples[0:int(len(examples)*subsetPercent)]
labelSubset = labels[0:int(len(labels)*subsetPercent)]
max_iterations = 10000
estimatedIters = []
allResults = []
for i in range(7):
plt.figure(i+1)
# Create Model for Keras
model = Sequential()
model.add(Dense(units=1, activation='linear', input_dim=featureSize))
# Choose GD Algorithm for Keras
if (i == 0):
myOpt = optimizers.SGD(lr=0.01, momentum=0., decay=0., nesterov=False)
plt.title("SGD")
elif (i == 1):
myOpt = optimizers.SGD(lr=0.01, momentum=0.9, decay=0., nesterov=False)
plt.title("Momentum")
elif (i == 2):
myOpt = optimizers.SGD(lr=0.01, momentum=0.9, decay=0., nesterov=True)
plt.title("Nesterov-Momentum")
elif (i == 3):
myOpt = optimizers.Adagrad(lr=0.01, epsilon=1e-6)
plt.title("Adagrad")
elif (i == 4):
myOpt = optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=1e-6)
plt.title("Adadelta")
elif (i == 5):
myOpt = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-6)
plt.title("RMSprop")
elif (i == 6):
myOpt = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8)
plt.title("Adam")
model.compile(optimizer=myOpt, loss=logloss)
# Create Custom Callback. Run GD. History saved in output. Use it to find the changes in loss per iteration
customCallback = EarlyStoppingByDeltaLossOrTime(desiredError, timeLimit)
myCallbacks = [customCallback]
output = model.fit(exampleSubset, labelSubset, epochs=max_iterations, batch_size=int(len(exampleSubset)/50), callbacks=myCallbacks)
losses = np.array(output.history['loss'])
deltaLosses = -np.diff(losses)
# Run again on the full dataset, for a few iterations. Use this to find the average time per iteration.
# Reset callback to reset time elapsed and history of losses.
model = Sequential()
model.add(Dense(units=1, activation='linear', input_dim=featureSize))
model.compile(optimizer=myOpt, loss=logloss)
customCallback = EarlyStoppingByDeltaLossOrTime(desiredError, timeLimit)
myCallbacks = [customCallback]
output = model.fit(examples, labels, epochs=5, batch_size=int(len(examples)/50), callbacks=myCallbacks)
losses = np.array(output.history['loss'])
timePerIter = myCallbacks[0].timeElapsed/len(losses)
# Pass in the following:
# 1. Array of DeltaLosses, iterations is length of array.
# 2. Average Time per Iteration on the full dataset.
results = fitGD(deltaLosses, timePerIter, desiredError)
estimatedIters.append(results[0])
print("ETI: ", results[0])
print("ETA: ", results[1])
allResults.append(results)
for i in range(len(allResults)):
print("Algo", i, "Iterations:", allResults[i][0], "ETA:", allResults[i][1])
plt.show()
return estimatedIters
