model_pre.load_weights('/home/ucl/cp3/fbury/storage/MoMEMtaModelNN/4lay50_inv_trainmass/DYweight.h5') # load weights
print ('[INFO] Loading pre-stage DNN weights : Success')
except:
sys.exit('Could not find pre-stage DNN model')
DNN.get_layer("L1").set_weights(model_pre.get_layer('dense_1').get_weights())
DNN.get_layer("L2").set_weights(model_pre.get_layer('dense_2').get_weights())
DNN.get_layer("L3").set_weights(model_pre.get_layer('dense_3').get_weights())
DNN.get_layer("Mjj").set_weights(model_pre.get_layer('dense_4').get_weights())
DNN.get_layer("Mll").set_weights(model_pre.get_layer('dense_5').get_weights())
DNN.get_layer("Mlljj").set_weights(model_pre.get_layer('dense_6').get_weights())
print (model_pre.get_config())
# Callbacks #
early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=20, verbose=1, mode='min')
csv_logger = CSVLogger(path_out+'training.log')
reduceLR = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5, verbose=1, mode='min', epsilon=0.0001, cooldown=0, min_lr=0.0001)
checkpoint = ModelCheckpoint(path_out+args.label+'weight.h5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True, mode='min', period=1)
Callback_list = [csv_logger,checkpoint,early_stopping,reduceLR]
# Fit #
epoch = 200
batch = 1000
DNN.fit(train_inputs,{'Mlljj':train_targets[:,0], 'Mjj':train_targets[:,1], 'Mll':train_targets[:,2]}, sample_weight={'Mlljj':train_weight, 'Mjj':train_weight, 'Mll':train_weight}, epochs=epoch, batch_size=batch, verbose=2, validation_data=(test_inputs,[test_targets[:,0], test_targets[:,1], test_targets[:,2]],[test_weight, test_weight, test_weight]), callbacks=Callback_list)
# Save model #
model_json = DNN.to_json()
with open(path_out+args.label+'model.json', "w") as json_file:
json_file.write(model_json)
model = Sequential()
model.add(Embedding(max_features, embedding_vecor_length,
input_length=max_len))
model.add(LSTM(128))
model.add(Dropout(0.1))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
checkpointer = callbacks.ModelCheckpoint(
filepath="logs/lstm/checkpoint-{epoch:02d}.hdf5",
verbose=1,
save_best_only=True,
monitor='val_acc',
mode='max')
csv_logger = CSVLogger('logs/lstm/training_set_lstmanalysis.csv',
separator=',',
append=False)
model.fit(X_train,
y_train,
batch_size=32,
nb_epoch=1000,
validation_split=0.33,
shuffle=True,
callbacks=[checkpointer, csv_logger])
score, acc = model.evaluate(X_test, y_test, batch_size=32)
print('Test score:', score)
print('Test accuracy:', acc)
finetune_model.summary()
num_train_images = utils.get_num_files(TRAIN_DIR)
num_val_images = utils.get_num_files(VAL_DIR)
def lr_decay(epoch):
if epoch % 20 == 0 and epoch != 0:
lr = K.get_value(model.optimizer.lr)
K.set_value(model.optimizer.lr, lr / 2)
print("LR changed to {}".format(lr / 2))
return K.get_value(model.optimizer.lr)
learning_rate_schedule = LearningRateScheduler(lr_decay)
# setup checkpoints
csv_logger = CSVLogger(os.path.join(OUT_DIR, "log.csv"), append=True, separator=";")
latest_filepath = os.path.join(OUT_DIR, args.model + "_model_latest.h5")
latest_checkpoint = ModelCheckpoint(latest_filepath, monitor="accuracy", verbose=1)
best_filepath = os.path.join(OUT_DIR, args.model + "_model_best.h5")
best_checkpoint = ModelCheckpoint(
best_filepath,
monitor="val_accuracy",
verbose=1,
save_best_only=True,
mode="max",
)
change_transfer_strategy = FixedThenFinetune(args.from_epoch + args.num_epochs)
def train_model(model: Model,
dataset_id,
method,
proto_num,
dataset_prefix,
nb_iterations=100000,
batch_size=128,
val_subset=None,
cutoff=None,
normalize_timeseries=False,
opt='Adam',
learning_rate=1e-3,
early_stop=False,
balance_classes=True):
X_train1, X_train2, y_train, X_test1, X_test2, y_test, is_timeseries = load_dataset(
dataset_id,
method,
proto_num,
normalize_timeseries=normalize_timeseries)
#calculate num of batches
nb_epochs = math.ceil(nb_iterations * (batch_size / X_train1.shape[0]))
if balance_classes == True:
classes = np.arange(0, nb_classes(dataset_id))
le = LabelEncoder()
y_ind = le.fit_transform(y_train.ravel())
recip_freq = len(y_train) / (len(le.classes_) *
np.bincount(y_ind).astype(np.float64))
class_weight = recip_freq[le.transform(classes)]
print("Class weights : ", class_weight)
y_train = to_categorical(y_train, nb_classes(dataset_id))
y_test = to_categorical(y_test, nb_classes(dataset_id))
reduce_lr = ReduceLROnPlateau(monitor='loss',
patience=math.ceil(nb_epochs / 20),
mode='auto',
factor=1 / 2.,
cooldown=0,
min_lr=learning_rate / 10.,
verbose=2)
model_checkpoint = ModelCheckpoint(
"./weights/%s_%s_%s_weights.h5" %
(dataset_prefix, method, str(proto_num)),
verbose=2,
monitor='loss',
save_best_only=True,
save_weights_only=True)
tensorboard = TensorBoard(log_dir='./logs/%s_%s_%s' %
(dataset_prefix, method, str(proto_num)),
batch_size=batch_size)
csv_logger = CSVLogger('./logs/%s_%s_%s.csv' %
(dataset_prefix, method, str(proto_num)))
if early_stop:
early_stopping = EarlyStopping(monitor='loss',
patience=500,
mode='auto',
verbose=2,
restore_best_weights=True)
callback_list = [
model_checkpoint, early_stopping, tensorboard, csv_logger
]
else:
callback_list = [model_checkpoint, tensorboard, csv_logger]
if opt == 'SGD':
optm = SGD(lr=learning_rate, momentum=0.9, decay=5e-4)
elif opt == 'Nadam':
optm = Nadam(lr=learning_rate)
elif opt == 'Adam_decay':
optm = Adam(lr=learning_rate, decay=9. / nb_iterations)
else:
optm = Adam(lr=learning_rate)
model.compile(optimizer=optm,
loss='categorical_crossentropy',
metrics=['accuracy'])
if balance_classes:
model.fit([X_train1, X_train2],
y_train,
batch_size=batch_size,
epochs=nb_epochs,
callbacks=callback_list,
class_weight=class_weight,
verbose=2,
validation_data=([X_test1, X_test2], y_test))
else:
model.fit([X_train1, X_train2],
y_train,
batch_size=batch_size,
epochs=nb_epochs,
callbacks=callback_list,
verbose=2,
validation_data=([X_test1, X_test2], y_test))
def main(run_dir, data_dir, nb_epoch, early_stopping_patience,
desired_sample_rate, fragment_length, batch_size, fragment_stride,
nb_output_bins, keras_verbose, _log, seed, _config, debug,
learn_all_outputs, train_only_in_receptive_field, _run, use_ulaw):
if run_dir is None:
run_dir = os.path.join(
'models',
datetime.datetime.now().strftime('run_%Y-%m-%d_%H:%M:%S'))
_config['run_dir'] = run_dir
print_config(_run)
_log.info('Running with seed %d' % seed)
if not debug:
if os.path.exists(run_dir):
raise EnvironmentError('Run with seed %d already exists' % seed)
os.mkdir(run_dir)
checkpoint_dir = os.path.join(run_dir, 'checkpoints')
json.dump(_config, open(os.path.join(run_dir, 'config.json'), 'w'))
_log.info('Loading data...')
data_generators, nb_examples = dataset.generators(
data_dir, desired_sample_rate, fragment_length, batch_size,
fragment_stride, nb_output_bins, learn_all_outputs, use_ulaw)
_log.info('Building model...')
model = build_model(fragment_length)
_log.info(model.summary())
optim = make_optimizer()
_log.info('Compiling Model...')
loss = objectives.categorical_crossentropy
all_metrics = [
metrics.categorical_accuracy, metrics.categorical_mean_squared_error
]
if train_only_in_receptive_field:
loss = skip_out_of_receptive_field(loss)
all_metrics = [skip_out_of_receptive_field(m) for m in all_metrics]
model.compile(optimizer=optim, loss=loss, metrics=all_metrics)
# TODO: Consider gradient weighting making last outputs more important.
callbacks = [
ReduceLROnPlateau(patience=early_stopping_patience / 2,
cooldown=early_stopping_patience / 4,
verbose=1),
EarlyStopping(patience=early_stopping_patience, verbose=1),
]
if not debug:
callbacks.extend([
ModelCheckpoint(os.path.join(
checkpoint_dir, 'checkpoint.{epoch:05d}-{val_loss:.3f}.hdf5'),
save_best_only=True),
CSVLogger(os.path.join(run_dir, 'history.csv')),
])
if not debug:
os.mkdir(checkpoint_dir)
_log.info('Starting Training...')
model.fit_generator(data_generators['train'],
nb_examples['train'],
nb_epoch=nb_epoch,
validation_data=data_generators['test'],
nb_val_samples=nb_examples['test'],
callbacks=callbacks,
verbose=keras_verbose)
# 1. define the network
model = Sequential()
model.add(Dense(1024, input_dim=41, activation='relu'))
model.add(Dropout(0.01))
model.add(Dense(768, activation='relu'))
model.add(Dropout(0.01))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.01))
model.add(Dense(1))
model.add(Activation('sigmoid'))
# try using different optimizers and different optimizer configs
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
checkpointer = callbacks.ModelCheckpoint(
filepath="kddresults/dnn3layer/checkpoint-{epoch:02d}.hdf5",
verbose=1,
save_best_only=True,
monitor='loss')
csv_logger = CSVLogger('kddresults/dnn3layer_csv/training_set_dnnanalysis.csv',
separator=',',
append=False)
model.fit(X_train,
y_train,
batch_size=batch_size,
nb_epoch=100,
callbacks=[checkpointer, csv_logger])
model.save("kddresults/dnn3layer/dnn3layer_model.hdf5")
model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['acc'])
# Fit Network
## Including fooling in naming
if args.fooling == 1:
fooling = "fooling"
else:
fooling = "no-fooling"
## Implement callbacks
#### CSVLogger
date = str(datetime.datetime.now().year) + "-" + str(
datetime.datetime.now().month) + "-" + str(datetime.datetime.now().day)
csv_name = date + "_{}-{}_{}_ratio-{}_{}".format(
args.src_lang, args.tgt_lang, args.optimizer, args.ratio, fooling)
csv_path = os.path.join("./training_logs/csv/", csv_name)
logger = CSVLogger(filename=csv_path)
#### Model Checkpoint
cp_path = "./models/monitoring/" + date + "_{}-{}_{}_ratio-{}_{}_cp".format(
args.src_lang, args.tgt_lang, args.optimizer, args.ratio, fooling)
checkpoint = ModelCheckpoint(filepath=cp_path,
monitor="val_loss",
save_best_only=True,
save_weights_only=False,
verbose=1)
callbacks_list = [logger, checkpoint]
if args.earlystopping:
callbacks_list.append(
EarlyStopping(patience=args.earlystopping_patience,
monitor="val_loss"))
if n_gpu > 1:
with tf.device('/cpu:0'):
model = Net5(im_size,1)
model.load_weights('model_128.h5')
parallel_model = multi_gpu_model(model, gpus=n_gpu)
else:
model = Net5(im_size, 1)
model.load_weights('model_128.h5')
print(model.summary())
parallel_model = model
modelcheck = ModelCheckpoint('model_256_best.h5', monitor='val_loss',
save_best_only=True)
lr_scheduler = LearningRateScheduler(scheduler256)
csv = CSVLogger('mode256.csv')
parallel_model.compile(optimizer='sgd',loss=loss,
metrics=['acc'])
parallel_model.fit_generator(
train_generator,
steps_per_epoch=nb_train_samples // batch_size,
epochs=epochs,
validation_data=test_generator,
validation_steps=nb_test_samples // batch_size,
class_weight= class_weight,
callbacks=[modelcheck,lr_scheduler,csv],
use_multiprocessing=True,
workers=cpus)
model.save_weights('model_256.h5')
def training_code():
#titanic-complete3.arff är som det vi fick men:
#har attributen normaliserade mellan 0 till 1
#Filter:ordinal to numeric,
#cabin och embarked attributet är borttagna
#Gör om arff filen till numpy arrays.
raw_data = loadarff('titanic-complete3.arff')
df_data = pd.DataFrame(raw_data[0])
arr = df_data.to_numpy()
#Survival attributet blev en string, fixas här
numbers = []
x = 0
for i in range(arr.size//6):
for word in arr[x,5].split():
if word.isdigit():
numbers.append(int(word))
x += 1
y=0
for i in range(len(numbers)):
arr[y,5]= numbers[y]
y+=1
arr = arr.astype("float32")
# X_input är de 5 attributen
# Y_output är de respektive matchande klasstillhörighetern
X_input = arr[:,0:5]
Y_output = arr[:,5]
num_folds = 10
acc_per_fold = []
loss_per_fold = []
# initialiserar vikterna
np.random.seed(42)
skf = StratifiedKFold(n_splits=num_folds, shuffle=True)
fold_no = 1
visualo = []
visuaacc = []
num_of_epochs = 2000
#genomför cross validation och tränar modellen
for train_index, test_index in skf.split(X_input, Y_output):
# One hot encoding
X_train, X_test = X_input[train_index], X_input[test_index]
Y_train, Y_test = np_utils.to_categorical(Y_output[train_index], 2), np_utils.to_categorical(Y_output[test_index], 2)
#bygger min mlp
model = Sequential()
model.add(Dense(4,input_dim = (5), activation="relu"))
model.add(Dense(2, activation="sigmoid"))
#Adam är en form av SGD
model.compile(optimizer="Adam", loss="binary_crossentropy", metrics = ["accuracy"])
print('------------------------------------------------------------------------')
print(f'Training for fold {fold_no} ...')
csv_logger = CSVLogger("training_mlp.log", append=True, separator=";")
history = model.fit(x=X_train, y=Y_train, epochs=num_of_epochs, callbacks=[csv_logger], verbose=0)
model.save("Titanic_mlp.h5")
scores = model.evaluate(X_test, Y_test, verbose=0)
print(f'Score for fold {fold_no}: {model.metrics_names[0]} of {scores[0]}; {model.metrics_names[1]} of {scores[1]*100}%')
acc_per_fold.append(scores[1] * 100)
loss_per_fold.append(scores[0])
fold_no += 1
visualo.append(history.history["loss"])
visuaacc.append(history.history["accuracy"])
print('------------------------------------------------------------------------')
print('Score per fold')
for i in range(0, len(acc_per_fold)):
print('------------------------------------------------------------------------')
print(f'> Fold {i+1} - Loss: {loss_per_fold[i]} - Accuracy: {acc_per_fold[i]}%')
print('------------------------------------------------------------------------')
print('Average scores for all folds:')
print(f'> Accuracy: {np.mean(acc_per_fold)} (+- {np.std(acc_per_fold)})')
print(f'> Loss: {np.mean(loss_per_fold)}')
print('------------------------------------------------------------------------')
#plottar träningen med snittet för varje epok per k_fold
herek = np.asarray(visualo, dtype=np.float32)
loss_visu = np.sum(herek, axis=0)
plt.plot(loss_visu/num_folds, label='Binary Crossentropy (loss)')
herek2 = np.asarray(visuaacc, dtype=np.float32)
acc_visu2 = np.sum(herek2, axis=0)
plt.plot(acc_visu2/num_folds, label='accuracy/100')
plt.title('training progress for titanic_mlp')
plt.ylabel('value')
plt.xlabel('No. epoch')
plt.legend(loc="upper left")
plt.show()
def init_callbacks(self, n_epochs, n_batches, **kwargs):
from keras.callbacks import TerminateOnNaN, EarlyStopping, \
ReduceLROnPlateau, CSVLogger, TensorBoard
from validation_checkpoint import ValidationCheckpoint
from clr_callback import CyclicLR
print('Initializing model callbacks')
use_tensorboard = kwargs.pop('use_tensorboard', False)
val_monitor = kwargs.pop('monitor', 'val_loss')
callbacks = kwargs.pop('callbacks', [])
# strides to test/save model during training
test_period = kwargs.pop('test_period', 1)
save_period = kwargs.pop('save_period', 0) # 0 = disable
warmup_epoch = kwargs.pop('warmup_epoch', 3)
random_state = kwargs.pop('random_state', 42)
verbose = kwargs.pop('verbose', 1)
test_ids = kwargs.pop('test_ids', [])
step_lr = kwargs.pop('step_lr', None)
clr_mult = kwargs.pop('clr_mult',
4) # total epochs before CLR changes signs
# exit early if the last [stop_early] test scores are all worse than the best
early_stop = int(n_epochs * 0.2)
stop_early = kwargs.pop('stop_early', None) or early_stop
stop_delta = optparams['stop_delta']
save_preds = kwargs.pop('save_preds', True)
save_model = kwargs.pop('save_model', True)
model_dir = self.model_dir
initial_monitor = self.start_monitor
initial_epoch = self.start_epoch
# configure callbacks
val_mode = 'auto'
ctimestr = epoch2str(gettime())
train_logf = pathjoin(
model_dir, 'training_log_%s_pid%d.csv' % (ctimestr, self.pid))
# if pathexists(train_logf) and pathsize(train_logf) != 0:
# #ctimestr = epoch2str(gettime())
# #ctimestr = epoch2str(pathctime(train_logf))
# ctimestr = '1'
# logf_base,logf_ext = splitext(train_logf)
# old_logf = logf_base+'_'+ctimestr+logf_ext
# print('Backing up existing log file "%s" to "%s"'%(train_logf,old_logf))
# os.rename(train_logf,old_logf)
self.val_monitor = val_monitor
self.save_preds = save_preds
self.save_model = save_model
self.save_period = save_period
self.test_period = test_period
self.stop_early = stop_early
self.stop_delta = stop_delta
self.test_ids = test_ids
self.val_cb = ValidationCheckpoint(val_monitor=val_monitor,
save_best_preds=save_preds,
save_best_model=save_model,
model_dir=model_dir,
mode=val_mode,
pid=self.pid,
initial_monitor=initial_monitor,
initial_epoch=initial_epoch,
warmup_epoch=warmup_epoch,
save_period=save_period,
test_period=test_period,
test_ids=test_ids,
verbose=verbose)
#self.val_cb = ModelCheckpoint(model_iterf,monitor=val_monitor,mode=val_mode, period=save_epoch,
# save_best_only=True, save_weights_only=False,
# verbose=False)
step_lr = step_lr or int(n_batches * clr_mult)
self.lr_cb = CyclicLR(base_lr=optparams['lr_min'],
max_lr=optparams['lr_max'],
step_size=step_lr)
# else:
# step_lr = step_lr or min(100,int(n_epochs*0.01))
# self.lr_cb = ReduceLROnPlateau(monitor=val_monitor,
# mode=val_mode,
# patience=step_lr,
# min_lr=optparams['lr_min'],
# factor=optparams['reduce_lr'],
# epsilon=optparams['tol'],
# verbose=verbose)
self.es_cb = EarlyStopping(monitor=val_monitor,
mode=val_mode,
patience=stop_early,
min_delta=stop_delta,
verbose=verbose)
self.tn_cb = TerminateOnNaN()
self.cv_cb = CSVLogger(filename=train_logf, append=True)
self.callbacks = callbacks + [
self.val_cb, self.lr_cb, self.es_cb, self.tn_cb, self.cv_cb
]
if self.backend == 'tensorflow' and use_tensorboard:
tb_batch_size = 32
tb_histogram_freq = 1
tb_embeddings_freq = 0
tb_log_dir = pathjoin(model_dir, 'tb_logs_pid%d' % self.pid)
if not pathexists(tb_log_dir):
os.makedirs(tb_log_dir)
self.tb_cb = TensorBoard(log_dir=tb_log_dir,
histogram_freq=tb_histogram_freq,
batch_size=tb_batch_size,
write_graph=True,
write_grads=True,
write_images=True,
embeddings_freq=tb_embeddings_freq,
embeddings_layer_names=None,
embeddings_metadata=None)
self.callbacks.append(self.tb_cb)
elif self.backend != 'tensorflow' and use_tensorboard:
print('Cannot use tensorboard with backend "%s"' % self.backend)
use_tensorboard = False
print('Initialized %d callbacks:' % len(self.callbacks),
str(self.callbacks))
y_train = load_images(output_train)
y_valid = load_images(output_valid)
ch_p = ModelCheckpoint(
'/home/polina/1tb/ss_prediction/centroid_90/redone/models_steps/step3/model1/best_weights.h5',
save_best_only=True,
save_weights_only=True,
monitor='val_loss',
mode='min')
model.compile(loss=comb_loss,
optimizer=optimizers.Adagrad(lr=0.03),
metrics=[f_mera])
csv_logger = CSVLogger(
'/home/polina/1tb/ss_prediction/centroid_90/redone/models_steps/step3/model1/training.log'
)
model.fit(x_train,
y_train,
validation_data=(x_valid, y_valid),
batch_size=batch_size,
epochs=epochs,
verbose=2,
shuffle=True,
callbacks=[csv_logger, ch_p])
model.save_weights(
'/home/polina/1tb/ss_prediction/centroid_90/redone/models_steps/step3/model1/weights.h5'
)
sample_factor=sample_factor,
batch_size=batch_size,
dims=dims,
levels=train_levels)
sampleset_size_validn = math.ceil(
len(test_true_list) / sample_factor) + len(test_true_list)
steps_per_epoch_validn = math.ceil(sampleset_size_validn / batch_size)
checkpointer = ModelCheckpoint(
checkpoint_dir + date + '_weights_' + saves_name +
'_{epoch:02d}--{categorical_accuracy:.4f}--{val_loss:.4f}.hdf5',
monitor='categorical_accuracy',
save_weights_only=True,
save_best_only=True)
csvlogger = CSVLogger(log_dir + 'fit.log', append=True)
if load_weights != None:
model.load_weights(load_weights)
print('Loaded weights from {}'.format(load_weights))
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['categorical_accuracy'])
history = model.fit_generator(train_generator,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
verbose=1,
callbacks=[checkpointer, csvlogger],
validation_data=validn_generator,
validation_steps=steps_per_epoch_validn,
model.add(Dense(num_classes))
model.add(Activation('softmax'))
model.summary()
# Compile the model
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# Callbacks
callbacks = []
modelcheckpoint = ModelCheckpoint('dnn/weights.{epoch:03d}-{val_acc:.5f}.h5', monitor='val_acc', save_best_only=True)
callbacks.append(modelcheckpoint)
csv_logger = CSVLogger('dnn_log.csv', separator=',', append=False)
callbacks.append(csv_logger)
# Fit the model
if isValid:
model.fit_generator(train_gen.flow(X_train, Y_train, batch_size=batch_size),
steps_per_epoch=10*X_train.shape[0]//batch_size,
epochs=epochs,
callbacks=callbacks,
validation_data=(X_valid, Y_valid))
else:
model.fit_generator(train_gen.flow(X_train, Y_train, batch_size=batch_size),
steps_per_epoch=10*X_train.shape[0]//batch_size,
epochs=epochs,
callbacks=callbacks)
metrics=['accuracy'])
model.summary()
patience = 50
filepath = './ensemble/Model.{epoch:02d}-{val_acc:.4f}.hdf5'
log_file_path = './ensemble/training.log'
model_checkpoint = ModelCheckpoint(filepath,
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='auto')
early_stop = EarlyStopping('val_acc', patience=patience)
reduce_lr = ReduceLROnPlateau('val_loss',
factor=0.1,
patience=int(patience / 4),
verbose=1)
csv_logger = CSVLogger(log_file_path, append=False)
datagen = ImageDataGenerator(featurewise_center=False,
featurewise_std_normalization=False,
rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=.1,
horizontal_flip=True,
data_format='channels_last')
datagen.fit(Train_x)
model.fit_generator(
datagen.flow(Train_x, Train_y, batch_size=batch_size),
samples_per_epoch=Train_x.shape[0],
mms=mms)
ctc_x, ctc_y = next(train_batch)
for name, value in ctc_x.items():
print(name, value.shape)
for name, value in ctc_y.items():
print(name, value.shape)
checkpointer = ModelCheckpoint(model_path,
verbose=1,
save_best_only=False,
save_weights_only=True,
period=1)
csv_to_log = CSVLogger(join(path, "logger_0627.csv"))
lr_change = ReduceLROnPlateau(monitor="loss",
factor=0.5,
patience=1,
min_lr=0.000,
epsilon=0.1,
verbose=1)
tfboard = TensorBoard(log_dir=log_path)
if os.path.exists(log_path):
shutil.rmtree(log_path)
callback_list = [checkpointer, tfboard, lr_change]
history = model.fit_generator(train_batch,
steps_per_epoch=40,
epochs=100,
metrics_callback = keras_utils.ReadmissionMetrics(
train_data=train_raw,
val_data=val_raw,
target_repl=(args.target_repl_coef > 0),
batch_size=args.batch_size,
verbose=args.verbose)
# make sure save directory exists
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname)
saver = ModelCheckpoint(path, verbose=1, period=args.save_every)
if not os.path.exists('keras_logs'):
os.makedirs('keras_logs')
csv_logger = CSVLogger(os.path.join('keras_logs',
model.final_name + '.csv'),
append=True,
separator=';')
print("==> training")
model.fit(x=train_raw[0],
y=train_raw[1],
validation_data=val_raw,
nb_epoch=n_trained_chunks + args.epochs,
initial_epoch=n_trained_chunks,
callbacks=[metrics_callback, saver, csv_logger],
shuffle=True,
verbose=args.verbose,
batch_size=args.batch_size)
elif args.mode == 'test':
#classWeight = compute_class_weight(TRAINDATA)
trainGenerator = myGenerator(TRAINDATA, batchSize=BATCHSIZE, n_classes=N_CLASS)
testGenerator = myGenerator(TESTDATA,
shuffle=False,
batchSize=BATCHSIZE,
n_classes=N_CLASS)
#tensorBoard = TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=BATCHSIZE, write_graph=True,
# write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None,
# embeddings_metadata=None)
filepath = '../model/weigh-' + str(
BATCHSIZE) + '-resnet-noweighted-fc-{epoch:02d}-{val_acc:.2f}.hdf5'
checkpoint = ModelCheckpoint(filepath,
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='max')
csv_logger = CSVLogger('resnet_' + str(BATCHSIZE) + '_noweighted_fc.log')
classifier.fit_generator(trainGenerator,
samples_per_epoch=int(np.floor(TRAINSIZE /
BATCHSIZE)),
epochs=EPOCHS,
verbose=VERBOSE,
class_weight=None,
validation_data=testGenerator,
validation_steps=int(np.floor(TESTSIZE / BATCHSIZE)),
workers=1,
callbacks=[csv_logger, checkpoint])
model = Sequential()
model.add(Flatten())
model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dense(256))
model.add(Dropout(0.50))
model.add(Activation('relu'))
model.add(Dense(hasy_tools.n_classes, activation='softmax'))
# Compile model
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# Fit the model
csv_logger = CSVLogger('log.csv', append=True, separator=';')
checkpointer = ModelCheckpoint(filepath='checkpoint.h5',
verbose=1,
period=10,
save_best_only=True)
model.fit(data['x_train'],
data['y_train'],
validation_data=(data['x_val'], data['y_val']),
epochs=500,
batch_size=128,
callbacks=[csv_logger, checkpointer])
# Serialize model
model.save('model.h5')
# evaluate the model
def tune_model():
# Build the Inception V3 network.
base_model = inception_v3.InceptionV3(include_top=False,
weights='imagenet',
pooling='avg')
print('Model loaded.')
# build a classifier model to put on top of the convolutional model
top_input = Input(shape=base_model.output_shape[1:])
top_output = Dense(5, activation='softmax')(top_input)
top_model = Model(top_input, top_output)
# Note that it is necessary to start with a fully-trained classifier,
# including the top classifier, in order to successfully do fine-tuning.
top_model.load_weights(top_model_weights_path)
# add the model on top of the convolutional base
model = Model(inputs=base_model.inputs,
outputs=top_model(base_model.outputs))
# Set all layers up to 'mixed8' to non-trainable (weights will not be updated)
last_train_layer = model.get_layer(name='mixed8')
for layer in model.layers[:model.layers.index(last_train_layer)]:
layer.trainable = False
# Compile the model with a SGD/momentum optimizer and a very slow learning rate.
model.compile(loss='categorical_crossentropy',
optimizer=SGD(lr=1e-4, momentum=0.9),
metrics=['accuracy'])
# Prepare data augmentation configuration
train_datagen = ImageDataGenerator(
preprocessing_function=inception_v3.preprocess_input,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_datagen = ImageDataGenerator(
preprocessing_function=inception_v3.preprocess_input)
train_generator = train_datagen.flow_from_directory(
train_data_dir,
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode='categorical')
validation_generator = test_datagen.flow_from_directory(
validation_data_dir,
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode='categorical')
loss = model.evaluate_generator(validation_generator,
nb_validation_samples // batch_size)
print('Model validation performance before fine-tuning:', loss)
csv_logger = CSVLogger(output_dir + 'model_tuning.csv')
# fine-tune the model
model.fit_generator(train_generator,
steps_per_epoch=nb_train_samples // batch_size,
epochs=tune_epochs,
validation_data=validation_generator,
validation_steps=nb_validation_samples // batch_size,
workers=4,
callbacks=[csv_logger])
model.save_weights(tuned_weights_path)
x, y = load_mnist()
elif args.dataset == 'usps':
x, y = load_usps('data/usps')
# define the model
model = CAE(input_shape=x.shape[1:], filters=[32, 64, 128, 10])
plot_model(model,
to_file=args.save_dir + '/%s-pretrain-model.png' % args.dataset,
show_shapes=True)
model.summary()
# compile the model and callbacks
optimizer = 'adam'
model.compile(optimizer=optimizer, loss='mse')
from keras.callbacks import CSVLogger
csv_logger = CSVLogger(args.save_dir +
'/%s-pretrain-log.csv' % args.dataset)
# begin training
t0 = time()
model.fit(x,
x,
batch_size=args.batch_size,
epochs=args.epochs,
callbacks=[csv_logger])
print('Training time: ', time() - t0)
model.save(args.save_dir + '/%s-pretrain-model-%d.h5' %
(args.dataset, args.epochs))
# extract features
feature_model = Model(inputs=model.input,
outputs=model.get_layer(name='embedding').output)
def main(encoder_name, resnet_layers, top_filters, reduction_layers, neurons, dropout, name):
mapping = {'DIPG': 0, 'MB': 1, 'EP': 2}
train = None
with open('train.txt', 'r') as fp:
train = [x.strip() for x in fp.readlines()]
test = None
with open('test.txt', 'r') as fp:
test = [x.strip() for x in fp.readlines()]
key = name
train_generator = DataGenerator(train, mapping, 17, 3, True, True)
test_generator = DataGenerator(test, mapping, len(test), 3, True, True)
K.clear_session()
model = None
if os.path.exists(os.path.join('models', '{}.h5'.format(key))):
model = load_model(os.path.join('models', '{}.h5'.format(key)))
print('Loading model from history...')
else:
model = classifier(encoder_name, resnet_layers, top_filters, reduction_layers, neurons, dropout, name)
model.compile('adadelta', 'categorical_crossentropy', [categorical_accuracy])
print('Created new model...')
initial_epoch = None
e = None
if os.path.exists('clf-epochs.json'):
e = json.load(open('clf-epochs.json'))
if key in e:
initial_epoch = e[key]['epochs']
else:
e[key] = {'trained': 'n'}
initial_epoch = -1
else:
e = {key: {'trained': 'n'}}
initial_epoch = -1
if not os.path.exists(os.path.join('logs', '{}_log'.format(key))):
os.mkdir(os.path.join('logs', '{}_log'.format(key)))
if e[key]['trained'] == 'n':
epoch_cbk = EpochCallback(key)
checkpoint = ModelCheckpoint(filepath=os.path.join('models', '{}.h5'.format(key)),
monitor='val_loss', verbose=1,
save_best_only=True, mode='min')
earlystop = EarlyStopping(monitor='val_loss',
patience=10,
mode='min')
tensorboard = TensorBoard(log_dir=os.path.join('logs', '{}_log'.format(key)),
batch_size=13, write_graph=False,
write_grads=False, write_images=True)
csvlogger = CSVLogger(filename=os.path.join('logs', '{}.csv'.format(key)))
history = model.fit_generator(generator=train_generator, epochs=100, verbose=1,
callbacks=[epoch_cbk, csvlogger, earlystop, checkpoint, tensorboard],
validation_data=test_generator,
use_multiprocessing=True, workers=4,
initial_epoch=initial_epoch + 1)
e = json.load(open('clf-epochs.json'))
e[key]['trained'] = 'y'
json.dump(e, open('clf-epochs.json', 'w'))
eval_results = model.evaluate_generator(generator=test_generator)
d = {}
if os.path.exists('clf-scores.json'):
d = json.load(open('clf-scores.json'))
d[key] = eval_results[1]
json.dump(d, open('clf-scores.json', 'w'))
def train(data_type,
seq_length,
model,
saved_model=None,
class_limit=None,
image_shape=None,
load_to_memory=False,
batch_size=32,
nb_epoch=100):
# Helper: Save the model.
checkpointer = ModelCheckpoint(
filepath=os.path.join('data', 'checkpoints', model + '-' + data_type + \
'.{epoch:03d}-{val_loss:.3f}.hdf5'),
verbose=1,
save_best_only=True)
# Helper: TensorBoard
tb = TensorBoard(log_dir=os.path.join('data', 'logs', model))
# Helper: Stop when we stop learning.
early_stopper = EarlyStopping(patience=10)
# Helper: Save results.
timestamp = time.time()
csv_logger = CSVLogger(os.path.join('data', 'logs', model + '-' + 'training-' + \
str(timestamp) + '.log'))
# Get the data and process it.
if image_shape is None:
data = DataSet(seq_length=seq_length, class_limit=class_limit)
else:
data = DataSet(seq_length=seq_length,
class_limit=class_limit,
image_shape=image_shape)
# Get samples per epoch.
# Multiply by 0.7 to attempt to guess how much of data.data is the train set.
steps_per_epoch = (len(data.data) * 0.7) // batch_size
if load_to_memory:
# Get data.
X, y = data.get_all_sequences_in_memory('train', data_type)
X_test, y_test = data.get_all_sequences_in_memory('test', data_type)
else:
# Get generators.
generator = data.frame_generator(batch_size, 'train', data_type)
val_generator = data.frame_generator(batch_size, 'test', data_type)
# Get the model.
rm = ResearchModels(len(data.classes), model, seq_length, saved_model)
# Fit!
if load_to_memory:
# Use standard fit.
rm.model.fit(X,
y,
batch_size=batch_size,
validation_data=(X_test, y_test),
verbose=1,
callbacks=[tb, early_stopper, csv_logger, checkpointer],
epochs=nb_epoch)
else:
# Use fit generator.
rm.model.fit_generator(
generator=generator,
steps_per_epoch=steps_per_epoch,
epochs=nb_epoch,
verbose=1,
callbacks=[tb, early_stopper, csv_logger, checkpointer],
validation_data=val_generator,
validation_steps=50,
workers=4)
model.compile(Adam(lr=learn), loss='binary_crossentropy', metrics=[dice_coef,'accuracy'])
return model
K.clear_session()
model = ResNet50()
model.summary()
from keras.callbacks import ModelCheckpoint, TensorBoard, CSVLogger
mc = ModelCheckpoint('model_loss', monitor='val_loss', save_best_only=True)
#tb = TensorBoard(write_images=True)
cv = CSVLogger('/content/drive/My Drive/Colab/results/latest/smriti_resnet_test1_l3.csv',append=True)
from google.colab import drive
drive.mount('/content/drive')
os.listdir('/content/drive/My Drive/Colab/results/best/')
epochs =160
bestdc=0
total=0
for i in range(epochs):
print(i)
history= model.fit_generator(datagen.flow(train_X, train_y, batch_size=4), nb_epoch=1, samples_per_epoch=len(train_X), callbacks=[cv,mc])
#history= model.fit(train_X, train_y, batch_size=4, nb_epoch=1, validation_data=(valid_X,valid_y), callbacks=[cv,mc])
total=0
for j in range(8):
def train(self, epochs, learning_rate, batch_size, wmap, vbal, model_name,
new_ex):
if ".hdf5" in model_name:
model_name = model_name.split(".hdf5")[0]
else:
pass
print("Loading data")
if wmap == False:
imgs_train, imgs_mask_train = self.load_data(wmap=wmap, vbal=vbal)
else:
imgs_train, imgs_mask_train, img_weights = self.load_data(
wmap=wmap, vbal=vbal)
print("Loading data done")
model = self.get_mitosegnet(wmap, learning_rate)
print("Got MitoSegNet")
print(self.path + os.sep + model_name)
if os.path.isfile(self.path + os.sep + model_name + ".hdf5"):
model.load_weights(self.path + os.sep + model_name + ".hdf5")
print("Loading weights")
else:
print(
"No previously optimized weights were loaded. Proceeding without"
)
# Set network weights saving mode.
# save previously established network weights (saving model after every epoch)
print('Fitting model...')
if new_ex == "New":
first_ep = 0
model_name = model_name + "_" + str(self.img_rows) + "_"
elif new_ex == "Finetuned_New":
first_ep = 0
else:
if os.path.isfile(self.path + os.sep + model_name +
'training_log.csv'):
prev_csv = True
prev_csv_file = pd.read_csv(self.path + os.sep + model_name +
'training_log.csv')
first_ep = len(prev_csv_file)
if prev_csv_file.shape[1] > 7:
prev_csv_file = prev_csv_file.drop(
prev_csv_file.columns[[0]], axis=1)
else:
prev_csv = False
csv_logger = CSVLogger(self.path + os.sep + model_name +
'training_log.csv')
tensorboard = TensorBoard(log_dir=self.path + os.sep +
"logs/{}".format(time()))
# Set callback functions to early stop training and save the best model so far
callbacks = [
EarlyStopping(monitor='val_loss', patience=20),
ModelCheckpoint(filepath=self.path + os.sep + model_name + ".hdf5",
monitor='val_loss',
verbose=1,
save_best_only=True), csv_logger, tensorboard
]
if wmap == True:
x = [imgs_train, img_weights]
else:
x = imgs_train
print(
"\nCopy the line below into the terminal, press enter and click on the link to evaluate the training "
"performance:\n\ntensorboard --logdir=" + self.path + os.sep +
"logs/\n")
### ECG see if this solves
# https://github.com/tensorflow/tensorflow/issues/34944
tf.config.experimental_run_functions_eagerly(True)
###
model.fit(x=x,
y=imgs_mask_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_split=0.2,
shuffle=True,
callbacks=callbacks)
csv_file = pd.read_csv(self.path + os.sep + model_name +
'training_log.csv')
if new_ex == "New" or new_ex == "Finetuned_New":
csv_file["epoch"] = list(range(1, len(csv_file) + 1))
last_ep = len(csv_file)
if new_ex == "Existing" and prev_csv == True:
frames = [prev_csv_file, csv_file]
merged = pd.concat(frames, names=[])
merged["epoch"] = list(range(1, len(merged) + 1))
last_ep = len(merged)
merged.to_csv(self.path + os.sep + model_name + 'training_log.csv')
if new_ex == "New":
info_file = open(
self.path + os.sep + model_name + str(first_ep) + "-" +
str(last_ep) + "_train_info.txt", "w")
info_file.write("Learning rate: " + str(learning_rate) +
"\nBatch size: " + str(batch_size) +
"\nClass balance weight factor: " + str(vbal))
info_file.close()
K.clear_session()
model.summary()
epochs = 1000
batch_size = 16
checkpointer = ModelCheckpoint(
filepath=os.path.join('//data/d14122793/human_protein_atlas_image_classification', 'checkpoints', 'Training-' + \
'.{epoch:03d}-{val_loss:.3f}.hdf5'),
verbose=1,
save_best_only=True)
early_stopper = EarlyStopping(patience=50)
# Helper: Save results.
timestamp = time.time()
csv_logger = CSVLogger(
os.path.join('.', 'logs', 'training-' + str(timestamp) + '.log'))
# split and suffle data
np.random.seed(2018)
indexes = np.arange(train_dataset_info.shape[0])
np.random.shuffle(indexes)
train_indexes = indexes[:27500]
valid_indexes = indexes[27501:]
# create train and valid datagens
train_generator = data_gen.create_train(train_dataset_info[train_indexes],
batch_size, (299, 299, 3))
validation_generator = data_gen.create_train(train_dataset_info[valid_indexes],
100, (299, 299, 3),
augument=False)
def main():
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.tensorflow_backend.set_session(sess)
args = parser.parse_args()
DATASET = args.dataset
MODEL = args.model
EPOCH = args.epoch
PATIENCE = args.patience
BATCH_SIZE = args.batch_size
NUM_WORKER = args.num_worker
paths, pose_label = load_data(DATASET)
n_fold = 1
losses = {"pose_prediction": custom_mse_pose}
metrics = {"pose_prediction": pose_metric}
print('[K-FOLD] Started...')
kf = KFold(n_splits=10, shuffle=True, random_state=1)
kf_split = kf.split(pose_label)
for train_idx, test_idx in kf_split:
model = None
if MODEL == 'vggFace':
model = Net(MODEL,1,5,8,2,8)
model = freeze_all_but_mid_and_top(model)
else:
model = Net(MODEL,1,5,8,2,8)
train_paths = paths[train_idx]
train_label = pose_label[train_idx]
test_paths = paths[test_idx]
test_label = pose_label[test_idx]
print(len(train_paths),len(test_paths))
model.summary()
weights_path = './train_weights/pose/{}/{}/'.format(DATASET,model.name)
logs_path = './train_log/pose/{}/{}/'.format(DATASET,model.name)
if not os.path.exists(weights_path):
os.makedirs(weights_path)
if not os.path.exists(logs_path):
os.makedirs(logs_path)
model_names = weights_path + '.{epoch:02d}-{val_pose_metric: 0.4f}.hdf5'
csv_name = logs_path + '{}.log'.format(n_fold)
board_name = logs_path + '{}'.format(n_fold)
checkpoint = ModelCheckpoint(model_names, verbose=1,save_weights_only = True,save_best_only=True)
csvlogger=CSVLogger(csv_name)
early_stop = EarlyStopping('val_loss', patience=PATIENCE)
reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1,
patience=int(PATIENCE/2), verbose=1)
tensorboard = TensorBoard(log_dir=board_name,batch_size=BATCH_SIZE)
callbacks = [checkpoint,csvlogger,early_stop,reduce_lr,tensorboard]
if MODEL == 'ssrnet':
callbacks = [
# ModelCheckpoint(
# "train_weight/{}-{val_gender_prediction_binary_accuracy:.4f}-{val_age_prediction_mean_absolute_error:.4f}.h5".format(
# MODEL),
# verbose=1, save_best_only=True, save_weights_only=True),
ModelCheckpoint(MODEL, 'val_loss', verbose=1,save_best_only=True),
CSVLogger('train_log/{}-{}.log'.format(MODEL, n_fold)),
DecayLearningRate([30, 60])]
model.compile(optimizer='adam', loss=losses, metrics = metrics)
model.fit_generator(
DataGenerator(model, train_paths, train_label, BATCH_SIZE),
validation_data=DataGenerator(model, test_paths, test_label, BATCH_SIZE),
epochs=EPOCH,
verbose=2,
workers=NUM_WORKER,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks
)
n_fold += 1
del train_paths, train_roll,train_pictch,train_yaw
del test_paths, test_roll,test_pitch,test_yaw
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
print('x_train shape:', x_train.shape)
print('y_train shape:', y_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
print('-' * 30)
print('Creating and compiling model...')
print('-' * 30)
model = getinceptionv3()
csv_logger = CSVLogger('log-inceptionv3.csv')
model_checkpoint = ModelCheckpoint('weights-inceptionv3.h5',
monitor='acc',
save_best_only=True)
gen = keras.preprocessing.image.ImageDataGenerator(rotation_range=15.,
width_shift_range=0.15,
height_shift_range=0.15,
shear_range=0.4,
zoom_range=0.4,
channel_shift_range=0.5,
horizontal_flip=True,
vertical_flip=False)
batch_size = 32
train_steps = int(x_train.shape[0] / batch_size) + 1
loss = 'binary_crossentropy'
embedding_dim = 768
claims_input, pred_label = training.lstm_model(max_seq_length,
embedding_dim, nb_classes)
model = Model([claims_input], pred_label)
print(model.summary())
print("labels ndim ", labels.shape)
print(claims_sents_vec.shape)
early_stopping = EarlyStopping(monitor='val_loss', patience=2)
model.compile(optimizer=optimizers.Adam(), loss=loss, metrics=['accuracy'])
csv_logger = CSVLogger("bert_training.log")
model_path = 'models_fever_full/claim_classifier_models/model_bert_fever_full_binary_claim_classifierAcc74.h5'
history = model.fit({'claims': claims_sents_vec},
labels,
epochs=60,
batch_size=64,
validation_split=0.12,
callbacks=[
early_stopping, metrics, csv_logger,
ModelCheckpoint(filepath=model_path,
monitor='val_loss',
save_best_only=True)
])
dataset_name = test_dataset_name
test_lstm_model = testModel(dataset_name, model_path, nb_classes)
test_lstm_model.get_results_on_test_data()
test_labels = np_utils.to_categorical(test_labels, nb_classes=5)
model = conv_model(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
if validation_switch:
val_cot = 1
kf = KFold(train_data.shape[0], n_folds)
for train_index, val_index in kf:
X_train, X_val = train_data[train_index], train_data[val_index]
y_train, y_val = train_labels[train_index], train_labels[
val_index]
csv_logger = CSVLogger('epoch' + str(nb_epoch) + '_' +
str(cot) + '_val_' + str(val_cot) +
'.csv')
val_cot += 1
model.fit(X_train,
y_train,
nb_epoch=nb_epoch,
batch_size=16,
verbose=1,
validation_data=(X_val, y_val),
callbacks=[csv_logger])
csv_logger = CSVLogger('epoch' + str(nb_epoch) + '_' + str(cot) +
'.csv')
model.fit(train_data,
train_labels,
def __init__(self, *args, **kws):
KerasCSVLogger.__init__(self, *args, **kws)
if self.filename.startswith("gs://"):
self.on_train_begin = self._gcp_on_train_begin
y_train = to_categorical(y_train, classes)
y_test = to_categorical(y_test, classes)
#building model
nn5 = Sequential()
nn5.add(Dense(1000, activation='relu', input_shape=(784,)))
nn5.add(Dense(1000, activation ='relu'))
nn5.add(Dense(1000, activation ='relu'))
nn5.add(Dense(1000, activation ='relu'))
nn5.add(Dense(1000, activation ='relu'))
nn5.add(Dense(1000, activation ='relu'))
nn5.add(Dense(1000, activation ='relu'))
nn5.add(Dense(1000, activation ='relu'))
nn5.add(Dense(1000, activation ='relu'))
nn5.add(Dense(classes, activation='softmax'))
nn5.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
print(nn5.summary())
csv_logger = CSVLogger('training2.log', separator=',', append=False)
nn5_history = nn5.fit(X_train, y_train, epochs = 50, verbose=1, validation_data=(X_test, y_test), callbacks=[csv_logger])
nn5_score = nn5.evaluate(X_test, y_test)
print('Test score:', nn5_score[0])
print('Test accuracy:', nn5_score[1])
plot_model_history(nn5_history)
