warmup_steps = 50
#optimizer
optimizer = RMSprop(lr=1e-3)
model.compile(loss=loss_mse_warmup, optimizer=optimizer)
print(model.summary())
#tensorboard setup
path_checkpoint = 'checkpoint.keras'
#validation loss abspeichern
callback_checkpoint = ModelCheckpoint(filepath=path_checkpoint,
monitor='val_loss',
verbose=1,
save_weights_only=True,
save_best_only=True)
callback_early_stopping = EarlyStopping(monitor='val_loss',
patience=5, verbose=1)
callback_tensorboard = TensorBoard(log_dir='./logs/',
histogram_freq=0,
write_graph=True)
callback_reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
min_lr=1e-4,
patience=0,
verbose=1)
'../sample_data/train'))
parser.add_argument("--valid_dir",
dest="valid_dir",
type=str,
default=os.path.join(os.path.dirname(__file__),
'../sample_data/validation'))
parser.add_argument("--model_path_output",
dest="model_path_output",
type=str,
default=save_model_path)
args = parser.parse_args()
# checkpoint to save model after each epoch
save_best = ModelCheckpoint(save_model_path,
monitor='val_loss',
verbose=verbose,
save_best_only=True,
mode='min')
# stop training if the validation error stops improving.
early_stop = EarlyStopping(monitor='val_loss',
min_delta=min_delta,
patience=patience,
verbose=verbose,
mode='auto')
callbacks_list = [save_best]
if use_early_stop:
callbacks_list.append(early_stop)
# import data
start = datetime.now()
op = Adam(lr=1e-3)
batch_size = 32
model.compile(optimizer=op,
loss="sparse_categorical_crossentropy",
metrics=['acc'])
es = EarlyStopping(monitor='val_loss',
patience=20,
restore_best_weights=True,
verbose=1)
lr = ReduceLROnPlateau(monitor='val_loss', vactor=0.5, patience=10, verbose=1)
path = 'C:/nmb/nmb_data/h5/5s_last/Conv2D_2_adam3.h5'
mc = ModelCheckpoint(path, monitor='val_loss', verbose=1, save_best_only=True)
tb = TensorBoard(log_dir='C:/nmb/nmb_data/graph/' + 'Conv2D_2_adam3' + "/",
histogram_freq=0,
write_graph=True,
write_images=True)
# history = model.fit(x_train, y_train, epochs=5000, batch_size=batch_size, validation_split=0.2, callbacks=[es, lr, mc, tb])
# 평가, 예측
model.load_weights('C:/nmb/nmb_data/h5/5s_last/Conv2D_2_adam3.h5')
result = model.evaluate(x_test, y_test, batch_size=batch_size)
print("loss : {:.5f}".format(result[0]))
print("acc : {:.5f}".format(result[1]) + '\n')
############################################ PREDICT ####################################
pred = [
'C:/nmb/nmb_data/predict/5s_last/F', 'C:/nmb/nmb_data/predict/5s_last/M'
Y = np.asarray(Y)
encoder = OneHotEncoder(categories='auto')
Y = encoder.fit_transform(np.expand_dims(Y, axis=1)).toarray()
X_mmw_rD_train, X_mmw_rD_test, Y_train, Y_test = train_test_split(
X_mmw_rD, Y, test_size=0.20, random_state=3, shuffle=True)
X_mmw_rA_train, X_mmw_rA_test, Y_train, Y_test = train_test_split(
X_mmw_rA, Y, test_size=0.20, random_state=3, shuffle=True)
# add early stopping
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=1000)
mc = ModelCheckpoint(
'../models/' +
str(datetime.datetime.now()).replace(':', '-').replace(' ', '_') + '.h5',
monitor='val_acc',
mode='max',
verbose=1,
save_best_only=True)
history = model.fit(
([X_mmw_rD_train, X_mmw_rA_train]),
Y_train,
validation_data=([X_mmw_rD_test, X_mmw_rA_test], Y_test),
epochs=50000,
batch_size=32,
callbacks=[es, mc],
verbose=1,
)
'''
def train(model, train, validation, weight_file):
checkpointer = ModelCheckpoint(filepath=weight_file, verbose=1, save_best_only=True)
return model.fit(train['x'], train['y'], batch_size=16, epochs=50, verbose=0,
validation_data=(validation['x'], validation['y']), callbacks=[checkpointer])
def createCNN1model(dataType):
def getDataWithLabel():
data = pd.read_pickle("DataSets/" + dataType + "_data.pkl")
data = data.sample(frac=1).reset_index(drop=True)
#
# put labels into y_train variable
labels = data['Category']
# Drop 'label' column
data = data.drop(labels=['Category'], axis=1)
return data, labels
def labelEncode(i):
if 'blues' == i:
return 0
elif 'classical' == i:
return 1
elif 'country' == i:
return 2
elif 'disco' == i:
return 3
elif 'hiphop' == i:
return 4
elif 'jazz' == i:
return 5
elif 'metal' == i:
return 6
elif 'pop' == i:
return 7
elif 'reggae' == i:
return 8
else:
return 9
def labelDecode(i):
if 0 == i:
return 'blues'
elif 1 == i:
return "classical"
elif 2 == i:
return "country"
elif 3 == i:
return "disco"
elif 4 == i:
return "hiphop"
elif 5 == i:
return "jazz"
elif 6 == i:
return "metal"
elif 7 == i:
return "pop"
elif 8 == i:
return "reggae"
else:
return "rock"
def fitLabelEncoder(labels):
labelsEncode = []
for i in range(labels.shape[0]):
labelsEncode.append(labelEncode(labels[i]))
labelsEncode = np.array(labelsEncode)
return labelsEncode
def fitLabelDecoder(labels):
labelsDecode = []
for i in range(labels.shape[0]):
labelsDecode.append(labelDecode(labels[i]))
labelsDecode = np.array(labelsDecode)
return labelsDecode
def createTestAndTrain():
X_train, Y_train = getDataWithLabel()
# Normalize the data
X_train = X_train.astype('float16')
X_train = X_train / 255.0
print("Data was normalized..")
print("Data shape: ", X_train.shape)
#Reshape to matrix
X_train = X_train.values.reshape(-1, 240, 240, 3)
print("Data was reshaped..")
#LabelEncode
#labels = preprocessing.LabelEncoder().fit_transform(labels)
Y_train = fitLabelEncoder(Y_train)
print("Data was encoded..")
#int to vector
Y_train = to_categorical(Y_train, num_classes=10)
#train and test data split
#X_train, X_test, Y_train, Y_test= train_test_split(X_train, Y_train, test_size=0.1, random_state=42)
#return X_train, X_test, Y_train, Y_test;
return X_train, Y_train
def createModel(X_train):
model = Sequential()
#
model.add(
Conv2D(filters=16,
kernel_size=(3, 3),
padding='Same',
activation='relu',
input_shape=(X_train.shape[1], X_train.shape[2],
X_train.shape[3])))
model.add(MaxPool2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
#
model.add(
Conv2D(filters=32,
kernel_size=(3, 3),
padding='Same',
activation='relu'))
model.add(MaxPool2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
#
model.add(Flatten())
model.add(Dense(256, activation="relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation="softmax"))
# Define the optimizer
#optimizer = tf.compat.v1.train.AdamOptimizer(1e-3, epsilon=1e-4)
optimizer = tf.keras.optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999)
model.compile(optimizer=optimizer,
loss="categorical_crossentropy",
metrics=["accuracy"])
return model
#set early stopping criteria
pat = 10 #this is the number of epochs with no improvment after which the training will stop
early_stopping = EarlyStopping(monitor='val_loss', patience=pat, verbose=1)
#define the model checkpoint callback -> this will keep on saving the model as a physical file
checkpointPath = dataType + '_CNN1_checkpoint.h5'
model_checkpoint = ModelCheckpoint(checkpointPath,
verbose=1,
save_best_only=True)
def plotAccLossGraphics(history):
plt.title('Accuracies vs Epochs')
plt.plot(history.history["val_accuracy"], label='Validation Acc')
plt.plot(history.history["accuracy"], label='Training Acc')
plt.legend()
plt.show()
# Plot the loss and accuracy curves for training and validation
plt.plot(history.history['val_loss'], label="validation loss ")
plt.plot(history.history['loss'], label="train loss ")
plt.title("Test Loss")
plt.xlabel("Number of Epochs")
plt.ylabel("Loss")
plt.legend()
plt.show()
def plotCategories(y_train, val_y):
Y_train_classes = np.argmax(y_train, axis=1)
Y_train_classes = fitLabelDecoder(Y_train_classes)
plt.figure(figsize=(15, 7))
g = sns.countplot(Y_train_classes, palette="icefire")
plt.title("Train Number of digit classes")
plt.show()
Y_val_classes = np.argmax(val_y, axis=1)
Y_val_classes = fitLabelDecoder(Y_val_classes)
plt.figure(figsize=(15, 7))
g = sns.countplot(Y_val_classes, palette="icefire")
plt.title("Validation Number of digit classes")
plt.show()
gc.collect()
def fit_and_evaluate(train_x, val_x, train_y, val_y):
model = None
gc.collect()
model = createModel(train_x)
batch_size = 32
epochs = 30
gc.collect()
datagen = ImageDataGenerator(zoom_range=0.2, horizontal_flip=False)
print("DataGen Started..")
datagen.fit(train_x)
print("DataGen Finished..")
gc.collect()
results = model.fit_generator(
datagen.flow(train_x, train_y, batch_size=batch_size),
epochs=epochs,
callbacks=[early_stopping, model_checkpoint],
verbose=1,
validation_data=(val_x, val_y))
gc.collect()
print("Val Score: ", model.evaluate(val_x, val_y))
return model, results
def fitWithKfold(X, Y):
n_folds = 5
cv = model_selection.KFold(n_splits=n_folds, shuffle=True)
t0 = time.time()
i = 0
maxAcc = 0
accuracies = []
for train_index, test_index in cv.split(X):
i = i + 1
xx_train, xx_test = X[train_index], X[test_index]
yy_train, yy_test = Y[train_index], Y[test_index]
gc.collect()
print("xx_train data shape: ", xx_train.shape)
print("xx_test data shape: ", xx_test.shape)
t_x, val_x, t_y, val_y = train_test_split(
xx_train,
yy_train,
test_size=0.1,
random_state=np.random.randint(1, 1000, 1)[0])
gc.collect()
print("t_x data shape: ", t_x.shape)
plotCategories(t_y, val_y)
model, history = fit_and_evaluate(t_x, val_x, t_y, val_y)
plotAccLossGraphics(history)
gc.collect()
print("Ended Fold: ", i)
acc = predictTest(xx_test, yy_test, model, i)
accuracies.append(acc)
if acc > maxAcc:
maxAcc = acc
maxAccFold = i
bestX_test = xx_test
bestY_test = yy_test
print("max accuracy: ", maxAcc, " on fold:", maxAccFold)
return accuracies, bestX_test, bestY_test
def predictTest(X_test, Y_test, lmodel, fold):
Y_pred = lmodel.predict(X_test)
# Convert predictions classes to one hot vectors
Y_pred_classes = np.argmax(Y_pred, axis=1)
Y_pred_classes = fitLabelDecoder(Y_pred_classes)
Y_test_label = np.argmax(Y_test, axis=1)
Y_test_label = fitLabelDecoder(Y_test_label)
# compute the confusion matrix
labels = [
"blues", "classical", "country", "disco", "hiphop", "jazz",
"metal", "pop", "reggae", "rock"
]
confusion_mtx = confusion_matrix(Y_test_label, Y_pred_classes)
acc = metrics.accuracy_score(Y_test_label, Y_pred_classes) * 100
print(fold, 'th Accuracy percentage:', acc)
return acc
# confusion matrix-precios-recall
def drawConfusionMatrix(X_test, Y_test, lmodel):
Y_pred = lmodel.predict(X_test)
# Convert predictions classes to one hot vectors
Y_pred_classes = np.argmax(Y_pred, axis=1)
Y_pred_classes = fitLabelDecoder(Y_pred_classes)
Y_test_label = np.argmax(Y_test, axis=1)
Y_test_label = fitLabelDecoder(Y_test_label)
# compute the confusion matrix
labels = [
"blues", "classical", "country", "disco", "hiphop", "jazz",
"metal", "pop", "reggae", "rock"
]
confusion_mtx = confusion_matrix(Y_test_label, Y_pred_classes)
# plot the confusion matrix
f, ax = plt.subplots(figsize=(8, 8))
sns.heatmap(confusion_mtx,
annot=True,
linewidths=0.01,
cmap="Greens",
linecolor="gray",
fmt='.1f',
ax=ax,
xticklabels=labels,
yticklabels=labels)
plt.yticks(rotation=0)
plt.xlabel("Predicted Label")
plt.ylabel("True Label")
plt.title("Confusion Matrix")
plt.show()
acc = metrics.accuracy_score(Y_test_label, Y_pred_classes) * 100
print(classification_report(Y_test_label, Y_pred_classes))
score = lmodel.evaluate(X_test, Y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
X_train, Y_train = createTestAndTrain()
#Visualize Model
from tensorflow.python.keras.utils.vis_utils import plot_model
model = createModel(X_train)
plot_model(model,
to_file='model_plot_Cnn1.png',
show_shapes=True,
show_layer_names=True)
accuraciesList, bestX_test, bestY_test = fitWithKfold(X_train, Y_train)
accuracies = np.array(accuraciesList)
meanAccuracy = np.mean(accuracies)
print(meanAccuracy)
bestModel = tf.keras.models.load_model(dataType + '_CNN1_checkpoint.h5')
drawConfusionMatrix(bestX_test, bestY_test, bestModel)
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer=Adam(lr=0.002), loss='binary_crossentropy')
model2 = Sequential()
model2.add(create_pretrained_embedding_layer(tokenizer, max_question_size))
model2.add(Conv1D(64, 4, activation='relu'))
model2.add(MaxPooling1D(2))
# model2.add(Conv1D(64, 4, activation='relu'))
# model2.add(MaxPooling1D(2))
model2.add(Flatten())
model2.add(Dense(1, activation='sigmoid'))
model2.compile(optimizer=Adam(lr=0.001), loss='binary_crossentropy')
earlystopping = EarlyStopping(patience=3, verbose=True)
checkpoint = ModelCheckpoint(checkpoint_path,
verbose=1,
save_best_only=True,
save_weights_only=False)
reduce_lr = ReduceLROnPlateau(verbose=True, patience=3)
tensorboard = TensorBoard(log_dir=tensorboard_path,
write_graph=False,
write_grads=True,
batch_size=128,
histogram_freq=1)
if kaggle_mode:
callbacks = None
val_data = None
verbose = False
else:
callbacks = [checkpoint, f1_callback, earlystopping]
val_data = (Xval, yval)
X_train, y_train = codes[train_idx], labels_vecs[train_idx]
X_test, y_test = codes[test_idx], labels_vecs[test_idx]
print("There are {} train images and {} test images.".format(
X_train.shape[0], X_test.shape[0]))
print('There are {} unique classes to predict.'.format(
np.unique(y_train).shape[0]))
#One-hot encoding the labels
num_classes = 21
y_train = to_categorical(y_train, num_classes)
y_test = to_categorical(y_test, num_classes)
#Creating a checkpointer
checkpointer = ModelCheckpoint(filepath='scratchmodel.best.hdf5',
verbose=1,
save_best_only=True)
#Loading the ResNet50 model with pre-trained ImageNet weights
model = ResNet50(weights='imagenet',
include_top=False,
input_shape=(200, 200, 3))
#Reshaping the training data
X_train_new = np.array([
imresize(X_train[i], (200, 200, 3)) for i in range(0, len(X_train))
]).astype('float32')
#Preprocessing the data, so that it can be fed to the pre-trained ResNet50 model.
resnet_train_input = preprocess_input(X_train_new)
def train(fold,
model_name,
weights='',
initial_epoch=0,
use_non_blank_frames=True,
nb_epochs=15,
checkpoints_period=16):
"""
Train the level 1 model.
:param fold: Fold to train model on, or -1 to train on the full dataset
:param model_name: name of the level 1 model to train
:param weights: optional, if non empty string passed, training starts from supplied weights,
otherwise imagenet pre-trained model is loaded
:param initial_epoch: Epoch to continue training from, used together with weight parameter to continue training
:param nb_epochs: number of epochs to train
:param use_non_blank_frames: should information about predicted non blank frame be used for frame sampling
:param checkpoints_period: period of checkpoints in epochs
:return: None
"""
K.clear_session()
model_info = MODELS[model_name]
dataset = SingleFrameCNNDataset(
preprocess_input_func=model_info.preprocess_input,
fold=fold,
batch_size=model_info.batch_size,
validation_batch_size=model_info.batch_size,
use_non_blank_frames=use_non_blank_frames)
model = model_info.factory(lock_base_model=True)
if initial_epoch == 0 and weights == '':
# train the first layer first unless continue training
model.compile(optimizer=Adam(lr=1e-4),
loss='binary_crossentropy',
metrics=['accuracy'])
model.fit_generator(dataset.generate(),
steps_per_epoch=dataset.train_steps_per_epoch(),
epochs=1,
verbose=1)
else:
print('load weights', weights)
model.load_weights(str(weights))
utils.lock_layers_until(model, model_info.unlock_layer_name)
# model.summary()
checkpoints_dir = cnnensemble_path / f'output/checkpoints/{model_name}_fold_{fold}'
tensorboard_dir = cnnensemble_path / f'output/tensorboard/{model_name}_fold_{fold}'
os.makedirs(checkpoints_dir, exist_ok=True)
os.makedirs(tensorboard_dir, exist_ok=True)
checkpoint_periodical = ModelCheckpoint(
str(checkpoints_dir /
"checkpoint-{epoch:03d}-{loss:.4f}-{val_loss:.4f}.hdf5"),
verbose=1,
save_weights_only=True,
period=checkpoints_period)
tensorboard = TensorBoard(tensorboard_dir,
histogram_freq=0,
write_graph=False,
write_images=False)
# SGD with lr=1e-4 seems to be training very slowly, but still keep it for initial weights adjustments
nb_sgd_epoch = 2
if initial_epoch < nb_sgd_epoch:
model.compile(optimizer=SGD(lr=1e-4, momentum=0.9),
loss='binary_crossentropy',
metrics=['accuracy'])
model.fit_generator(dataset.generate(),
steps_per_epoch=dataset.train_steps_per_epoch(),
epochs=nb_sgd_epoch,
verbose=1,
validation_data=dataset.generate_test(),
validation_steps=dataset.validation_steps(),
callbacks=[checkpoint_periodical, tensorboard],
initial_epoch=initial_epoch)
model.compile(optimizer=Adam(lr=5e-5),
loss='binary_crossentropy',
metrics=['accuracy'])
model.fit_generator(dataset.generate(),
steps_per_epoch=dataset.train_steps_per_epoch(),
epochs=nb_epochs,
verbose=1,
validation_data=dataset.generate_test(),
validation_steps=dataset.validation_steps(),
callbacks=[checkpoint_periodical, tensorboard],
initial_epoch=max(initial_epoch, nb_sgd_epoch))
model.save_weights(str(model_weights_path(model_name, fold)))
def CNN_onehot(
modelfile,
Dropout1=0,
Epochs=20,
Batch_size=64,
):
# 优化器选择 Adam 优化器。
# 损失函数使用 sparse_categorical_crossentropy，
# 还有一个损失函数是 categorical_crossentropy，两者的区别在于输入的真实标签的形式，
# sparse_categorical 输入的是整形的标签，例如 [1, 2, 3, 4]，categorical 输入的是 one-hot 编码的标签。
train_Feature = np.load("../../data_all/TCRA_train_feature_array.npy")
train_Label = np.load("../../data_all/TCRA_train_label_array.npy")
test_Feature = np.load("../../data_all/TCRA_test_feature_array.npy")
test_Label = np.load("../../data_all/TCRA_test_label_array.npy")
X_train = train_Feature[:, 0:29, :]
Y_train = train_Label
X_test = test_Feature[:, 0:29, :]
Y_test = test_Label
X_train, Y_train = shuffle(X_train, Y_train)
X_test, Y_test = shuffle(X_test, Y_test)
X_train = X_train.reshape([len(X_train), 29, 20, 1])
X_test = X_test.reshape([len(X_test), 29, 20, 1])
X_test = tf.cast(X_test, tf.float32)
model = tf.keras.models.Sequential([
# tf.keras.layers.Conv2D(16, (7,7),padding = 'same', input_shape=(29,20,2),activation='relu'),
# #tf.keras.layers.LeakyReLU(alpha=0.05),
# tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Conv2D(64, (5, 5),
padding='same',
input_shape=(29, 20, 1),
activation='relu'),
#tf.keras.layers.LeakyReLU(alpha=0.05),
#tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.AveragePooling2D(2, 2),
tf.keras.layers.Conv2D(128, (3, 3), padding='same', activation='relu'),
#tf.keras.layers.LeakyReLU(alpha=0.05),
#tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.AveragePooling2D(2, 2),
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(
512, activation='relu'
), # kernel_regularizer=regularizers.l2(0.01)),# activation='relu',
tf.keras.layers.Dense(
256, activation='relu'
), # kernel_regularizer=regularizers.l2(0.01)),# activation='relu',
#tf.keras.layers.LeakyReLU(alpha=0.05),
tf.keras.layers.Dense(128, activation='relu'),
#tf.keras.layers.LeakyReLU(alpha=0.05),
tf.keras.layers.Dense(64, activation='relu'),
#tf.keras.layers.LeakyReLU(alpha=0.05),
tf.keras.layers.Dropout(Dropout1), # Dropout:在 0 和 1 之间浮动。需要丢弃的输入比例
tf.keras.layers.Dense(2, activation='softmax')
])
model.compile(optimizer="Adam",
loss=keras.losses.binary_crossentropy,
metrics=['accuracy'])
checkpoint = ModelCheckpoint(
filepath=modelfile, monitor='val_loss', verbose=0,
save_best_only=True) #,save_weights_only=True)
cbs = [checkpoint] #, lr_reducer, lr_scheduler]
history = model.fit(X_train,
Y_train,
epochs=Epochs,
batch_size=Batch_size,
verbose=0,
validation_data=[X_test, Y_test],
shuffle=False,
callbacks=cbs)
return history
del model
def preTrainingModel(trainingUsers,
testUsers,
allNumOfTrainingSamples,
trainingStepsPerEpoch,
allNumOfValidationSamples,
validationStepsPerEpoch,
amountOfRepetitions,
amountOfGestures,
preTrainingNumOfEpochs,
trial,
batchSize,
totalGestures,
totalRepetitions,
directory,
testUser,
workingDirectory,
trainingDataset,
testDataset,
trainingRepetitions,
testRepetitions,
number_of_classes,
saveCheckpoints):
base_model, multi_model = buildModel(classes=number_of_classes,
features=NUMBER_OF_FEATURES,
cellNeurons=cellNeurons,
cellDropout=recurrent_dropout,
denseDropout=dropout,
denseNeurons=denseNeurons,
sequenceLength=seq_len,
stacked=True,
bidirectional=False,
l2=0.0)
histories = {}
path = workingDirectory+str(testUser)
if not os.path.exists(path):
os.makedirs(path)
my_training_batch_generator = sequenceBatchGeneratorAbsMean2(batchSize=batchSize,
seq_len=seq_len,
indexes=trainingUsers,
stride=stride,
allNumOfSamples=allNumOfTrainingSamples,
meanOf=mean,
shuffling=True,
standardize=False,
amountOfRepetitions=totalRepetitions,
amountOfGestures=totalGestures,
totalGestures=totalGestures,
totalRepetitions=totalRepetitions,
directory=directory,
dataset=trainingDataset,
repetitions=trainingRepetitions,
number_of_classes=number_of_classes)
my_validation_batch_generator = sequenceBatchGeneratorAbsMean2(batchSize=batchSize,
seq_len=seq_len,
indexes=testUsers,
stride=stride,
allNumOfSamples=allNumOfValidationSamples,
meanOf=mean,
shuffling=False,
standardize=False,
amountOfRepetitions=totalRepetitions,
amountOfGestures=totalGestures,
totalGestures=totalGestures,
totalRepetitions=totalRepetitions,
directory=directory,
dataset=testDataset,
repetitions=testRepetitions,
number_of_classes=number_of_classes)
filepath=path + "/e{epoch:03d}-a{val_acc:.3f}.hdf5"
if saveCheckpoints == True:
if onTpu:
modelCheckpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=0, save_best_only=True, mode='max')
else:
modelCheckpoint = AltModelCheckpoint(filepath, alternate_model=base_model, monitor='val_acc', verbose=0, save_best_only=True, mode='max')
callbacks_list = [modelCheckpoint]
else:
callbacks_list = []
startTime = int(round(time.time()))
print("\n##### Start Time with test user "+str(testUser)+": "+str(startTime))
histories[testUser] = multi_model.fit_generator(generator=my_training_batch_generator,
steps_per_epoch=trainingStepsPerEpoch,
epochs=preTrainingNumOfEpochs,
#max_queue_size=5,
verbose=2,
callbacks=callbacks_list,
validation_data=my_validation_batch_generator,
validation_steps=validationStepsPerEpoch,
use_multiprocessing=True)
endTime = int(round(time.time()))
print("\n##### End Time with test user "+str(testUser)+": "+str(endTime))
toLog = str(preTrainingNumOfEpochs) + ',' + str(seq_len) + ',' + str(stride) + ',' + str(batchSize) + ',' + str(mean)
with open(workingDirectory+"history.csv", "a") as myfile:
myfile.write(str(endTime)\
+ ',' + str(trial)\
+ ',' + str(testUser)\
+ ',' + str(max(histories[testUser].history['acc']))\
+ ',' + str(max(histories[testUser].history['val_acc']))\
+ ',' + str(endTime-startTime)\
+ ',' + toLog\
+ ',' + str(amountOfRepetitions)\
+ ',' + str(amountOfGestures) + '\n')
del histories
del base_model, multi_model
del my_training_batch_generator, my_validation_batch_generator
model = GraphSAGE(feature_dim=features.shape[1],
neighbor_num=neigh_maxlen,
n_hidden=16,
n_classes=y_train.shape[1],
use_bias=True,
activation=tf.nn.relu,
aggregator_type='mean',
dropout_rate=0.5, l2_reg=2.5e-4)
model.compile(Adam(0.01), 'categorical_crossentropy',
weighted_metrics=['categorical_crossentropy', 'acc'])
val_data = (model_input, y_val, val_mask)
mc_callback = ModelCheckpoint('./best_model.h5',
monitor='val_weighted_categorical_crossentropy',
save_best_only=True,
save_weights_only=True)
print("start training")
model.fit(model_input, y_train, sample_weight=train_mask, validation_data=val_data,
batch_size=A.shape[0], epochs=200, shuffle=False, verbose=2,
callbacks=[mc_callback])
model.load_weights('./best_model.h5')
eval_results = model.evaluate(
model_input, y_test, sample_weight=test_mask, batch_size=A.shape[0])
print('Done.\n'
'Test loss: {}\n'
'Test weighted_loss: {}\n'
'Test accuracy: {}'.format(*eval_results))
import tensorflow as tf
from tensorflow.python.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint
from tensorflow.python.keras.optimizers import Nadam
from neural_network_solver.dataset import dataset
from neural_network_solver.model import autoencoder
dataset_generator = dataset()
model = autoencoder()
model.compile(optimizer=Nadam(lr=0.001),
loss=tf.keras.losses.categorical_crossentropy,
metrics=['accuracy'])
history = model.fit(dataset_generator,
validation_split=0.2,
batch_size=128,
epochs=999,
verbose=1,
callbacks=[
EarlyStopping(patience=8, verbose=1),
ReduceLROnPlateau(patience=4, min_lr=1e-05),
ModelCheckpoint(
'./neural_network_solver/models/autoencoder.hdfs',
monitor='val_loss',
verbose=False,
save_best_only=True,
save_weights_only=False,
mode='auto',
)
])
raw_image_dataset = tf.data.TFRecordDataset(tfrecord_path_list)
parsed_image_dataset = raw_image_dataset.map(_read_tfrecord)
parsed_image_dataset = parsed_image_dataset.cache()
parsed_image_dataset = parsed_image_dataset.repeat()
parsed_image_dataset = parsed_image_dataset.shuffle(131072)
parsed_image_dataset = parsed_image_dataset.batch(BATCH_SIZE)
'''Training the Model'''
model.compile(optimizer=OPTIMIZER(lr=LR, epsilon=1e-8),
loss='categorical_crossentropy',
metrics=['accuracy'])
# Save the model after every epoch
check_pointer = ModelCheckpoint(filepath='../Models/MobileFaceNet_train.h5',
verbose=1,
save_best_only=False)
# Interrupt the training when the validation loss is not decreasing
early_stopping = EarlyStopping(monitor='val_loss', patience=10000)
# Record the loss history
class LossHistory(Callback):
def on_train_begin(self, logs={}):
self.losses = []
def on_batch_end(self, batch, logs={}):
self.losses.append(logs.get('loss'))
def sparse_cross_entropy(y_true, y_pred):
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_true,
logits=y_pred)
loss_mean = tf.reduce_mean(loss)
return loss_mean
optimizer = optimizers.RMSprop()
decoder_target = tf.placeholder(dtype='int32', shape=(None, None))
language_model.compile(optimizer=optimizer,
loss=sparse_cross_entropy,
target_tensors=[decoder_target])
path_checkpoint = 'model_weights.keras'
callback_checkpoint = ModelCheckpoint(filepath=path_checkpoint,
verbose=1,
save_weights_only=True)
callback_tensorboard = TensorBoard(log_dir='./train_logs/',
histogram_freq=0,
write_graph=False)
callbacks = [callback_checkpoint, callback_tensorboard]
for i in range(epoch_start, epoch_end, 1):
generator = data_generator(train_tokens,
id_map,
vgg_activations,
batch_size=batch_size,
single_caption=False)
steps = int(len(all_caps_train) / batch_size)
language_model.fit_generator(generator,
steps_per_epoch=steps,
# reshape
shape = (28, 28, 1)
x_train = x_train.reshape(-1, shape[0], shape[1], shape[2])
x_test = x_test.reshape(-1, shape[0], shape[1], shape[2])
# train
epochs = 10
batch_size = 128
# ckpt file path
param_folder = '../param_resnet'
if not os.path.isdir(param_folder):
os.makedirs(param_folder)
# callback
cbk = ModelCheckpoint(
filepath=os.path.join(param_folder, 'param{epoch:02d}.hdf5'))
# train
#builder = SimpleCNNBuilder()
builder = ResnetBuilder()
model = builder.build(input_shape=shape, class_num=10)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# train
model.fit(x_train,
y_train,
batch_size,
epochs=epochs,
verbose=1,
def neural_net_train(model,
x_train,
y_train,
val_split=0.1,
validation_data=None,
fig_prefix=''):
"""
训练神经网络
:return:
"""
print('Training the model: ' + model_store_path)
"""
earlystopping and modelcheckpoint
"""
# Callbacks are passed to the model fit the `callbacks` argument in `fit`,
# which takes a list of callbacks. You can pass any number of callbacks.
callbacks_list = [
# This callback will interrupt training when a monitored quantity has stopped improving.
EarlyStopping(
# This callback will monitor the validation accuracy of the model
monitor='val_acc',
# Training will be interrupted when the validation accuracy has stopped improving for 3 epochs
patience=5,
),
# This callback will save the current weights after every epoch
ModelCheckpoint(
filepath=model_store_path, # Path to the destination model file
# The two arguments below mean that we will not overwrite the
# model file unless `val_acc` has improved, which
# allows us to keep the best model every seen during training.
monitor='val_acc',
save_best_only=True,
),
ReduceLROnPlateau(
# This callback will monitor the validation loss of the model
monitor='val_loss',
# It will divide the learning by 10 when it gets triggered
factor=0.1,
# It will get triggered after the validation loss has stopped improving
# for at least 10 epochs
patience=5,
)
]
# Note that since the callback will be monitoring validation accuracy,
# we need to pass some `validation_data` to our call to `fit`.
model.fit(x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=EPOCHES,
callbacks=callbacks_list,
validation_split=val_split)
# acc curve during the training
fig = plt.figure()
acc = model.history.history['acc']
val_acc = model.history.history['val_acc']
epochs = range(1, len(acc) + 1)
plt.plot(epochs, acc, '-ko', label='Train accuracy')
plt.plot(epochs, val_acc, '-k^', label='Validation accuracy')
plt.title('Train and validation accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend(['train', 'validation'], loc='lower right')
plt.xlim((1, len(acc) + 1))
plt.xticks(np.arange(1, len(acc) + 1, 1))
fig.savefig(fig_prefix + '#accuracy-curve.eps')
plt.close(fig)
# loss curve during the training
fig = plt.figure()
plt.plot(epochs, model.history.history['loss'], '-ko')
plt.plot(epochs, model.history.history['val_loss'], '-k^')
plt.title('Train and validation loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Validation'], loc='upper right')
plt.xlim((1, len(acc) + 1))
plt.xticks(np.arange(1, len(acc) + 1, 1))
fig.savefig(fig_prefix + '#loss-curve.eps')
plt.close(fig)
return model
conv.add(MaxPooling2D((1, 2)))
conv.add(Conv2D(256, (1, 3), activation = 'relu'))
conv.add(MaxPooling2D((1, 2)))
conv.add(Flatten())
conv.add(Dense(64, activation = 'relu'))
conv.add(Dropout(0.5))
conv.add(Dense(5, activation = 'softmax'))
conv.summary()
#conv.load_weights('weights/sleepnet-rnn-epoch7.hdf5')
conv.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
#filepath = 'weights/sleepnet-' + conv.name + '-epoch20.hdf5'
checkpoint = ModelCheckpoint(model_file, monitor='val_loss', verbose=0, save_weights_only=True,
save_best_only=True, mode='auto', period=1)
tensor_board = TensorBoard(log_dir='logs/', histogram_freq=0, batch_size=batch_size)
history = conv.fit_generator(train_gen,
steps_per_epoch=500,
epochs=20,
validation_data=val_gen,
validation_steps = 100,
callbacks=[checkpoint, tensor_board])
raw_labels.close()
raw_features.close()
plotLoss(history)
test_target = (test_target + 1) / 2
model = Sequential()
model.add(LSTM(256, input_shape=(seq_len, 4)))
model.add(Dense(1, activation='sigmoid'))
model.summary()
adam = Adam(lr=0.001)
# model_filename = "test-Epoch-{epoch:02d}"
# checkpoint_path = os.path.join('models/', model_filename)
chk = ModelCheckpoint('best_model.pkl',
monitor='val_accuracy',
verbose=1,
save_best_only=False,
save_weights_only=False,
mode='max')
# chk = ModelCheckpoint('best_model.pkl', monitor='val_acc', save_best_only=True, mode='max', verbose=1)
model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])
model.fit(train,
train_target,
epochs=5,
batch_size=128,
callbacks=[chk],
validation_data=(validation, validation_target))
#
# # #loading the model and checking accuracy on the test data
model = load_model('best_model.pkl')
#
def train_model(self):
'''
---- This method overrides the one from Deep_CellSpike. It is almost exactly identical
---- except the callbacks_list now contains a
---- TuneReportCheckpointCallback() object The TuneReportCheckpointCallback
---- class handles reporting metrics and checkpoints. The metric it reports is validation loss.
'''
if self.verb == 0: print('train silently, myRank=', self.myRank)
hpar = self.hparams
callbacks_list = [TuneReportCheckpointCallback(metrics=['val_loss'])]
if self.useHorovod:
# Horovod: broadcast initial variable states from rank 0 to all other processes.
# This is necessary to ensure consistent initialization of all workers when
# training is started with random weights or restored from a checkpoint.
callbacks_list.append(
self.hvd.callbacks.BroadcastGlobalVariablesCallback(0))
# Note: This callback must be in the list before the ReduceLROnPlateau,
# TensorBoard or other metrics-based callbacks.
callbacks_list.append(self.hvd.callbacks.MetricAverageCallback())
#Scale the learning rate `lr = 1.0` ---> `lr = 1.0 * hvd.size()` before
# the first five epochs. See https://arxiv.org/abs/1706.02677 for details.
mutliAgent = hpar['train_conf']['multiAgent']
if mutliAgent['warmup_epochs'] > 0:
callbacks_list.append(
self.hvd.callbacks.LearningRateWarmupCallback(
warmup_epochs=mutliAgent['warmup_epochs'],
verbose=self.verb))
if self.verb:
print('added LearningRateWarmupCallback(%d epochs)' %
mutliAgent['warmup_epochs'])
lrCb = MyLearningTracker()
callbacks_list.append(lrCb)
trlsCb = None
if self.train_loss_EOE:
print('enable end-epoch true train-loss as callBack')
assert self.cellName == None # not tested
genConf = copy.deepcopy(self.sumRec['inpGen']['train'])
# we need much less stats for the EOE loss:
genConf['numLocalSamples'] //= 8 # needs less data
genConf['name'] = 'EOF' + genConf['name']
inpGen = CellSpike_input_generator(genConf, verb=1)
trlsCb = MyEpochEndLoss(inpGen)
callbacks_list.append(trlsCb)
if self.checkPtOn and self.myRank == 0:
outF5w = self.outPath + '/' + self.prjName + '.weights_best.h5'
chkPer = 1
ckpt = ModelCheckpoint(outF5w,
save_best_only=True,
save_weights_only=True,
verbose=1,
period=chkPer,
monitor='val_loss')
callbacks_list.append(ckpt)
if self.verb: print('enabled ModelCheckpoint, save_freq=', chkPer)
LRconf = hpar['train_conf']['LRconf']
if LRconf['patience'] > 0:
[pati, fact] = LRconf['patience'], LRconf['reduceFactor']
redu_lr = ReduceLROnPlateau(monitor='val_loss',
factor=fact,
patience=pati,
min_lr=0.0,
verbose=self.verb,
min_delta=LRconf['min_delta'])
callbacks_list.append(redu_lr)
if self.verb:
print('enabled ReduceLROnPlateau, patience=%d, factor =%.2f' %
(pati, fact))
if self.earlyStopPatience > 0:
earlyStop = EarlyStopping(monitor='val_loss',
patience=self.earlyStopPatience,
verbose=self.verb,
min_delta=LRconf['min_delta'])
callbacks_list.append(earlyStop)
if self.verb:
print('enabled EarlyStopping, patience=',
self.earlyStopPatience)
#pprint(hpar)
if self.verb:
print('\nTrain_model goalEpochs=%d' % (self.goalEpochs),
' modelDesign=', self.modelDesign, 'localBS=',
hpar['train_conf']['localBS'], 'globBS=',
hpar['train_conf']['localBS'] * self.numRanks)
fitVerb = 1 # prints live: [=========>....] - ETA: xx s
if self.verb == 2: fitVerb = 1
if self.verb == 0: fitVerb = 2 # prints 1-line summary at epoch end
if self.numRanks > 1: # change the logic
if self.verb:
fitVerb = 2
else:
fitVerb = 0 # keras is silent
startTm = time.time()
hir = self.model.fit( # TF-2.1
self.inpGenD['train'],
callbacks=callbacks_list,
epochs=self.goalEpochs,
max_queue_size=10,
workers=1,
use_multiprocessing=False,
shuffle=self.shuffle_data,
verbose=fitVerb,
validation_data=self.inpGenD['val'])
fitTime = time.time() - startTm
hir = hir.history
totEpochs = len(hir['loss'])
earlyStopOccured = totEpochs < self.goalEpochs
#print('hir keys',hir.keys(),lrCb.hir)
for obs in hir:
rec = [float(x) for x in hir[obs]]
self.train_hirD[obs].extend(rec)
# this is a hack, 'lr' is returned by fit only when --reduceLr is used
if 'lr' not in hir:
self.train_hirD['lr'].extend(lrCb.hir)
if trlsCb: # add train-loss for Kris
nn = len(self.train_hirD['loss'])
self.train_hirD['train_loss'] = trlsCb.hir[-nn:]
self.train_hirD['stepTimeHist'] = self.inpGenD['train'].stepTime
#report performance for the last epoch
lossT = self.train_hirD['loss'][-1]
lossV = self.train_hirD['val_loss'][-1]
lossVbest = min(self.train_hirD['val_loss'])
if self.verb:
print('end-hpar:')
pprint(hpar)
print(
'\n End Val Loss=%s:%.3f, best:%.3f' %
(hpar['train_conf']['lossName'], lossV, lossVbest),
', %d totEpochs, fit=%.1f min, earlyStop=%r' %
(totEpochs, fitTime / 60., earlyStopOccured))
self.train_sec = fitTime
xxV = np.array(self.train_hirD['loss'][-5:])
trainLoss_avr_last5 = -1
if xxV.shape[0] > 3: trainLoss_avr_last5 = xxV.mean()
# add info to summary
rec = {}
rec['earlyStopOccured'] = int(earlyStopOccured)
rec['fitTime_min'] = fitTime / 60.
rec['totEpochs'] = totEpochs
rec['trainLoss_avr_last5'] = float(trainLoss_avr_last5)
rec['train_loss'] = float(lossT)
rec['val_loss'] = float(lossV)
rec['steps_per_epoch'] = self.inpGenD['train'].__len__()
rec['state'] = 'model_trained'
rec['rank'] = self.myRank
rec['num_ranks'] = self.numRanks
rec['num_open_files'] = len(self.inpGenD['train'].conf['cellList'])
self.sumRec.update(rec)
def train_model(autoencoder=False, reload=False):
models_trainable.initialized()
data_array = []
data_src = []
data_dest = []
if reload:
text_filter = TextFilter()
if not autoencoder:
keyword_models = models_trainable.Keyword.select().where(
models_trainable.Keyword.t_type >= 1,
models_trainable.Keyword.t_type <= 2)
for keyword in keyword_models:
try:
videos = models_trainable.Video.select()\
.join(models_trainable.Relationship, on=models_trainable.Relationship.to_video)\
.where(models_trainable.Relationship.from_keyword == keyword)
print(keyword.name, len(videos))
for video in videos:
title = video.title
text_filter.set_text(title)
text_filter.regex_from_text(r'\[[^)]*\]')
text_filter.remove_texts_from_text()
text_filter.remove_pumsas_from_list()
text_filter.remove_texts_from_text()
data_array.append([
mark_start + keyword.name + mark_end,
str(text_filter)
])
except models_trainable.DoesNotExist:
print("does not exist")
else:
videos = models_trainable.Video.select()
for video in videos:
title = video.title
text_filter.set_text(title)
text_filter.regex_from_text(r'\[[^)]*\]')
text_filter.remove_texts_from_text()
text_filter.remove_pumsas_from_list()
text_filter.remove_texts_from_text()
data_array.append([
mark_start + str(text_filter) + mark_end,
str(text_filter)
])
for value in data_array:
data_src.append(value[1])
data_dest.append(value[0])
# saving
with open('data_src.pickle', 'wb') as handle:
pickle.dump(data_src, handle, protocol=pickle.HIGHEST_PROTOCOL)
# saving
with open('data_dest.pickle', 'wb') as handle:
pickle.dump(data_dest, handle, protocol=pickle.HIGHEST_PROTOCOL)
else:
# saving
with open('data_src.pickle', 'rb') as handle:
data_src = pickle.load(handle)
# saving
with open('data_dest.pickle', 'rb') as handle:
data_dest = pickle.load(handle)
tokenizer_src = TokenizerWrap(texts=data_src,
padding='pre',
reverse=True,
num_words=num_words)
if autoencoder:
tokenizer_dest = TokenizerWrap(texts=data_dest,
padding='post',
reverse=False,
num_words=num_words)
else:
tokenizer_dest = TokenizerWrap(texts=data_dest,
padding='post',
reverse=False,
num_words=len(queries) + 3)
tokens_src = tokenizer_src.tokens_padded
tokens_dest = tokenizer_dest.tokens_padded
#
encoder_input_data = tokens_src
decoder_input_data = tokens_dest[:, :-1]
decoder_output_data = tokens_dest[:, 1:]
model_train, model_encoder, model_decoder = get_model(autoencoder)
callback_checkpoint = ModelCheckpoint(filepath=path_checkpoint,
monitor='val_loss',
verbose=1,
save_weights_only=True,
save_best_only=True)
callback_early_stopping = EarlyStopping(monitor='val_loss',
patience=3,
verbose=1)
callback_tensorboard = TensorBoard(log_dir='./21_logs/',
histogram_freq=0,
write_graph=False)
callbacks = [
callback_early_stopping, callback_checkpoint, callback_tensorboard
]
try:
model_train.load_weights(path_checkpoint)
except Exception as error:
print("Error trying to load checkpoint.")
print(error)
x_data = \
{
'encoder_input': encoder_input_data,
'decoder_input': decoder_input_data
}
y_data = \
{
'decoder_output': decoder_output_data
}
validation_split = 500 / len(encoder_input_data)
model_train.fit(x=x_data,
y=y_data,
batch_size=120,
epochs=10,
validation_split=validation_split,
callbacks=callbacks)
def combine(*args):
x = tf.keras.layers.Add()(args)
x = Dense(2, activation='softmax', name='fc3')(x)
model = Model(inputs=input_img, outputs=x)
return model
model = build_model()
# opt = RMSprop(lr=0.0001, decay=1e-6)
opt = Adam(lr=0.0001, decay=1e-5)
es = EarlyStopping(patience=5)
chkpt = ModelCheckpoint(filepath='best_model_todate_5_13_7p',
save_best_only=True,
save_weights_only=True)
model.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer=opt)
batch_size = 16
nb_epochs = 20
# Get a train data generator
train_data_gen = data_gen(data=train_data, batch_size=batch_size)
# Define the number of training steps
nb_train_steps = train_data.shape[0] // batch_size
print("Number of training and validation steps: {} and {}".format(
nb_train_steps, len(valid_data)))
def main(run_dir, data_dir, nb_epoch, early_stopping_patience, desired_sample_rate, fragment_length, batch_size,
fragment_stride, nb_output_bins, keras_verbose, _log, seed, _config, debug, learn_all_outputs,
train_only_in_receptive_field, _run, use_ulaw, train_with_soft_target_stdev):
if run_dir is None:
if not os.path.exists("models"):
os.makedirs("models", exist_ok=True)
run_dir = os.path.join('models', datetime.datetime.now().strftime('run_%Y%m%d_%H%M%S'))
_config['run_dir'] = run_dir
print_config(_run)
_log.info('Running with seed %d' % seed)
if not debug:
if os.path.exists(run_dir):
raise EnvironmentError('Run with seed %d already exists' % seed)
os.makedirs(run_dir, exist_ok=True)
checkpoint_dir = os.path.join(run_dir, 'checkpoints')
json.dump(_config, open(os.path.join(run_dir, 'config.json'), 'w'))
_log.info('Loading data...')
data_generators, nb_examples = get_generators()
_log.info('Building model...')
model = build_model(fragment_length)
_log.info(model.summary())
optim = make_optimizer()
_log.info('Compiling Model...')
loss = losses.categorical_crossentropy
all_metrics = [
metrics.categorical_accuracy,
categorical_mean_squared_error
]
if train_with_soft_target_stdev:
loss = make_targets_soft(loss)
if train_only_in_receptive_field:
loss = skip_out_of_receptive_field(loss)
all_metrics = [skip_out_of_receptive_field(m) for m in all_metrics]
model.compile(optimizer=optim, loss=loss, metrics=all_metrics)
# TODO: Consider gradient weighting making last outputs more important.
tictoc = strftime("%a_%d_%b_%Y_%H_%M_%S", gmtime())
directory_name = tictoc
log_dir = 'tensorboard\\wavenet_' + directory_name
os.makedirs(log_dir, exist_ok=True)
tensorboard = TensorBoard(log_dir=log_dir)
callbacks = [
tensorboard,
ReduceLROnPlateau(patience=early_stopping_patience / 2, cooldown=early_stopping_patience / 4, verbose=1),
EarlyStopping(patience=early_stopping_patience, verbose=1),
]
if not debug:
os.makedirs(checkpoint_dir, exist_ok=True)
callbacks.extend([
ModelCheckpoint(os.path.join(checkpoint_dir, 'checkpoint.{epoch:05d}-{val_loss:.3f}.hdf5'),
save_best_only=True),
CSVLogger(os.path.join(run_dir, 'history.csv')),
])
_log.info('Starting Training...')
print("nb_examples['train'] {0}".format(nb_examples['train']))
print("nb_examples['test'] {0}".format(nb_examples['test']))
model.fit_generator(data_generators['train'],
steps_per_epoch=nb_examples['train'] // batch_size,
epochs=nb_epoch,
validation_data=data_generators['test'],
validation_steps=nb_examples['test'] // batch_size,
callbacks=callbacks,
verbose=keras_verbose)
def train(data,label_score, opts, vars):
try:
module = __import__(opts['network'])
set_opts_from_config(opts, module.conf)
n_input = data[0].dim
if not opts['is_regression']:
#adding one output for the restclass
n_output = int(max(label_score)+1)
vars['monitor'] = "acc"
else:
vars['monitor'] = "loss"
n_output = 1
print ('load network architecture from ' + opts['network'] + '.py')
print('#input: {} \n#output: {}'.format(n_input, n_output))
model = module.getModel(n_input, n_output)
nts = opts['n_timesteps']
if nts < 1:
raise ValueError('n_timesteps must be >= 1 but is {}'.format(nts))
elif nts == 1:
sample_shape = (n_input, )
else:
sample_shape = (int(n_input / nts), nts)
#(number of samples, number of features, number of timesteps)
sample_list_shape = ((len(data),) + sample_shape)
x = np.empty(sample_list_shape)
y = np.empty((len(label_score), n_output))
print('Input data array should be of shape: {} \nLabel array should be of shape {} \nStart reshaping input accordingly...\n'.format(x.shape, y.shape))
sanity_check(x)
#reshaping
for sample in range(len(data)):
#input
temp = np.reshape(data[sample], sample_shape)
x[sample] = temp
#output
if not opts['is_regression']:
y[sample] = convert_to_one_hot(label_score[sample], n_output)
else:
y[sample] = label_score[sample]
log_path, ckp_path = get_paths()
experiment_id = opts['experiment_id'] if opts['experiment_id'] else opts['network'] + '-' + str(time.strftime("%Y_%m_%d-%H_%M_%S"))
print('Checkpoint dir: {}\nLogdir: {}\nExperiment ID: {}'.format(ckp_path, log_path, experiment_id))
checkpoint = ModelCheckpoint(
filepath = os.path.join(ckp_path, experiment_id + '.trainer.PythonModel.model.h5'),
monitor=vars['monitor'],
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1)
callbacklist = [checkpoint]
model.summary()
print('Loss: {}\nOptimizer: {}\n'.format(opts['loss_function'], opts['optimizier']))
model.compile(loss=opts['loss_function'],
optimizer=opts['optimizier'],
metrics=opts['metrics'])
print('train_x shape: {} | train_x[0] shape: {}'.format(x.shape, x[0].shape))
model.fit(x,
y,
epochs=opts['n_epoch'],
batch_size=opts['batch_size'],
callbacks=callbacklist)
vars['n_input'] = n_input
vars['n_output'] = n_output
vars['model_id'] = experiment_id
vars['model'] = model
except Exception as e:
print_exception(e, 'train')
exit()
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
loaded_model.load_weights("./Weights/modelCat2.h5")
print("Loaded model from disk")
optimizer = Adam(lr=1e-5)
loss = ['binary_crossentropy']
metrics = ['mae', 'accuracy']
loaded_model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
filepath = "./Weights/ModelCropCat2/weights-{epoch:02d}-{val_acc:.2f}.hdf5"
checkpoint = ModelCheckpoint(filepath,
save_best_only=False,
monitor='val_acc',
mode='max',
verbose=1,
save_weights_only=True,
period=1)
tensorboard = TensorBoard(log_dir='./logs/ModelCrop5',
write_graph=True,
write_images=True,
batch_size=batchSize)
callback_list = [checkpoint, tensorboard]
history = loaded_model.fit_generator(
generator=generator_train,
epochs=10,
steps_per_epoch=len(partition['train']) / batchSize,
metrics=['accuracy'])
model_dir = './model/' # 모델을 저장할 폴더
if not os.path.exists(model_dir):
os.mkdir(model_dir)
# epoch : 에포크
# val_loss : 검증용 데이터셋의 손실 오차
model_name = model_dir + '{epoch:02d}-{val_loss:.4f}.hdf5'
# filepath: 파일을 저장할 경로, monitor: 모니터링할 대상
# loss(학습셋 오차), val_loss(검증용셋 오차), acc(학습 정확도), val_acc(검증용셋 정확도)
# save_best_only: 이전 단계보다 모델이 더 개선되었을 경우에만 저장
mcp = ModelCheckpoint(filepath=model_name,
monitor='val_loss',
verbose=1,
save_best_only=True)
# patience=100: 테스트의 오차가 개선되지 않더라도 100번 기다림
es = EarlyStopping(monitor='val_loss', patience=100)
"""
validation_split: 훈련에 반영하지 않고, 테스트용으로 사용할 데이터셋 비율
실제 학습에 사용되지는 않고, 매 epoch의 끝자락에서
metrics를 평가하는 용도로 사용
callbacks: 콜백 함수 객체들을 리스트 형식으로 지정해 줌
History 객체: fit() 함수의 반환 객체
"""
history = model.fit(x,
y,
epochs=3500,
batch_size=500,
class_mode='categorical')
# Max number of steps that these generator will have opportunity to process their source content
# len(train_generator) should be 'no. of available train images / BATCH_SIZE_TRAINING'
# len(valid_generator) should be 'no. of available train images / BATCH_SIZE_VALIDATION'
(BATCH_SIZE_TRAINING, len(train_generator), BATCH_SIZE_VALIDATION,
len(validation_generator))
# ### Train Our Model With Cats & Dogs Train (splitted) Data Set
# Early stopping & checkpointing the best model in ../working dir & restoring that as our model for prediction
from tensorflow.python.keras.callbacks import EarlyStopping, ModelCheckpoint
cb_early_stopper = EarlyStopping(monitor='val_loss',
patience=EARLY_STOP_PATIENCE)
cb_checkpointer = ModelCheckpoint(filepath='working/best.hdf5',
monitor='val_loss',
save_best_only=True,
mode='auto')
# Grid Search is an ideal candidate for distributed machine learning
# Pseudo code for hyperparameters Grid Search
'''
from sklearn.grid_search import ParameterGrid
param_grid = {'epochs': [5, 10, 15], 'steps_per_epoch' : [10, 20, 50]}
grid = ParameterGrid(param_grid)
# Accumulate history of all permutations (may be for viewing trend) and keep watching for lowest val_loss as final model
for params in grid:
print(params)
'''
def main(argv):
args = parse_arguments(argv=argv)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
# Load a bunch of training records
logger.info("Loading training samples tfrecords")
parsed_trainset = fileio.read_tfrecords_dir(
dirname=args.train_dir,
image_width=args.input_image_width,
image_height=args.input_image_height,
image_channels=args.input_image_channels,
)
# Read validation dataset
logger.info("Loading validation samples tfrecords")
parsed_validationset = fileio.read_tfrecords_dir(
dirname=args.val_dir,
image_width=args.input_image_width,
image_height=args.input_image_height,
image_channels=args.input_image_channels,
)
num_train_samples = sum(1 for record in parsed_trainset)
num_val_samples = sum(1 for record in parsed_validationset)
print("Number of training / validation samples %d/%d" %
(num_train_samples, num_val_samples))
# Initialise model with given architecture
if args.model_arch_name == "basiclinear":
model = basic_linear_model.create_model(
image_width=args.input_image_width,
image_height=args.input_image_height,
image_channels=args.input_image_channels,
crop_margin_from_top=args.input_image_vertical_crop_pixels,
)
elif args.model_arch_name == "mobilenetv2":
model = mobilenet_v2.create_model(
image_width=args.input_image_width,
image_height=args.input_image_height,
image_channels=args.input_image_channels,
crop_margin_from_top=args.input_image_vertical_crop_pixels,
)
if os.path.exists(os.path.dirname(args.output_model_file)) == False:
os.makedirs(os.path.dirname(args.output_model_file))
# checkpoint to save model after each epoch
save_best = ModelCheckpoint(
args.output_model_file,
monitor="val_loss",
verbose=args.verbose,
save_best_only=True,
mode="min",
)
# stop training if the validation error stops improving.
early_stop = EarlyStopping(
monitor="val_loss",
min_delta=args.min_delta,
patience=args.patience,
verbose=args.verbose,
mode="auto",
)
# Train the car
model.fit(
parsed_trainset,
validation_data=parsed_validationset,
steps_per_epoch=num_train_samples // args.batch_size,
validation_steps=num_val_samples // args.batch_size,
epochs=args.epochs,
callbacks=[tf_tools.JsonLogger(), save_best, early_stop],
verbose=args.verbose,
)
model.load_weights(args.checkpoint_path + args.checkpoint_name,
by_name=True)
initial_epoch = int(args.checkpoint_path.split('-')[1])
else:
initial_epoch = 0
#优化器
optimizer = tf.keras.optimizers.Adam(lr=args.learning_rate, )
#callbacks list
json_logging_callback = LoggingCallback(log_file_path)
# 中途训练效果提升, 则将文件保存, 每提升一次, 保存一次
filepath = args.checkpoint_path + args.model_name + "-{epoch: 02d}-{val_auc:.4f}.hdf5"
checkpoint = ModelCheckpoint(filepath,
monitor='val_auc',
verbose=1,
save_best_only=True,
mode='max')
earlystop = EarlyStopping(monitor='val_auc', patience=3, mode='max')
model.compile(
optimizer,
"binary_crossentropy",
metrics=['binary_crossentropy', auc],
)
print(model.summary())
if args.is_train:
history = model.fit(
train_model_input,
train_set[target].values,
def train(
batch_build_length: int,
batch_size: int,
metadata: dict,
model_config: Dict,
output_dir: str,
test_data: Data,
train_data: Data,
initial_epoch: int = 0,
initial_weights: str = "",
tensorboard_verbose: bool = False,
train_epochs: int = 10,
train_steps_per_epoch_limit: int = None,
validation_freq: int = 1,
verbose_logging: bool = False,
):
if initial_epoch > 0 and initial_weights:
raise ValueError(
"Only one of: initial_epoch or initial_weights - can be set at once"
)
model_arch_name = model_config["model_arch_name"]
weights_best_filename = os.path.join(output_dir, "model_weights.h5")
checkpoint_filename = os.path.join(output_dir, "model_checkpoint.h5")
logdir = os.path.join(output_dir, "tf_logs")
if initial_epoch > 0:
if os.path.exists(checkpoint_filename):
logger.info("Loading model: {}".format(checkpoint_filename))
model = tf.keras.models.load_model(checkpoint_filename)
else:
raise RuntimeError(
"{} not found, you should start from scratch".format(
checkpoint_filename))
else: # initial_epoch == 0
patch_size = model_config["patch_size"]
feature_shape = (
patch_size[0],
patch_size[1],
patch_size[2],
len(model_config["stencil_channels"]),
)
model_make_function = models_collection[model_arch_name]
model_params = deepcopy(model_config)
model_params.update({
"feature_shape": feature_shape,
"voxel_size": tuple(train_data.voxel_size),
})
model = model_make_function(params=model_params)
model.compile(optimizer="adam", loss="mse", metrics=["mse"])
ensure_dir(output_dir)
with open(os.path.join(output_dir, "model.json"), "w") as json_file:
json_file.write(model.to_json())
metadata["model_params"] = model_params
with open(os.path.join(output_dir, "model_metadata.json"),
"w") as json_file:
json.dump(metadata, json_file, indent=4, sort_keys=True)
if initial_weights:
logger.info("Loading weights: {}".format(initial_weights))
model.load_weights(initial_weights)
if os.path.exists(logdir):
shutil.rmtree(logdir)
checkpoint_best_callback = ModelCheckpoint(
filepath=weights_best_filename,
monitor="val_loss",
verbose=1,
save_best_only=True,
save_weights_only=True,
mode="auto",
period=1,
)
checkpoint_all_callback = ModelCheckpoint(
filepath=checkpoint_filename,
verbose=1,
save_best_only=False,
save_weights_only=False,
mode="auto",
period=1,
)
tensorboard_callback = TensorBoard(
log_dir=logdir,
histogram_freq=25 if tensorboard_verbose else 0,
write_grads=tensorboard_verbose,
write_images=tensorboard_verbose,
profile_batch=0,
)
def process_timings(epoch, logs):
this_epoch_train_timings = train_data.get_and_clean_timings()
this_epoch_test_timings = test_data.get_and_clean_timings()
update_timings_dict(TIMINGS_TRAIN, this_epoch_train_timings)
update_timings_dict(TIMINGS_VALIDATE, this_epoch_test_timings)
logger.info("Epoch {}".format(epoch))
logger.info("Data.train timings: {}".format(
pformat(this_epoch_train_timings)))
logger.info("Data.test timings: {}".format(
pformat(this_epoch_test_timings)))
process_timings_callback = LambdaCallback(
on_epoch_begin=lambda epoch, logs: print(),
on_epoch_end=process_timings)
make_batch_function = (make_batch_swap_axes if model_arch_name
in ("planar_first",
"first_baseline") else make_batch)
train_steps_per_epoch = math.ceil(len(train_data) / batch_size)
if train_steps_per_epoch_limit is not None:
train_steps_per_epoch = min(train_steps_per_epoch,
train_steps_per_epoch_limit)
initial_train_data_batch_start_index = (
initial_epoch * train_steps_per_epoch * batch_size) % len(train_data)
initial_data_shuffle_random_seed = (initial_epoch * train_steps_per_epoch *
batch_size) // len(train_data)
callbacks = [
checkpoint_best_callback,
checkpoint_all_callback,
tensorboard_callback,
process_timings_callback,
]
model.fit_generator(
generator=make_batches_gen(
data=train_data,
data_name="train_data",
batch_size=batch_size,
batch_build_length=batch_build_length,
initial_data_batch_start_index=initial_train_data_batch_start_index,
initial_data_shuffle_random_seed=initial_data_shuffle_random_seed,
make_batch_function=make_batch_function,
reshuffle=True,
),
steps_per_epoch=train_steps_per_epoch,
validation_data=make_batches_gen(
data=test_data,
data_name="validation_data",
batch_size=batch_size,
make_batch_function=make_batch_function,
reshuffle=False,
),
validation_freq=validation_freq,
validation_steps=math.ceil(len(test_data) / batch_size),
epochs=train_epochs,
verbose=2,
callbacks=callbacks,
initial_epoch=initial_epoch,
)
