def test_initial_epoch_set():
model, x_train, y_train = build_model_and_data()
# Cannot set initial_epoch
with pytest.raises(AssertionError):
resumable_model = ResumableModel(model,
save_every_epochs=4,
to_path='/tmp/nonexistentmodel.h5')
history = resumable_model.fit(x=x_train,
y=y_train,
validation_split=0.1,
batch_size=256,
verbose=2,
epochs=12,
initial_epoch=1)
def test_negative_epochs():
# Cannot have negative epochs
model, _, _ = build_model_and_data()
with pytest.raises(AssertionError):
resumable_model = ResumableModel(model,
save_every_epochs=-4,
to_path='/tmp/nonexistentmodel.h5')
def single_run(num_epochs,
save_every_epochs,
custom_loss=None,
to_path='/tmp/mymodel.h5'):
model, x_train, y_train = build_model_and_data(custom_loss=custom_loss)
prefix = os.path.splitext(to_path)[0]
epoch_num_file = prefix + "_epoch_num.pkl"
history_file = prefix + "_history.pkl"
filesCreated = [to_path, epoch_num_file, history_file]
# test resumable model with random data
for path in filesCreated:
if os.path.exists(path) and os.path.isfile(path):
os.remove(path)
elif os.path.exists(path) and os.path.isdir(path):
shutil.rmtree(path)
resumable_model = ResumableModel(
model,
save_every_epochs=save_every_epochs,
to_path=to_path,
custom_objects={'custom_loss': custom_loss})
history = resumable_model.fit(x=x_train,
y=y_train,
validation_split=0.1,
batch_size=256,
verbose=2,
epochs=num_epochs)
assert history != {}
# Check the saved history is identical to the one returned
assert history == pickle.load(open(history_file, 'rb'))
assert isinstance(history, dict)
for k, v in history.items():
assert len(v) == num_epochs
for path in filesCreated:
assert os.path.exists(path)
if os.path.isfile(path):
os.remove(path)
else:
shutil.rmtree(path)
def test_resumable_model_interrupted_run():
model, x_train, y_train = build_model_and_data()
filesCreated = ['/tmp/mymodel.h5', '/tmp/mymodel_epoch_num.pkl', '/tmp/mymodel_history.pkl']
# test resumable model with random data
NUM_EPOCHS = 12
for filePath in filesCreated:
if os.path.exists(filePath):
os.remove(filePath)
resumable_model = ResumableModel(model, save_every_epochs=4, to_path='/tmp/mymodel.h5')
history = resumable_model.fit(x=x_train, y=y_train, validation_split=0.1, batch_size=256, verbose=2, epochs=NUM_EPOCHS)
assert history != {}
assert isinstance(history, dict)
for k, v in history.items():
assert len(v) == NUM_EPOCHS
for filePath in filesCreated:
assert os.path.exists(filePath)
# add more epochs and retrain (it should open the old history dicts and keep adding)
NUM_MORE_EPOCHS = 20
NUM_NEW_EPOCHS = NUM_EPOCHS + NUM_MORE_EPOCHS
model, _, _ = build_model_and_data() # refresh model so weights are lost
resumable_model = ResumableModel(model, save_every_epochs=4, to_path='/tmp/mymodel.h5') # recover model
new_history = resumable_model.fit(x=x_train, y=y_train, validation_split=0.1, batch_size=256, verbose=2, epochs=NUM_NEW_EPOCHS)
# Ensure the new history dict is a longer version of the older history dict
old_history = history
for k, v in new_history.items():
assert len(v) == NUM_NEW_EPOCHS
assert k in old_history
assert old_history[k] == v[:len(old_history[k])]
for filePath in filesCreated:
os.remove(filePath)
def single_run(num_epochs, save_every_epochs):
model, x_train, y_train = build_model_and_data()
filesCreated = ['/tmp/mymodel.h5', '/tmp/mymodel_epoch_num.pkl', '/tmp/mymodel_history.pkl']
# test resumable model with random data
for filePath in filesCreated:
if os.path.exists(filePath):
os.remove(filePath)
x_train = np.random.rand(1000, 16)
y_train = np.random.randint(2, size=1000)
resumable_model = ResumableModel(model, save_every_epochs=save_every_epochs, to_path='/tmp/mymodel.h5')
history = resumable_model.fit(x=x_train, y=y_train, validation_split=0.1, batch_size=256, verbose=2, epochs=num_epochs)
assert history != {}
# Check the saved history is identical to the one returned
assert history == pickle.load(open('/tmp/mymodel_history.pkl', 'rb'))
assert isinstance(history, dict)
for k, v in history.items():
assert len(v) == num_epochs
for filePath in filesCreated:
assert os.path.exists(filePath)
for filePath in filesCreated:
os.remove(filePath)
def main():
print("START")
if not os.path.exists(OutputPathModels):
os.makedirs(OutputPathModels)
loadParametersFromFile("PARAMETERS_CNN.txt")
#callback=EarlyStopping(monitor='val_acc', min_delta=0, patience=0, verbose=0, mode='auto', baseline=None)
# custom early stop is incompatible with keras-buoy, but built-in early stop does not work for this project, falling back to custom early stop with val_acc
earlystop = EarlyStoppingByLossVal(monitor='val_acc',
value=0.975,
verbose=1,
lower=False)
#earlystop = EarlyStopping(monitor='val_acc',min_delta=0,baseline=0.975,restore_best_weights=False,patience=0,mode='max',verbose=1)
# no need for modelcheckpoint callback
# modelcheckpoint = ModelCheckpoint(f'{OutputPathModels}/checkpoints/checkpoint_.h5',monitor='val_accuracy',verbose=1,save_best_only=False,save_weights_only=False)
print("Parameters loaded")
if args.resume and os.path.exists(
f'{OutputPathModels}/resume_indices.txt'):
with open(f'{OutputPathModels}/resume_indices.txt', 'r') as resf:
resind = resf.readline()
indPat = int(resind.split('.')[0])
indi = int(resind.split('.')[1])
else:
indPat = 0
indi = 0
for indexPat in range(indPat, len(patients)):
print('Patient ' + patients[indexPat])
if not os.path.exists(OutputPathModels + "ModelPat" +
patients[indexPat] + "/"):
os.makedirs(OutputPathModels + "ModelPat" + patients[indexPat] +
"/")
os.makedirs(OutputPathModels + "ModelPat" + patients[indexPat] +
"/checkpoints/")
else:
if not args.resume:
print('You said to not resume, deleting checkpoints...')
# This will delete the pkl, h5 and txt file used to store checkpoints
if os.listdir(OutputPathModels + "ModelPat" +
patients[indexPat] + "/checkpoints/"):
os.remove(
os.path.join(OutputPathModels, 'resume_indices.txt'))
for f in os.listdir(OutputPathModels + "ModelPat" +
patients[indexPat] + "/checkpoints/"):
os.remove(
os.path.join(
OutputPathModels + "ModelPat" +
patients[indexPat] + "/checkpoints/", f))
loadSpectogramData(indexPat)
print('Spectograms data loaded')
filesPath, groups = getFilesPathWithGroup(indexPat, shuffles=True)
logo = LeaveOneGroupOut()
result = 'Patient ' + patients[indexPat] + '\n'
result = 'Out Seizure, True Positive, False Positive, False negative, Second of Inter in Test, Sensitivity, FPR \n'
#for i in range(indi, nSeizure):
for i in range(0, 1):
print('SEIZURE OUT: ' + str(i + 1))
with open(f'{OutputPathModels}/resume_indices.txt', 'w') as resf:
resf.write(f'{indexPat}.{i}')
# create the model and make it resumable
model = createModel()
resumable_model = ResumableModel(
model,
save_every_epochs=1,
to_path=
f'{OutputPathModels}ModelPat{patients[indexPat]}/checkpoints/model_checkpoint_s{i}.h5'
)
finalWeightsOutputPath = WeightsOutputPath + f'/paz{indexPat+1}/seizure{i+1}'
if args.save_weights:
# Define output dir for weights based on patient and seizure
weights_callback = SaveCompressedWeightsNetwork(
finalWeightsOutputPath, resume=args.resume)
print('You asked to save weights, adding callback...')
callback = [earlystop, weights_callback]
else:
print('Defaulting callback to earlystop...')
callback = [earlystop]
print('Training start')
# Cross-validation sklearn
'''
model = KerasClassifier(build_fn=createModel,shuffle=True,callbacks=callback,validation_data=generate_arrays_for_training(indexPat, filesPath, start=75),
steps_per_epoch=int((len(filesPath)-int(len(filesPath)/100*25))),
validation_steps=int((len(filesPath)-int(len(filesPath)/100*75)))
)
params_grid = dict(
verbose=[1],
epochs=[300],
max_queue_size=[2]
)
Xtrain,ytrain = generate_arrays_for_training(indexPat, filesPath, end=75)
search = GridSearchCV(model,params_grid,n_jobs=-1,cv=logo,verbose=4,refit=True)
search.fit(Xtrain,ytrain,groups=groups[0:len(Xtrain)])
print('number of total CV splits:', search.n_splits_)
print('best estimator:',search.best_estimator_)
print('best score:',search.best_score_)
'''
Xtrain, ytrain = generate_arrays_for_training(indexPat, filesPath)
#Xval,yval = generate_arrays_for_training(indexPat, filesPath, start=75)
history = resumable_model.fit(
Xtrain,
ytrain, #end=75),#It take the first 75%
validation_split=0.25, #start=75), #It take the last 25%
#steps_per_epoch=10000, epochs=10)
steps_per_epoch=int((len(filesPath) -
int(len(filesPath) / 100 * 25))), #*25),
validation_steps=int(
(len(filesPath) - int(len(filesPath) / 100 * 75))), #*75),
verbose=1, #no progress bar -> faster
epochs=300,
max_queue_size=2,
shuffle=True,
callbacks=callback
) # 100 epochs è meglio #aggiungere criterio di stop in base accuratezza
print('Training end')
# Save model history
with open(f'{finalWeightsOutputPath}_hist.pkl', 'wb') as h:
pickle.dump(history, h)
print('Testing start')
filesPath = interictalSpectograms[i]
#Xtest,ytest = generate_arrays_for_predict(indexPat,filesPath)
interPrediction = resumable_model.predict(
generate_arrays_for_predict(indexPat, filesPath),
max_queue_size=4,
steps=len(filesPath))
filesPath = preictalRealSpectograms[i]
#Xtest,ytest = generate_arrays_for_predict(indexPat,filesPath)
preictPrediction = resumable_model.predict(
generate_arrays_for_predict(indexPat, filesPath),
max_queue_size=4,
steps=len(filesPath))
print('Testing end')
# Creates a HDF5 file
model.save(OutputPathModels + "ModelPat" + patients[indexPat] +
"/" + 'ModelOutPatient' + str(indexPat + 1) + '.h5')
print("Model saved")
#to plot the model
#plot_model(model, to_file="CNNModel", show_shapes=True, show_layer_names=True)
if not os.path.exists(OutputPathModels + "OutputTest" + "/"):
os.makedirs(OutputPathModels + "OutputTest" + "/")
np.savetxt(OutputPathModels + "OutputTest" + "/" + "Int_" +
patients[indexPat] + "_" + str(i + 1) + ".csv",
interPrediction,
delimiter=",")
np.savetxt(OutputPathModels + "OutputTest" + "/" + "Pre_" +
patients[indexPat] + "_" + str(i + 1) + ".csv",
preictPrediction,
delimiter=",")
secondsInterictalInTest = len(
interictalSpectograms[i]
) * 50 * 30 #50 spectograms for file, 30 seconds for each spectogram
acc = 0 #accumulator
fp = 0
tp = 0
fn = 0
lastTenResult = list()
for el in interPrediction:
if (el[1] > 0.5):
acc = acc + 1
lastTenResult.append(1)
else:
lastTenResult.append(0)
if (len(lastTenResult) > 10):
acc = acc - lastTenResult.pop(0)
if (acc >= 8):
fp = fp + 1
lastTenResult = list()
acc = 0
lastTenResult = list()
for el in preictPrediction:
if (el[1] > 0.5):
acc = acc + 1
lastTenResult.append(1)
else:
lastTenResult.append(0)
if (len(lastTenResult) > 10):
acc = acc - lastTenResult.pop(0)
if (acc >= 8):
tp = tp + 1
else:
if (len(lastTenResult) == 10):
fn = fn + 1
sensitivity = tp / (tp + fn)
FPR = fp / (secondsInterictalInTest / (60 * 60))
result = result + str(i + 1) + ',' + str(tp) + ',' + str(
fp) + ',' + str(fn) + ',' + str(secondsInterictalInTest) + ','
result = result + str(sensitivity) + ',' + str(FPR) + '\n'
print(
'True Positive, False Positive, False negative, Second of Inter in Test, Sensitivity, FPR'
)
print(
str(tp) + ',' + str(fp) + ',' + str(fn) + ',' +
str(secondsInterictalInTest) + ',' + str(sensitivity) + ',' +
str(FPR))
indi = 0
with open(OutputPath, "a+") as myfile:
myfile.write(result)
def test_path_ending():
model, _, _ = build_model_and_data()
# Path should end with .h5
with pytest.raises(AssertionError):
resumable_model = ResumableModel(model, save_every_epochs=4, to_path='/tmp/mymodel')