def evaluate(self, data_dir, txt_file):
# The format of txt should match file in the dataset folder
# TXT should contain the label to have the evaluate accuracy
# ----- SETTINGS: DATASET BUILDERS
datasetVal = MyDatasetGenerator(data_dir, txt_file,
self.TransformSequenceVal)
dataLoaderVal = DataLoader(dataset=datasetVal,
batch_size=8,
shuffle=False,
num_workers=8,
pin_memory=True)
outGT = torch.LongTensor().to(device)
outPRED = torch.LongTensor().to(device)
# ----- SETTINGS: LOSS FUNCTION
loss = torch.nn.CrossEntropyLoss()
lossVal = 0
lossValNorm = 0
#losstensorMean = 0
with torch.no_grad():
for i, (input, target) in enumerate(dataLoaderVal):
# target = target.cuda(async=True)
target = target.flatten()
target = target.to(device)
varInput = input.to(device)
varOutput = self.model(varInput)
varOutput_softmax = torch.nn.functional.log_softmax(varOutput,
dim=1)
varOutput_class = torch.max(varOutput_softmax, dim=1).indices
losstensor = loss(varOutput, target)
lossVal += losstensor.item()
lossValNorm += 1
outGT = torch.cat((outGT, target), 0)
outPRED = torch.cat((outPRED, varOutput_class), 0)
# print(f'GT shape: {outGT.shape} - PRED shape: {outPRED.shape}')
outLoss = lossVal / lossValNorm
accuracy = torch.sum(outGT == outPRED).item() / outGT.size()[0]
# losstensorMean = losstensorMean / lossValNorm
return outLoss, accuracy
def evaluate(self, data_df):
# ----- SETTINGS: DATASET BUILDERS
datasetVal = MyDatasetGenerator(df=data_df)
dataLoaderVal = DataLoader(dataset=datasetVal,
batch_size=4,
shuffle=False,
num_workers=8,
pin_memory=True)
outGT = torch.LongTensor().to(device)
outPRED = torch.LongTensor().to(device)
# ----- SETTINGS: LOSS FUNCTION
loss = torch.nn.CrossEntropyLoss()
lossVal = 0
lossValNorm = 0
#losstensorMean = 0
# self.model.eval()
with torch.no_grad():
for i, (img, tab, target) in enumerate(dataLoaderVal):
# target = target.cuda(async=True)
target = target.flatten()
target = target.to(device)
varImgInput = img.to(device)
varTabInput = tab.to(device)
varOutput = self.model(varImgInput, varTabInput)
varOutput_softmax = torch.nn.functional.log_softmax(varOutput,
dim=1)
varOutput_class = torch.max(varOutput_softmax, dim=1).indices
losstensor = loss(varOutput, target)
lossVal += losstensor.item()
lossValNorm += 1
outGT = torch.cat((outGT, target), 0)
outPRED = torch.cat((outPRED, varOutput_class), 0)
# print(f'GT shape: {outGT.shape} - PRED shape: {outPRED.shape}')
outLoss = lossVal / lossValNorm
accuracy = torch.sum(outGT == outPRED).item() / outGT.size()[0]
# losstensorMean = losstensorMean / lossValNorm
return outLoss, accuracy
def train(self):
# ----- SETTINGS: LOAD PRETRAINED MODEL
self.load_trained_model()
# ----- SETTINGS: DATASET BUILDERS
datasetTrain = MyDatasetGenerator(pathImageDirectory=self.data_dir,
pathDatasetFile=self.train_txt,
transform=self.TransformSequence)
datasetVal = MyDatasetGenerator(pathImageDirectory=self.data_dir,
pathDatasetFile=self.val_txt,
transform=self.TransformSequenceVal)
dataLoaderTrain = DataLoader(dataset=datasetTrain,
batch_size=self.batch_size,
shuffle=True,
num_workers=8,
pin_memory=True)
dataLoaderVal = DataLoader(dataset=datasetVal,
batch_size=8,
shuffle=False,
num_workers=8,
pin_memory=True)
# ----- SETTINGS: OPTIMIZER & SCHEDULER
if self.optimizer.lower() == 'adam':
optimizer = optim.Adam(self.model.parameters(),
lr=self.learning_rate,
betas=(0.9, 0.999),
eps=1e-08,
weight_decay=1e-5)
if self.optimizer.lower() == 'sgd':
optimizer = optim.SGD(self.model.parameters(),
lr=self.learning_rate,
momentum=0.9,
weight_decay=1e-5)
scheduler = ReduceLROnPlateau(optimizer,
factor=0.2,
patience=4,
mode='min',
verbose=True,
min_lr=1e-5)
# ----- SETTINGS: LOSS FUNCTION
loss = torch.nn.CrossEntropyLoss()
# ----- SETTINGS: REPORT
timestampTime = time.strftime("%H%M%S")
timestampDate = time.strftime("%d%m%Y")
self.timestampLaunch = timestampDate + '-' + timestampTime
f_log = open(
f"{self.save_dir}/{self.model_name}-{self.timestampLaunch}-REPORT.log",
"w")
writer = SummaryWriter(
f'{self.save_dir}/{self.model_name}-{self.timestampLaunch}/')
self.extract_info()
# ---- TRAIN THE NETWORK
lossMIN = 100000
accMax = 0
flag = 0
count_change_loss = 0
for epochID in range(0, self.max_epoch):
if epochID > int(self.max_epoch / 2) and flag == 0:
flag = 1
optimizer = optim.SGD(self.model.parameters(),
lr=0.001,
momentum=0.9,
weight_decay=1e-5)
scheduler = ReduceLROnPlateau(optimizer,
factor=0.5,
patience=3,
mode='min',
verbose=True,
min_lr=1e-5)
print(f"Training {epochID}/{self.max_epoch-1}")
timestampTime = time.strftime("%H%M%S")
timestampDate = time.strftime("%d%m%Y")
timestampSTART = timestampDate + '-' + timestampTime
lossTrain = self.epochTrain(dataLoaderTrain, optimizer, loss)
lossVal, acc = self.epochVal(dataLoaderVal, loss)
timestampTime = time.strftime("%H%M%S")
timestampDate = time.strftime("%d%m%Y")
timestampEND = timestampDate + '-' + timestampTime
scheduler.step(lossVal + 2 * (1 - acc))
if lossVal < lossMIN or acc > accMax:
count_change_loss = 0
if lossVal < lossMIN:
lossMIN = lossVal
if acc > accMax:
accMax = acc
torch.save(
{
'epoch': epochID + 1,
'state_dict': self.model.state_dict(),
'best_loss': lossMIN,
'best_acc': accMax
},
f"{self.save_dir}/{self.model_name}-{self.timestampLaunch}.pth.tar"
)
print('Epoch [' + str(epochID + 1) + '] [save] [' +
timestampEND + '] lossVal= ' + str(lossVal) +
' --- ACC: ' + str(acc))
f_log.write(
f"[{timestampEND} - {epochID+1} - SAVE]\nLoss_Train: {lossTrain}\nLoss_Val: {lossVal}\nACC_Val: {acc}\n"
)
else:
count_change_loss += 1
print('Epoch [' + str(epochID + 1) + '] [----] [' +
timestampEND + '] lossVal= ' + str(lossVal) +
' --- ACC: ' + str(acc))
f_log.write(
f"[{timestampEND} - {epochID+1}]\nLoss_Train: {lossTrain}\nLoss_Val: {lossVal}\nACC_Val: {acc}\n"
)
writer.add_scalars('Loss', {'train': lossTrain}, epochID)
writer.add_scalars('Loss', {'val': lossVal}, epochID)
writer.add_scalars('Loss', {'val-best': lossMIN}, epochID)
writer.add_scalars('Accuracy', {'val': acc}, epochID)
current_lr = optimizer.param_groups[0]['lr']
writer.add_scalar('Learning Rate', current_lr, epochID)
if count_change_loss >= 25:
print(
f'Early stopping: {count_change_loss} epoch not decrease the loss'
)
break
f_log.close()
writer.close()
def train(self):
# ----- SETTINGS: LOAD PRETRAINED MODEL
self.load_trained_model()
# ----- SETTINGS: DATASET BUILDERS
datasetTrain = MyDatasetGenerator(pathImageDirectory=self.data_dir,
pathDatasetFile=self.train_txt,
transform=self.TransformSequence)
datasetVal = MyDatasetGenerator(pathImageDirectory=self.data_dir,
pathDatasetFile=self.val_txt,
transform=self.TransformSequenceVal)
# ----- SETTINGS: REPORT
timestampTime = time.strftime("%H%M%S")
timestampDate = time.strftime("%d%m%Y")
self.timestampLaunch = timestampDate + '-' + timestampTime
self.extract_info()
writer = SummaryWriter(
f'{self.save_dir}/{self.model_name}-{self.timestampLaunch}/')
# ----- SETTINGS: CALLBACKS
lrscheduler = LRScheduler(policy=ReduceLROnPlateau,
factor=0.2,
patience=4,
mode='min',
verbose=True,
min_lr=1e-5,
monitor='valid_loss')
# auc = EpochScoring(scoring='roc_auc', lower_is_better=False)
checkpoint = Checkpoint(
f_params=
f"{self.save_dir}/{self.model_name}-{self.timestampLaunch}.pth.tar",
monitor='valid_acc_best',
f_history=
f"{self.save_dir}/{self.model_name}-{self.timestampLaunch}-REPORT.json"
)
earlystopping = EarlyStopping(monitor='valid_loss', patience=15)
tensorboard = TensorBoard(writer)
# ----- SETTINGS: DECLARE SKORCH
if self.optimizer.lower() == 'adam':
net = NeuralNetClassifier(
self.model,
criterion=torch.nn.CrossEntropyLoss,
lr=self.learning_rate,
batch_size=self.batch_size,
max_epochs=self.max_epoch,
optimizer=optim.Adam,
#optimizer__params=self.model.parameters(),
optimizer__betas=(0.9, 0.999),
optimizer__eps=1e-8,
optimizer__weight_decay=1e-5,
iterator_train__shuffle=True,
iterator_train__num_workers=8,
iterator_valid__shuffle=True,
iterator_valid__num_workers=8,
train_split=predefined_split(datasetVal),
callbacks=[
lrscheduler, checkpoint, earlystopping, tensorboard
],
device='cuda' # comment to train on cpu
)
if self.optimizer.lower() == 'sgd':
net = NeuralNetClassifier(
self.model,
criterion=torch.nn.CrossEntropyLoss,
lr=self.learning_rate,
batch_size=self.batch_size,
max_epochs=self.max_epoch,
optimizer=optim.SGD,
#optimizer__params=self.model.parameters(),
optimizer__momentum=0.9,
optimizer__weight_decay=1e-5,
iterator_train__shuffle=True,
iterator_train__num_workers=8,
iterator_valid__shuffle=True,
iterator_valid__num_workers=8,
train_split=predefined_split(datasetVal),
callbacks=[
lrscheduler, checkpoint, earlystopping, tensorboard
],
device='cuda' # comment to train on cpu
)
# y = np.array([y for _, y in iter(datasetTrain)])
net.fit(datasetTrain, y=None)