def validate_epoch(val_loader, model, criterion, epoch, device):
model.eval()
val_loss = 0
metric = Metric(['accuracy', 'mean_iou'], len(val_loader.dataset.classes))
for image, label in tqdm(val_loader, total=len(val_loader)):
image = image.to(device)
label = label.to(device)
with torch.no_grad():
pred = model(image)
loss = criterion(pred, label) / len(image)
val_loss += loss.item()
metric.update(pred.data.cpu().numpy(), label.data.cpu().numpy())
metrics = metric.compute()
return val_loss / len(val_loader), metrics['accuracy'], metrics['mean_iou']
class PostProcessing():
def __init__(self, fold):
self.fold = fold
self.threshold = float(open(f"threshold_{self.fold}.txt",
"r").read()) #0.5#0.1
self.metric = Metric()
def run(self, predictions):
predictions_processed = predictions.copy()
#if somth is found, its not a normal
predictions_processed[np.where(
predictions_processed >= self.threshold)] = 1
predictions_processed[np.where(
predictions_processed < self.threshold)] = 0
return predictions_processed
def find_opt_thresold(self, labels, outputs):
threshold_grid = np.arange(0.05, 0.99, 0.05).tolist()
threshold_opt = np.zeros((27))
unit_threshold = partial(self._unit_threshold,
labels=labels,
outputs=outputs)
start = time.time()
with ProcessPoolExecutor(max_workers=20) as pool:
result = pool.map(unit_threshold, threshold_grid)
scores = list(result)
print(f'Processing time: {(time.time() - start)/60}')
# print('Finding the optimal threshold')
# for threshold in tqdm(threshold_grid):
#
# predictions = outputs.copy()
#
# predictions[np.where(predictions >= threshold)] = 1
# predictions[np.where(predictions < threshold)] = 0
#
# scores.append(self.metric.compute(labels, predictions))
scores = np.array(scores)
a = np.where(scores == np.max(scores))
if len(a) > 1:
a = [0]
threshold_opt = threshold_grid[a[0]]
else:
threshold_opt = threshold_grid[a[0][0]]
return threshold_opt
def _unit_threshold(self, threshold, labels, outputs):
predictions = outputs.copy()
predictions[np.where(predictions >= threshold)] = 1
predictions[np.where(predictions < threshold)] = 0
return self.metric.compute(labels, predictions)
def update_threshold(self, threshold):
f = open(f"threshold_{self.fold}.txt", "w")
f.write(str(threshold))
f.close()
self.threshold = threshold
class Model:
"""
This class handles basic methods for handling the model:
1. Fit the model
2. Make predictions
3. Save
4. Load
"""
def __init__(self, input_size, n_channels, hparams):
self.hparams = hparams
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
# define the models
self.model = WaveNet(n_channels=n_channels).to(self.device)
summary(self.model, (input_size, n_channels))
# self.model.half()
if torch.cuda.device_count() > 1:
print("Number of GPUs will be used: ",
torch.cuda.device_count() - 3)
self.model = DP(self.model,
device_ids=list(
range(torch.cuda.device_count() - 3)))
else:
print('Only one GPU is available')
self.metric = Metric()
self.num_workers = 1
########################## compile the model ###############################
# define optimizer
self.optimizer = torch.optim.Adam(params=self.model.parameters(),
lr=self.hparams['lr'],
weight_decay=1e-5)
# weights = torch.Tensor([0.025,0.033,0.039,0.046,0.069,0.107,0.189,0.134,0.145,0.262,1]).cuda()
self.loss = nn.BCELoss() # CompLoss(self.device)
# define early stopping
self.early_stopping = EarlyStopping(
checkpoint_path=self.hparams['checkpoint_path'] + '/checkpoint.pt',
patience=self.hparams['patience'],
delta=self.hparams['min_delta'],
)
# lr cheduler
self.scheduler = ReduceLROnPlateau(
optimizer=self.optimizer,
mode='max',
factor=0.2,
patience=3,
verbose=True,
threshold=self.hparams['min_delta'],
threshold_mode='abs',
cooldown=0,
eps=0,
)
self.seed_everything(42)
self.threshold = 0.75
self.scaler = torch.cuda.amp.GradScaler()
def seed_everything(self, seed):
np.random.seed(seed)
os.environ['PYTHONHASHSEED'] = str(seed)
torch.manual_seed(seed)
def fit(self, train, valid):
train_loader = DataLoader(
train,
batch_size=self.hparams['batch_size'],
shuffle=True,
num_workers=self.num_workers) # ,collate_fn=train.my_collate
valid_loader = DataLoader(
valid,
batch_size=self.hparams['batch_size'],
shuffle=False,
num_workers=self.num_workers) # ,collate_fn=train.my_collate
# tensorboard object
writer = SummaryWriter()
for epoch in range(self.hparams['n_epochs']):
# trian the model
self.model.train()
avg_loss = 0.0
train_preds, train_true = torch.Tensor([]), torch.Tensor([])
for (X_batch, y_batch) in tqdm(train_loader):
y_batch = y_batch.float().to(self.device)
X_batch = X_batch.float().to(self.device)
self.optimizer.zero_grad()
# get model predictions
pred = self.model(X_batch)
X_batch = X_batch.cpu().detach()
# process loss_1
pred = pred.view(-1, pred.shape[-1])
y_batch = y_batch.view(-1, y_batch.shape[-1])
train_loss = self.loss(pred, y_batch)
y_batch = y_batch.float().cpu().detach()
pred = pred.float().cpu().detach()
train_loss.backward(
) #self.scaler.scale(train_loss).backward() #
# torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1)
# torch.nn.utils.clip_grad_value_(self.model.parameters(), 0.5)
self.optimizer.step() # self.scaler.step(self.optimizer) #
self.scaler.update()
# calc metric
avg_loss += train_loss.item() / len(train_loader)
train_true = torch.cat([train_true, y_batch], 0)
train_preds = torch.cat([train_preds, pred], 0)
# calc triaing metric
train_preds = train_preds.numpy()
train_preds[np.where(train_preds >= self.threshold)] = 1
train_preds[np.where(train_preds < self.threshold)] = 0
metric_train = self.metric.compute(labels=train_true.numpy(),
outputs=train_preds)
# evaluate the model
print('Model evaluation...')
self.model.zero_grad()
self.model.eval()
val_preds, val_true = torch.Tensor([]), torch.Tensor([])
avg_val_loss = 0.0
with torch.no_grad():
for X_batch, y_batch in valid_loader:
y_batch = y_batch.float().to(self.device)
X_batch = X_batch.float().to(self.device)
pred = self.model(X_batch)
X_batch = X_batch.float().cpu().detach()
pred = pred.reshape(-1, pred.shape[-1])
y_batch = y_batch.view(-1, y_batch.shape[-1])
avg_val_loss += self.loss(
pred, y_batch).item() / len(valid_loader)
y_batch = y_batch.float().cpu().detach()
pred = pred.float().cpu().detach()
val_true = torch.cat([val_true, y_batch], 0)
val_preds = torch.cat([val_preds, pred], 0)
# evalueate metric
val_preds = val_preds.numpy()
val_preds[np.where(val_preds >= self.threshold)] = 1
val_preds[np.where(val_preds < self.threshold)] = 0
metric_val = self.metric.compute(val_true.numpy(), val_preds)
self.scheduler.step(avg_val_loss)
res = self.early_stopping(score=avg_val_loss, model=self.model)
# print statistics
if self.hparams['verbose_train']:
print(
'| Epoch: ',
epoch + 1,
'| Train_loss: ',
avg_loss,
'| Val_loss: ',
avg_val_loss,
'| Metric_train: ',
metric_train,
'| Metric_val: ',
metric_val,
'| Current LR: ',
self.__get_lr(self.optimizer),
)
# # add history to tensorboard
writer.add_scalars(
'Loss',
{
'Train_loss': avg_loss,
'Val_loss': avg_val_loss
},
epoch,
)
writer.add_scalars('Metric', {
'Metric_train': metric_train,
'Metric_val': metric_val
}, epoch)
if res == 2:
print("Early Stopping")
print(
f'global best min val_loss model score {self.early_stopping.best_score}'
)
break
elif res == 1:
print(f'save global val_loss model score {avg_val_loss}')
writer.close()
self.model.zero_grad()
return True
def predict(self, X_test):
# evaluate the model
self.model.eval()
test_loader = torch.utils.data.DataLoader(
X_test,
batch_size=self.hparams['batch_size'],
shuffle=False,
num_workers=self.num_workers) # ,collate_fn=train.my_collate
test_preds = torch.Tensor([])
print('Start generation of predictions')
with torch.no_grad():
for i, (X_batch, y_batch) in enumerate(tqdm(test_loader)):
X_batch = X_batch.float().to(self.device)
pred = self.model(X_batch)
X_batch = X_batch.float().cpu().detach()
test_preds = torch.cat([test_preds, pred.cpu().detach()], 0)
return test_preds.numpy()
def get_heatmap(self, X_test):
# evaluate the model
self.model.eval()
test_loader = torch.utils.data.DataLoader(
X_test,
batch_size=self.batch_size,
shuffle=False,
num_workers=self.num_workers) # ,collate_fn=train.my_collate
test_preds = torch.Tensor([])
with torch.no_grad():
for i, (X_batch) in enumerate(test_loader):
X_batch = X_batch.float().to(self.device)
pred = self.model.activatations(X_batch)
pred = torch.sigmoid(pred)
X_batch = X_batch.float().cpu().detach()
test_preds = torch.cat([test_preds, pred.cpu().detach()], 0)
return test_preds.numpy()
def model_save(self, model_path):
torch.save(self.model, model_path)
return True
def model_load(self, model_path):
self.model = torch.load(model_path)
return True
################## Utils #####################
def __get_lr(self, optimizer):
for param_group in optimizer.param_groups:
return param_group['lr']
class Model:
"""
This class handles basic methods for handling the model:
1. Fit the model
2. Make predictions
3. Save
4. Load
"""
def __init__(self, input_size, n_channels, hparams, gpu, inference=False):
self.hparams = hparams
if inference:
self.device = torch.device('cpu')
self.model = ECGNet(n_channels=n_channels,
hparams=self.hparams).to(self.device)
else:
if torch.cuda.device_count() > 1:
if len(gpu) > 0:
print("Number of GPUs will be used: ", len(gpu))
self.device = torch.device(f"cuda:{gpu[0]}" if torch.cuda.
is_available() else "cpu")
self.model = ECGNet(n_channels=n_channels,
hparams=self.hparams).to(self.device)
self.model = DP(self.model,
device_ids=gpu,
output_device=gpu[0])
else:
print("Number of GPUs will be used: ",
torch.cuda.device_count() - 5)
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
self.model = ECGNet(n_channels=n_channels,
hparams=self.hparams).to(self.device)
self.model = DP(self.model,
device_ids=list(
range(torch.cuda.device_count() - 5)))
else:
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
self.model = ECGNet(n_channels=n_channels,
hparams=self.hparams).to(self.device)
print('Only one GPU is available')
# define the models
#summary(self.model, (input_size, n_channels))
#print(torch.cuda.is_available())
self.metric = Metric()
self.num_workers = 18
self.threshold = 0.5
########################## compile the model ###############################
# define optimizer
self.optimizer = torch.optim.Adam(params=self.model.parameters(),
lr=self.hparams['lr'])
weights = torch.Tensor([
1., 1., 1., 1., 0.5, 1., 1., 1., 1., 1., 1., 1., 0.5, 0.5, 1., 1.,
1., 1., 0.5, 1., 1., 1., 1., 0.5, 1., 1., 0.5
]).to(self.device)
self.loss = nn.BCELoss(weight=weights) # CompLoss(self.device) #
self.decoder_loss = nn.MSELoss()
# define early stopping
self.early_stopping = EarlyStopping(
checkpoint_path=self.hparams['checkpoint_path'] + '/checkpoint' +
str(self.hparams['start_fold']) + '.pt',
patience=self.hparams['patience'],
delta=self.hparams['min_delta'],
is_maximize=True,
)
# lr cheduler
self.scheduler = ReduceLROnPlateau(
optimizer=self.optimizer,
mode='max',
factor=0.2,
patience=1,
verbose=True,
threshold=self.hparams['min_delta'],
threshold_mode='abs',
cooldown=0,
eps=0,
)
self.seed_everything(42)
self.postprocessing = PostProcessing(fold=self.hparams['start_fold'])
self.scaler = torch.cuda.amp.GradScaler()
def seed_everything(self, seed):
np.random.seed(seed)
os.environ['PYTHONHASHSEED'] = str(seed)
torch.manual_seed(seed)
def fit(self, train, valid):
train_loader = DataLoader(train,
batch_size=self.hparams['batch_size'],
shuffle=True,
num_workers=self.num_workers)
valid_loader = DataLoader(valid,
batch_size=self.hparams['batch_size'],
shuffle=False,
num_workers=self.num_workers)
# tensorboard object
writer = SummaryWriter()
for epoch in range(self.hparams['n_epochs']):
# trian the model
self.model.train()
avg_loss = 0.0
train_preds, train_true = torch.Tensor([]), torch.Tensor([])
for (X_batch, y_batch) in tqdm(train_loader):
y_batch = y_batch.float().to(self.device)
X_batch = X_batch.float().to(self.device)
self.optimizer.zero_grad()
# get model predictions
pred, pred_decoder = self.model(X_batch)
# process loss_1
pred = pred.view(-1, pred.shape[-1])
pred = pred**2
y_batch = y_batch.view(-1, y_batch.shape[-1])
train_loss = self.loss(pred, y_batch)
y_batch = y_batch.float().cpu().detach()
pred = pred.float().cpu().detach()
# process loss_2
pred_decoder = pred_decoder.view(-1, pred_decoder.shape[-1])
X_batch = X_batch.view(-1, X_batch.shape[-1])
decoder_train_loss = self.decoder_loss(pred_decoder, X_batch)
X_batch = X_batch.float().cpu().detach()
pred_decoder = pred_decoder.float().cpu().detach()
# calc loss
avg_loss += train_loss.item() / len(train_loader)
#sum up multi-head losses
train_loss = train_loss + decoder_train_loss
self.scaler.scale(
train_loss).backward() # train_loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1)
torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1)
torch.nn.utils.clip_grad_value_(self.model.parameters(), 0.5)
self.scaler.step(self.optimizer) # self.optimizer.step()
self.scaler.update()
train_true = torch.cat([train_true, y_batch], 0)
train_preds = torch.cat([train_preds, pred], 0)
# calc triaing metric
train_preds = train_preds.numpy()
train_true = train_true.numpy()
threshold = self.postprocessing.find_opt_thresold(
train_true, train_preds)
self.postprocessing.update_threshold(threshold)
train_preds = self.postprocessing.run(train_preds)
metric_train = self.metric.compute(labels=train_true,
outputs=train_preds)
# evaluate the model
print('Model evaluation...')
self.model.eval()
val_preds, val_true = torch.Tensor([]), torch.Tensor([])
avg_val_loss = 0.0
with torch.no_grad():
for X_batch, y_batch in valid_loader:
y_batch = y_batch.float().to(self.device)
X_batch = X_batch.float().to(self.device)
pred, pred_decoder = self.model(X_batch)
pred_decoder = pred_decoder.float().cpu().detach()
X_batch = X_batch.float().cpu().detach()
pred = pred.reshape(-1, pred.shape[-1])
pred = pred**2
y_batch = y_batch.view(-1, y_batch.shape[-1])
avg_val_loss += self.loss(
pred, y_batch).item() / len(valid_loader)
y_batch = y_batch.float().cpu().detach()
pred = pred.float().cpu().detach()
val_true = torch.cat([val_true, y_batch], 0)
val_preds = torch.cat([val_preds, pred], 0)
# evalueate metric
val_preds = val_preds.numpy()
val_true = val_true.numpy()
# val_true, val_preds = self.metric.find_opt_thresold(val_true, val_preds)
val_preds = self.postprocessing.run(val_preds)
metric_val = self.metric.compute(val_true, val_preds)
self.scheduler.step(metric_val) #avg_val_loss)
res = self.early_stopping(score=metric_val,
model=self.model,
threshold=threshold)
# print statistics
if self.hparams['verbose_train']:
print(
'| Epoch: ',
epoch + 1,
'| Train_loss: ',
avg_loss,
'| Val_loss: ',
avg_val_loss,
'| Metric_train: ',
metric_train,
'| Metric_val: ',
metric_val,
'| Current LR: ',
self.__get_lr(self.optimizer),
)
# # add history to tensorboard
writer.add_scalars(
'Loss',
{
'Train_loss': avg_loss,
'Val_loss': avg_val_loss
},
epoch,
)
writer.add_scalars('Metric', {
'Metric_train': metric_train,
'Metric_val': metric_val
}, epoch)
if res == 2:
print("Early Stopping")
print(
f'global best max val_loss model score {self.early_stopping.best_score}'
)
break
elif res == 1:
print(f'save global val_loss model score {metric_val}')
writer.close()
self.model = self.early_stopping.load_best_weights()
self.postprocessing.update_threshold(self.early_stopping.threshold)
return True
def predict(self, X_test):
# evaluate the model
self.model.eval()
test_loader = torch.utils.data.DataLoader(
X_test,
batch_size=self.hparams['batch_size'],
shuffle=False,
num_workers=self.num_workers) # ,collate_fn=train.my_collate
test_preds = torch.Tensor([])
test_val = torch.Tensor([])
print('Start generation of predictions')
with torch.no_grad():
for i, (X_batch, y_batch) in enumerate(tqdm(test_loader)):
X_batch = X_batch.float().to(self.device)
pred, pred_decoder = self.model(X_batch)
pred = pred**2
X_batch = X_batch.float().cpu().detach()
test_preds = torch.cat([test_preds, pred.cpu().detach()], 0)
test_val = torch.cat([test_val, y_batch.cpu().detach()], 0)
return test_val.numpy(), test_preds.numpy()
def get_heatmap(self, X_test):
# evaluate the model
self.model.eval()
test_loader = torch.utils.data.DataLoader(
X_test,
batch_size=self.batch_size,
shuffle=False,
num_workers=self.num_workers) # ,collate_fn=train.my_collate
test_preds = torch.Tensor([])
with torch.no_grad():
for i, (X_batch) in enumerate(test_loader):
X_batch = X_batch.float().to(self.device)
pred = self.model.activatations(X_batch)
pred = torch.sigmoid(pred)
pred = pred**2
X_batch = X_batch.float().cpu().detach()
test_preds = torch.cat([test_preds, pred.cpu().detach()], 0)
return test_preds.numpy()
def model_save(self, model_path):
torch.save(self.model.state_dict(), model_path)
# self.model.module.state_dict(), PATH
# torch.save(self.model, model_path)
return True
def model_load(self, model_path):
self.model.load_state_dict(
torch.load(model_path, map_location=self.device))
return True
def model_load_old(self, model_path):
self.model = torch.load(model_path, map_location=self.device)
return True
def inference(self, X, y):
preprocessing = Preprocessing(aug=False)
X = preprocessing.run(X, y, label_process=False)
X = X.reshape(1, -1, X.shape[1])
self.model.eval()
predictions, pred_decoder = self.model.forward(torch.Tensor(X))
predictions = predictions**2
predictions = predictions.detach().numpy()
print(np.round(predictions, 3))
return predictions
################## Utils #####################
def __get_lr(self, optimizer):
for param_group in optimizer.param_groups:
return param_group['lr']
class CVPipeline:
def __init__(self, hparams, split_table_path, split_table_name, debug_folder, model, gpu,downsample):
# load the model
self.hparams = hparams
self.model = model
self.gpu = gpu
self.downsample = downsample
print('\n')
print('Selected Learning rate:', self.hparams['lr'])
print('\n')
self.debug_folder = debug_folder
self.split_table_path = split_table_path
self.split_table_name = split_table_name
self.exclusions = ['S0431',
'S0326'
'S0453'
'S0458'
'A5766'
'A0227'
'A0238'
'A1516'
'A5179'
'Q1807'
'Q3568'
'E10256'
'E07341'
'E05758']
self.splits = self.load_split_table()
self.metric = Metric()
def load_split_table(self):
splits = []
split_files = [i for i in os.listdir(self.split_table_path) if i.find('fold') != -1]
for i in range(len(split_files)):
data = json.load(open(self.split_table_path + str(i) + '_' + self.split_table_name))
train_data = data['train']
for index, i in enumerate(train_data):
i = i.split('\\')
i = i[-1]
train_data[index] = i
val_data = data['val']
for index, i in enumerate(val_data):
i = i.split('\\')
i = i[-1]
val_data[index] = i
dataset_train = []
for i in train_data:
if i in self.exclusions:
continue
if i[0] != 'Q' and i[0] != 'S' and i[0] != 'A' and i[0] != 'H' and i[0] != 'E': # A, B , D, E datasets
continue
dataset_train.append(i)
dataset_val = []
for i in val_data:
if i in self.exclusions:
continue
if i[0] != 'Q' and i[0] != 'S' and i[0] != 'A' and i[0] != 'H' and i[0] != 'E': # A, B , D, E datasets
continue
dataset_val.append(i)
data['train'] = dataset_train#+self.additinal_data
data['val'] = dataset_val
splits.append(data)
splits = pd.DataFrame(splits)
return splits
def train(self):
score = 0
for fold in range(self.splits.shape[0]):
if fold is not None:
if fold != self.hparams['start_fold']:
continue
#TODO
train = Dataset_train(self.splits['train'].values[fold], aug=False,downsample=self.downsample)
valid = Dataset_train(self.splits['val'].values[fold], aug=False,downsample=self.downsample)
X, y = train.__getitem__(0)
self.model = self.model(
input_size=X.shape[0], n_channels=X.shape[1], hparams=self.hparams, gpu=self.gpu
)
# train model
self.model.fit(train=train, valid=valid)
# get model predictions
y_val,pred_val = self.model.predict(valid)
self.postprocessing = PostProcessing(fold=self.hparams['start_fold'])
pred_val_processed = self.postprocessing.run(pred_val)
# TODO: add activations
# heatmap = self.model.get_heatmap(valid)
fold_score = self.metric.compute(y_val, pred_val_processed)
print("Model's final scrore: ",fold_score)
# save the model
self.model.model_save(
self.hparams['model_path']
+ self.hparams['model_name']+f"_{self.hparams['start_fold']}"
+ '_fold_'
+ str(fold_score)
+ '.pt'
)
# create a dictionary for debugging
self.save_debug_data(pred_val, self.splits['val'].values[fold])
return fold_score
def save_debug_data(self, pred_val, validation_list):
for index, data in enumerate(validation_list):
if data[0] == 'A':
data_folder = 'A'
elif data[0] == 'Q':
data_folder = 'B'
elif data[0] == 'I':
data_folder = 'C'
elif data[0] == 'S':
data_folder = 'D'
elif data[0] == 'H':
data_folder = 'E'
elif data[0] == 'E':
data_folder = 'F'
data_folder = f'./data/CV_debug/{data_folder}/'
prediction = {}
prediction['predicted_label'] = pred_val[index].tolist()
# save debug data
with open(data_folder + data + '.json', 'w') as outfile:
json.dump(prediction, outfile)
return True
