def __init__(self, configer):
self.configer = configer
self.data_path = configer.get(
"data", "data_path") #: str: Path to data directory
# Train val and test accuracy
self.accuracy = AverageMeter()
# DataLoaders
self.data_loader = None
# Module load and save utility
self.device = self.configer.get("device")
self.model_utility = ModuleUtilizer(
self.configer
) #: Model utility for load, save and update optimizer
self.net = None
# Training procedure
self.transforms = None
# Other useful data
self.in_planes = np.prod(self.configer.get(
"data", "n_features")) #: int: Input channels
self.window = self.configer.get(
"data", "n_frames") #: int: Number of frames per sequence
self.n_classes = self.configer.get(
"data", "n_classes") #: int: Total number of classes for dataset
self.dataset = self.configer.get(
"dataset").lower() #: str: Type of dataset
def __init__(self, configer):
self.configer = configer
self.data_path = configer.get(
"data", "data_path") #: str: Path to data directory
# Train val and test accuracy
self.accuracy = AverageMeter()
# DataLoaders
self.data_loader = None
# Module load and save utility
self.device = self.configer.get("device")
self.model_utility = ModuleUtilizer(
self.configer
) #: Model utility for load, save and update optimizer
self.net = None
# Training procedure
self.transforms = None
# Other useful data
self.backbone = self.configer.get("network",
"backbone") #: str: Backbone type
self.in_planes = None #: int: Input channels
self.clip_length = self.configer.get(
"data", "n_frames") #: int: Number of frames per sequence
self.n_classes = self.configer.get(
"data", "n_classes") #: int: Total number of classes for dataset
self.data_type = self.configer.get(
"data", "type") #: str: Type of data (rgb, depth, ir, leapmotion)
self.dataset = self.configer.get(
"dataset").lower() #: str: Type of dataset
self.optical_flow = self.configer.get("data", "optical_flow")
if self.optical_flow is None:
self.optical_flow = True
class GestureTestRealTime(object):
"""Gesture Recognition Test class
Attributes:
configer (Configer): Configer object, contains procedure configuration.
train_loader (torch.utils.data.DataLoader): Train data loader variable
val_loader (torch.utils.data.DataLoader): Val data loader variable
test_loader (torch.utils.data.DataLoader): Test data loader variable
net (torch.nn.Module): Network used for the current procedure
lr (int): Learning rate value
optimizer (torch.nn.optim.optimizer): Optimizer for training procedure
iters (int): Starting iteration number, not zero if resuming training
epoch (int): Starting epoch number, not zero if resuming training
scheduler (torch.optim.lr_scheduler): Scheduler to utilize during training
"""
def __init__(self, configer):
self.configer = configer
self.data_path = configer.get(
"data", "data_path") #: str: Path to data directory
# Train val and test accuracy
self.accuracy = AverageMeter()
# DataLoaders
self.data_loader = None
# Module load and save utility
self.device = self.configer.get("device")
self.model_utility = ModuleUtilizer(
self.configer
) #: Model utility for load, save and update optimizer
self.net = None
# Training procedure
self.transforms = None
# Other useful data
self.in_planes = np.prod(self.configer.get(
"data", "n_features")) #: int: Input channels
self.window = self.configer.get(
"data", "n_frames") #: int: Number of frames per sequence
self.n_classes = self.configer.get(
"data", "n_classes") #: int: Total number of classes for dataset
self.dataset = self.configer.get(
"dataset").lower() #: str: Type of dataset
def init_model(self):
"""Initialize model and other data for procedure"""
# Selecting correct model and normalization variable based on type variable
if self.dataset == "shrec":
self.net = ShrecTrasformer(
self.in_planes,
self.n_classes,
n_head=self.configer.get("network", "n_head"),
dff=self.configer.get("network", "ff_size"),
n_module=self.configer.get("network", "n_module"))
Dataset = ShrecRealTime
self.train_transforms = None
self.val_transforms = None
else:
raise NotImplementedError("Error {}".format(self.dataset))
self.net, _, _, _ = self.model_utility.load_net(self.net)
# Setting Dataloaders
self.data_loader = DataLoader(Dataset(self.configer,
self.data_path,
split="test",
transforms=self.transforms,
n_frames=self.window),
batch_size=1,
shuffle=False,
drop_last=True,
num_workers=self.configer.get(
'solver', 'workers'),
pin_memory=True,
worker_init_fn=worker_init_fn)
def __test(self):
"""Testing function."""
self.net.eval()
data_list = list()
with torch.no_grad():
for i, data_tuple in enumerate(tqdm(self.data_loader,
desc="Test")):
"""
input, gt
"""
inputs = data_tuple[0].to(self.device)
tmp = "{};".format(len(data_list) + 1)
start = list()
end = list()
prediction_data = list()
s = StateMachine(threshold_window_S1=10, threshold_S3=25)
for idx, window in enumerate(inputs):
output = self.net(window.unsqueeze(0))
predicted = torch.argmax(output.detach(),
dim=-1).cpu().numpy()
prediction_data.append(predicted.squeeze())
retval = s.update(predicted.squeeze(), idx)
if retval:
start.append(s.get_start())
end.append(s.get_end())
pred = np.array(prediction_data)[start[-1]:end[-1] + 1]
pred = pred[pred != CLASS.index("NO_GESTURE")]
pred = np.bincount(pred.flatten()).argmax()
tmp += "{};{};{};".format(CLASS[pred], start[-1] + 2,
end[-1] + 2)
s.reset()
if s.active in [2, 3]:
if (len(inputs) - 1 - s.get_start()) > 3:
start.append(s.get_start())
end.append(len(inputs) - 1)
pred = np.array(prediction_data)[start[-1]:end[-1] + 1]
pred = pred[pred != CLASS.index("NO_GESTURE")]
pred = np.bincount(pred.flatten()).argmax()
tmp += "{};{};{};".format(CLASS[pred], start[-1] + 2,
end[-1] + 2)
s.reset()
tmp += "\n"
data_list.append(tmp)
if not os.path.exists("./outputs"):
os.makedirs("./outputs")
with open(
"./outputs/" + self.configer.get("checkpoints", "save_name") +
".txt", "wt") as outfile:
outfile.write("".join(data_list))
def test(self):
self.__test()
def update_metrics(self, split: str, loss, bs, accuracy=None):
self.losses[split].update(loss, bs)
if accuracy is not None:
self.accuracy[split].update(accuracy, bs)
if split == "train" and self.iters % self.save_iters == 0:
self.tbx_summary.add_scalar('{}_loss'.format(split),
self.losses[split].avg, self.iters)
self.tbx_summary.add_scalar('{}_accuracy'.format(split),
self.accuracy[split].avg, self.iters)
self.losses[split].reset()
self.accuracy[split].reset()
class GestureTest(object):
"""Gesture Recognition Test class
Attributes:
configer (Configer): Configer object, contains procedure configuration.
train_loader (torch.utils.data.DataLoader): Train data loader variable
val_loader (torch.utils.data.DataLoader): Val data loader variable
test_loader (torch.utils.data.DataLoader): Test data loader variable
net (torch.nn.Module): Network used for the current procedure
lr (int): Learning rate value
optimizer (torch.nn.optim.optimizer): Optimizer for training procedure
iters (int): Starting iteration number, not zero if resuming training
epoch (int): Starting epoch number, not zero if resuming training
scheduler (torch.optim.lr_scheduler): Scheduler to utilize during training
"""
def __init__(self, configer):
self.configer = configer
self.data_path = configer.get("data", "data_path") #: str: Path to data directory
# Train val and test accuracy
self.accuracy = AverageMeter()
# DataLoaders
self.data_loader = None
# Module load and save utility
self.device = self.configer.get("device")
self.model_utility = ModuleUtilizer(self.configer) #: Model utility for load, save and update optimizer
self.net = None
# Training procedure
self.transforms = None
# Other useful data
self.in_planes = np.prod(self.configer.get("data", "n_features")) #: int: Input channels
self.window = self.configer.get("data", "n_frames") #: int: Number of frames per sequence
self.n_classes = self.configer.get("data", "n_classes") #: int: Total number of classes for dataset
self.dataset = self.configer.get("dataset").lower() #: str: Type of dataset
self.frame_error_start = list()
self.frame_error_end = list()
self.AP = list()
self.AR = list()
self.CORR_CLASS = list()
self.MISLAB = list()
self.FALSE_POS = list()
self.MISSED = list()
self.confusion = np.zeros((self.configer.get("data", "n_classes") - 1, self.configer.get("data", "n_classes") - 1))
def init_model(self):
"""Initialize model and other data for procedure"""
# Selecting correct model and normalization variable based on type variable
if self.dataset == "shrec":
self.net = ShrecTrasformer(self.in_planes, self.n_classes,
n_head=self.configer.get("network", "n_head"),
dff=self.configer.get("network", "ff_size"),
n_module=self.configer.get("network", "n_module"))
Dataset = Shrec
self.train_transforms = None
self.val_transforms = None
else:
raise NotImplementedError("Error {}".format(self.dataset))
self.net, _, _, _ = self.model_utility.load_net(self.net)
# Setting Dataloaders
self.data_loader = DataLoader(
Dataset(self.configer, self.data_path, split="val",
transforms=self.transforms, n_frames=self.window,
idx_crossval=self.configer.get("data", "idx_crossval")),
batch_size=1, shuffle=False, drop_last=True,
num_workers=self.configer.get('solver', 'workers'), pin_memory=True, worker_init_fn=worker_init_fn)
def __test(self):
"""Testing function."""
self.net.eval()
data_list = list()
data_list_gt = list()
with torch.no_grad():
for i, data_tuple in enumerate(tqdm(self.data_loader, desc="Test")):
"""
input, gt
"""
inputs = data_tuple[0][0].to(self.device)
gt = data_tuple[1][0].to(self.device)
start_end = data_tuple[2]
output = self.net(inputs)
predicted = torch.argmax(output.detach(), dim=-1)
correct = gt.detach().squeeze(dim=1)
self.state_machine(predicted.cpu().numpy(), correct.cpu().numpy(), start_end.cpu().numpy(), data_list, data_list_gt)
if not os.path.exists("./outputs"):
os.makedirs("./outputs")
with open("./outputs/"+self.configer.get("checkpoints", "save_name")+".txt", "wt") as outfile:
outfile.write("".join(data_list))
with open("./outputs/"+self.configer.get("checkpoints", "save_name")+"_GT.txt", "wt") as outfile:
outfile.write("".join(data_list_gt))
plot_confusion_matrix(self.confusion.astype(np.int16), ["{}".format(i) for i in range(self.n_classes)], False, "./outputs/" + self.configer.get("checkpoints", "save_name") + "_matrix")
print(f"AP: {np.array(self.AP).mean()}, AR: {np.array(self.AR).mean()}")
print(f"CORR_CLASS: {np.array(self.CORR_CLASS).mean()}, FALSE_POS: {np.array(self.FALSE_POS).mean()}")
def test(self):
self.__test()
def update_metrics(self, split: str, loss, bs, accuracy=None):
self.losses[split].update(loss, bs)
if accuracy is not None:
self.accuracy[split].update(accuracy, bs)
if split == "train" and self.iters % self.save_iters == 0:
self.tbx_summary.add_scalar('{}_loss'.format(split), self.losses[split].avg, self.iters)
self.tbx_summary.add_scalar('{}_accuracy'.format(split), self.accuracy[split].avg, self.iters)
self.losses[split].reset()
self.accuracy[split].reset()
def state_machine(self, predicted, correct, start_end, data_list = None, data_list_gt = None):
s = StateMachine(threshold_window_S1=10, threshold_S3=25)
start_gt = list()
end_gt = list()
for idx, window in enumerate(correct):
retval = s.update(window, idx)
if retval:
start_gt.append(s.get_start())
end_gt.append(s.get_end())
s.reset()
if s.active in [2, 3]:
if (len(correct) - 1 - s.get_start()) > 3:
start_gt.append(s.get_start())
end_gt.append(len(correct) - 1)
s.reset()
start = list()
end = list()
for idx, window in enumerate(predicted):
retval = s.update(window, idx)
if retval:
start.append(s.get_start())
end.append(s.get_end())
s.reset()
if s.active in [2, 3]:
if (len(predicted) - 1 - s.get_start()) > 3:
start.append(s.get_start())
end.append(len(predicted) - 1)
TP = 0
FP = 0
couples = dict()
start_end = start_end[0]
for i in range(len(start)):
s = start[i]
e = end[i]
best_idx = ((np.abs((np.array(start_end[..., 0]) - s)) + np.abs((np.array(start_end[..., 1]) - e)) / 2)).argmin()
if best_idx not in couples:
couples[best_idx] = [i, ((np.abs((np.array(start_end[..., 0]) - s)) + np.abs((np.array(start_end[..., 1]) - e))) / 2).min()]
else:
if ((np.abs((np.array(start_end[..., 0]) - s)) + np.abs((np.array(start_end[..., 1]) - e))) / 2).min() < couples[best_idx][1]:
couples[best_idx] = [i, ((np.abs((np.array(start_end[..., 0]) - s)) + np.abs((np.array(start_end[..., 1]) - e))) / 2).min()]
FP += 1
FN = len(start_gt) - len(couples)
for k, v in couples.items():
self.frame_error_start.append(np.abs(start_end[k, 0] - start[v[0]]))
self.frame_error_end.append(np.abs(start_end[k, 1] - end[v[0]]))
pred = predicted[start[v[0]]:end[v[0]] + 1]
pred = pred[pred != CLASS.index("NO_GESTURE")]
try:
pred = np.bincount(pred.flatten()).argmax()
except ValueError:
print("OK")
gt = correct[start_end[k][0]:start_end[k][1] + 1]
gt = gt[gt != CLASS.index("NO_GESTURE")]
gt = np.bincount(gt.flatten()).argmax()
self.confusion[gt, pred] += 1
if pred == gt:
TP += 1
else:
FP += 1
if data_list is not None:
tmp = "{};".format(len(data_list) + 1)
for start, end in zip(start, end):
pred = predicted[start:end + 1]
pred = pred[pred != CLASS.index("NO_GESTURE")]
pred = np.bincount(pred.flatten()).argmax()
tmp += "{};{};{};".format(CLASS[pred], start+2, end+2)
tmp += "\n"
data_list.append(tmp)
if data_list_gt is not None:
tmp = "{};".format(len(data_list_gt) + 1)
for i, (s, e) in enumerate(zip(start_gt, end_gt)):
gt = correct[s:e + 1]
gt = gt[gt != CLASS.index("NO_GESTURE")]
gt = np.bincount(gt.flatten()).argmax()
tmp += "{};{};{};".format(CLASS[gt], start_end[i, 0], start_end[i, 1])
tmp += "\n"
data_list_gt.append(tmp)
self.AP.append((TP / (TP + FP)) if TP + FP != 0 else 0)
self.AR.append((TP / (TP + FN)) if TP + FN != 0 else 0)
self.FALSE_POS.append(FP)
self.CORR_CLASS.append(TP / len(couples))
self.MISLAB.append(1 - TP / len(couples))
def __init__(self, configer):
self.configer = configer
self.data_path = configer.get(
"data", "data_path") #: str: Path to data directory
# Losses
self.losses = {
'train': AverageMeter(), #: Train loss avg meter
'val': AverageMeter(), #: Val loss avg meter
'test': AverageMeter() #: Test loss avg meter
}
# Train val and test accuracy
self.accuracy = {
'train': AverageMeter(), #: Train accuracy avg meter
'val': AverageMeter(), #: Val accuracy avg meter
'test': AverageMeter() #: Test accuracy avg meter
}
self.frame_error_start = list()
self.frame_error_end = list()
self.AP = list()
self.AR = list()
self.CORR_CLASS = list()
self.MISLAB = list()
self.FALSE_POS = list()
self.MISSED = list()
# DataLoaders
self.train_loader = None
self.val_loader = None
self.test_loader = None
# Module load and save utility
self.device = self.configer.get("device")
self.model_utility = ModuleUtilizer(
self.configer
) #: Model utility for load, save and update optimizer
self.net = None
self.lr = None
# Training procedure
self.optimizer = None
self.iters = None
self.epoch = 0
self.train_transforms = None
self.val_transforms = None
self.loss = None
# Tensorboard and Metrics
self.tbx_summary = SummaryWriter(
str(
Path(configer.get('checkpoints',
'tb_path')) #: Summary Writer plot
/ configer.get("dataset") #: data with TensorboardX
/ configer.get('checkpoints', 'save_name')))
self.tbx_summary.add_text('parameters',
str(self.configer).replace("\n", "\n\n"))
self.save_iters = self.configer.get('checkpoints',
'save_iters') #: int: Saving ratio
# Other useful data
self.in_planes = np.prod(self.configer.get(
"data", "n_features")) #: int: Input channels
self.window = self.configer.get(
"data", "n_frames") #: int: Number of frames per sequence
self.n_classes = self.configer.get(
"data", "n_classes") #: int: Total number of classes for dataset
self.dataset = self.configer.get(
"dataset").lower() #: str: Type of dataset
self.stride = self.configer.get("data", "stride")
self.window = self.configer.get("data", "n_frames")
class GestureTrainer(object):
"""Gesture Recognition Train class
Attributes:
configer (Configer): Configer object, contains procedure configuration.
train_loader (torch.utils.data.DataLoader): Train data loader variable
val_loader (torch.utils.data.DataLoader): Val data loader variable
test_loader (torch.utils.data.DataLoader): Test data loader variable
net (torch.nn.Module): Network used for the current procedure
lr (int): Learning rate value
optimizer (torch.nn.optim.optimizer): Optimizer for training procedure
iters (int): Starting iteration number, not zero if resuming training
epoch (int): Starting epoch number, not zero if resuming training
scheduler (torch.optim.lr_scheduler): Scheduler to utilize during training
"""
def __init__(self, configer):
self.configer = configer
self.data_path = configer.get(
"data", "data_path") #: str: Path to data directory
# Losses
self.losses = {
'train': AverageMeter(), #: Train loss avg meter
'val': AverageMeter(), #: Val loss avg meter
'test': AverageMeter() #: Test loss avg meter
}
# Train val and test accuracy
self.accuracy = {
'train': AverageMeter(), #: Train accuracy avg meter
'val': AverageMeter(), #: Val accuracy avg meter
'test': AverageMeter() #: Test accuracy avg meter
}
self.frame_error_start = list()
self.frame_error_end = list()
self.AP = list()
self.AR = list()
self.CORR_CLASS = list()
self.MISLAB = list()
self.FALSE_POS = list()
self.MISSED = list()
# DataLoaders
self.train_loader = None
self.val_loader = None
self.test_loader = None
# Module load and save utility
self.device = self.configer.get("device")
self.model_utility = ModuleUtilizer(
self.configer
) #: Model utility for load, save and update optimizer
self.net = None
self.lr = None
# Training procedure
self.optimizer = None
self.iters = None
self.epoch = 0
self.train_transforms = None
self.val_transforms = None
self.loss = None
# Tensorboard and Metrics
self.tbx_summary = SummaryWriter(
str(
Path(configer.get('checkpoints',
'tb_path')) #: Summary Writer plot
/ configer.get("dataset") #: data with TensorboardX
/ configer.get('checkpoints', 'save_name')))
self.tbx_summary.add_text('parameters',
str(self.configer).replace("\n", "\n\n"))
self.save_iters = self.configer.get('checkpoints',
'save_iters') #: int: Saving ratio
# Other useful data
self.in_planes = np.prod(self.configer.get(
"data", "n_features")) #: int: Input channels
self.window = self.configer.get(
"data", "n_frames") #: int: Number of frames per sequence
self.n_classes = self.configer.get(
"data", "n_classes") #: int: Total number of classes for dataset
self.dataset = self.configer.get(
"dataset").lower() #: str: Type of dataset
self.stride = self.configer.get("data", "stride")
self.window = self.configer.get("data", "n_frames")
def init_model(self):
"""Initialize model and other data for procedure"""
if self.dataset == "shrec":
self.net = ShrecTrasformer(
self.in_planes,
self.n_classes,
n_head=self.configer.get("network", "n_head"),
dff=self.configer.get("network", "ff_size"),
n_module=self.configer.get("network", "n_module"))
Dataset = Shrec
self.train_transforms = None
self.val_transforms = None
self.loss = nn.CrossEntropyLoss().to(self.device)
# self.loss = FocalLoss().to(self.device)
# Selecting correct model and normalization variable based on type variable
if self.dataset == "shrec":
self.net = ShrecTrasformer(
self.in_planes,
self.n_classes,
n_head=self.configer.get("network", "n_head"),
dff=self.configer.get("network", "ff_size"),
n_module=self.configer.get("network", "n_module"))
else:
raise NotImplementedError("Error")
# Initializing training
self.iters = 0
self.epoch = None
phase = self.configer.get('phase')
# Starting or resuming procedure
if phase == 'train':
self.net, self.iters, self.epoch, optim_dict = self.model_utility.load_net(
self.net)
else:
raise ValueError('Phase: {} is not valid.'.format(phase))
if self.epoch is None:
self.epoch = 0
# ToDo Restore optimizer and scheduler from checkpoint
self.optimizer, self.lr = self.model_utility.update_optimizer(
self.net, self.iters)
# Resuming training, restoring optimizer value
if optim_dict is not None:
print("Resuming training from epoch {}.".format(self.epoch))
self.optimizer.load_state_dict(optim_dict)
# Setting Dataloaders
self.train_loader = DataLoader(
Dataset(self.configer,
self.data_path,
split="train",
transforms=self.train_transforms,
n_frames=self.window,
idx_crossval=self.configer.get("data", "idx_crossval")),
batch_size=self.configer.get('data', 'batch_size'),
shuffle=True,
drop_last=True,
num_workers=self.configer.get('solver', 'workers'),
pin_memory=True,
worker_init_fn=worker_init_fn,
collate_fn=None)
self.val_loader = DataLoader(
Dataset(self.configer,
self.data_path,
split="val",
transforms=self.val_transforms,
n_frames=self.window,
idx_crossval=self.configer.get("data", "idx_crossval")),
batch_size=1,
shuffle=False,
drop_last=True,
num_workers=self.configer.get('solver', 'workers'),
pin_memory=True,
worker_init_fn=worker_init_fn,
collate_fn=None)
def __train(self):
"""Train function for every epoch."""
self.net.train()
for data_tuple in tqdm(self.train_loader, desc="Train"):
"""
input, gt
"""
inputs = data_tuple[0].to(self.device)
output = self.net(inputs)
gt = data_tuple[1].to(self.device)
self.optimizer.zero_grad()
loss = self.loss(output.view(-1, output.shape[-1]), gt.view(-1))
loss.backward()
torch.nn.utils.clip_grad_norm_(self.net.parameters(), max_norm=1)
self.optimizer.step()
predicted = torch.argmax(output.detach(), dim=-1).view(-1)
correct = gt.detach().squeeze(dim=-1).view(-1)
self.iters += 1
self.update_metrics(
"train", loss.item(), inputs.size(0),
float((predicted == correct).sum()) / len(correct))
def __val(self):
"""Validation function."""
self.net.eval()
with torch.no_grad():
for i, data_tuple in enumerate(tqdm(self.val_loader, desc="Val")):
"""
input, gt
"""
inputs = data_tuple[0][0].to(self.device)
gt = data_tuple[1][0].to(self.device)
start_end = data_tuple[2]
output = self.net(inputs)
loss = self.loss(output.view(-1, output.shape[-1]),
gt.view(-1))
predicted = torch.argmax(output.detach(), dim=-1)
correct = gt.detach().squeeze(dim=-1)
self.iters += 1
self.update_metrics(
"val", loss.item(), inputs.size(0),
float((predicted.view(-1) == correct.view(-1)).sum()) /
len(correct.view(-1)))
self.state_machine(predicted.cpu().numpy(),
correct.cpu().numpy(),
start_end.cpu().numpy())
self.tbx_summary.add_scalar('val_loss', self.losses["val"].avg,
self.epoch + 1)
self.tbx_summary.add_scalar('val_accuracy', self.accuracy["val"].avg,
self.epoch + 1)
self.tbx_summary.add_scalar('frame_error_start',
np.array(self.frame_error_start).mean(),
self.epoch + 1)
self.tbx_summary.add_scalar('frame_error_end',
np.array(self.frame_error_end).mean(),
self.epoch + 1)
self.tbx_summary.add_scalar('AP',
np.array(self.AP).mean(), self.epoch + 1)
self.tbx_summary.add_scalar('AR',
np.array(self.AR).mean(), self.epoch + 1)
self.tbx_summary.add_scalar('FALSE_POS',
np.array(self.FALSE_POS).mean(),
self.epoch + 1)
self.tbx_summary.add_scalar('CORR_CLASS',
np.array(self.CORR_CLASS).mean(),
self.epoch + 1)
self.tbx_summary.add_scalar('MISLAB',
np.array(self.MISLAB).mean(),
self.epoch + 1)
accuracy = np.array(self.AP).mean()
self.accuracy["val"].reset()
self.losses["val"].reset()
ret = self.model_utility.save(accuracy, self.net, self.optimizer,
self.iters, self.epoch + 1)
return ret
def train(self):
for n in range(self.configer.get("epochs")):
print("Starting epoch {}".format(self.epoch + 1))
self.__train()
if self.configer.get("data", "k_fold") is not None:
ret = self.__val()
if ret < 0:
print(
"Got no improvement for {} epochs, current epoch is {}."
.format(self.configer.get("checkpoints", "early_stop"),
n))
break
self.epoch += 1
if self.configer.get("data", "k_fold") is None:
self.model_utility.save(1, self.net, self.optimizer, self.iters,
self.epoch + 1)
def update_metrics(self, split: str, loss, bs, accuracy=None):
self.losses[split].update(loss, bs)
if accuracy is not None:
self.accuracy[split].update(accuracy, bs)
if split == "train" and self.iters % self.save_iters == 0:
self.tbx_summary.add_scalar('{}_loss'.format(split),
self.losses[split].avg, self.iters)
self.tbx_summary.add_scalar('{}_accuracy'.format(split),
self.accuracy[split].avg, self.iters)
self.losses[split].reset()
self.accuracy[split].reset()
def state_machine(self,
predicted,
correct,
start_end,
data_list=None,
data_list_gt=None):
s = StateMachine(threshold_window_S1=10, threshold_S3=25)
start_gt = list()
end_gt = list()
for idx, window in enumerate(correct):
retval = s.update(window, idx)
if retval:
start_gt.append(s.get_start())
end_gt.append(s.get_end())
s.reset()
if s.active in [2, 3]:
if (len(correct) - 1 - s.get_start()) > 3:
start_gt.append(s.get_start())
end_gt.append(len(correct) - 1)
s.reset()
start = list()
end = list()
for idx, window in enumerate(predicted):
retval = s.update(window, idx)
if retval:
start.append(s.get_start())
end.append(s.get_end())
s.reset()
if s.active in [2, 3]:
if (len(predicted) - 1 - s.get_start()) > 3:
start.append(s.get_start())
end.append(len(predicted) - 1)
TP = 0
FP = 0
couples = dict()
start_end = start_end[0]
for i in range(len(start)):
s = start[i]
e = end[i]
best_idx = ((np.abs((np.array(start_end[..., 0]) - s)) + np.abs(
(np.array(start_end[..., 1]) - e)) / 2)).argmin()
if best_idx not in couples:
couples[best_idx] = [
i,
((np.abs((np.array(start_end[..., 0]) - s)) + np.abs(
(np.array(start_end[..., 1]) - e))) / 2).min()
]
else:
if ((np.abs((np.array(start_end[..., 0]) - s)) + np.abs((np.array(start_end[..., 1]) - e))) / 2).min() < \
couples[best_idx][1]:
couples[best_idx] = [
i,
((np.abs((np.array(start_end[..., 0]) - s)) + np.abs(
(np.array(start_end[..., 1]) - e))) / 2).min()
]
FP += 1
FN = len(start_gt) - len(couples)
for k, v in couples.items():
self.frame_error_start.append(np.abs(start_end[k, 0] -
start[v[0]]))
self.frame_error_end.append(np.abs(start_end[k, 1] - end[v[0]]))
pred = predicted[start[v[0]]:end[v[0]] + 1]
pred = pred[pred != CLASS.index("NO_GESTURE")]
try:
pred = np.bincount(pred.flatten()).argmax()
except ValueError:
print("OK")
gt = correct[start_end[k][0]:start_end[k][1] + 1]
gt = gt[gt != CLASS.index("NO_GESTURE")]
gt = np.bincount(gt.flatten()).argmax()
if pred == gt:
TP += 1
else:
FP += 1
if data_list is not None:
tmp = "{};".format(len(data_list) + 1)
for start, end in zip(start, end):
pred = predicted[start:end + 1]
pred = pred[pred != CLASS.index("NO_GESTURE")]
pred = np.bincount(pred.flatten()).argmax()
tmp += "{};{};{};".format(CLASS[pred], start + 2, end + 2)
tmp += "\n"
data_list.append(tmp)
if data_list_gt is not None:
tmp = "{};".format(len(data_list_gt) + 1)
for i, (s, e) in enumerate(zip(start_gt, end_gt)):
gt = correct[s:e + 1]
gt = gt[gt != CLASS.index("NO_GESTURE")]
gt = np.bincount(gt.flatten()).argmax()
tmp += "{};{};{};".format(CLASS[gt], start_end[i, 0],
start_end[i, 1])
tmp += "\n"
data_list_gt.append(tmp)
self.AP.append((TP / (TP + FP)) if TP + FP != 0 else 0)
self.AR.append((TP / (TP + FN)) if TP + FN != 0 else 0)
self.FALSE_POS.append(FP)
self.CORR_CLASS.append(TP / len(couples) if len(couples) == 0 else 0)
self.MISLAB.append(1 - TP / len(couples) if len(couples) == 0 else 0)
def __init__(self, configer):
self.configer = configer
self.data_path = configer.get(
"data", "data_path") #: str: Path to data directory
# Losses
self.losses = {
'train': AverageMeter(), #: Train loss avg meter
'val': AverageMeter(), #: Val loss avg meter
'test': AverageMeter() #: Test loss avg meter
}
# Train val and test accuracy
self.accuracy = {
'train': AverageMeter(), #: Train accuracy avg meter
'val': AverageMeter(), #: Val accuracy avg meter
'test': AverageMeter() #: Test accuracy avg meter
}
# DataLoaders
self.train_loader = None
self.val_loader = None
self.test_loader = None
# Module load and save utility
self.device = self.configer.get("device")
self.model_utility = ModuleUtilizer(
self.configer
) #: Model utility for load, save and update optimizer
self.net = None
self.lr = None
# Training procedure
self.optimizer = None
self.iters = None
self.epoch = 0
self.train_transforms = None
self.val_transforms = None
self.loss = None
# Tensorboard and Metrics
self.tbx_summary = SummaryWriter(
str(
Path(configer.get('checkpoints',
'tb_path')) #: Summary Writer plot
/ configer.get("dataset") #: data with TensorboardX
/ configer.get('checkpoints', 'save_name')))
self.tbx_summary.add_text('parameters',
str(self.configer).replace("\n", "\n\n"))
self.save_iters = self.configer.get('checkpoints',
'save_iters') #: int: Saving ratio
# Other useful data
self.backbone = self.configer.get("network",
"backbone") #: str: Backbone type
self.in_planes = None #: int: Input channels
self.clip_length = self.configer.get(
"data", "n_frames") #: int: Number of frames per sequence
self.n_classes = self.configer.get(
"data", "n_classes") #: int: Total number of classes for dataset
self.data_type = self.configer.get(
"data", "type") #: str: Type of data (rgb, depth, ir, leapmotion)
self.dataset = self.configer.get(
"dataset").lower() #: str: Type of dataset
self.optical_flow = self.configer.get("data", "optical_flow")
if self.optical_flow is None:
self.optical_flow = True
self.scheduler = None
class GestureTrainer(object):
"""Gesture Recognition Train class
Attributes:
configer (Configer): Configer object, contains procedure configuration.
train_loader (torch.utils.data.DataLoader): Train data loader variable
val_loader (torch.utils.data.DataLoader): Val data loader variable
test_loader (torch.utils.data.DataLoader): Test data loader variable
net (torch.nn.Module): Network used for the current procedure
lr (int): Learning rate value
optimizer (torch.nn.optim.optimizer): Optimizer for training procedure
iters (int): Starting iteration number, not zero if resuming training
epoch (int): Starting epoch number, not zero if resuming training
scheduler (torch.optim.lr_scheduler): Scheduler to utilize during training
"""
def __init__(self, configer):
self.configer = configer
self.data_path = configer.get(
"data", "data_path") #: str: Path to data directory
# Losses
self.losses = {
'train': AverageMeter(), #: Train loss avg meter
'val': AverageMeter(), #: Val loss avg meter
'test': AverageMeter() #: Test loss avg meter
}
# Train val and test accuracy
self.accuracy = {
'train': AverageMeter(), #: Train accuracy avg meter
'val': AverageMeter(), #: Val accuracy avg meter
'test': AverageMeter() #: Test accuracy avg meter
}
# DataLoaders
self.train_loader = None
self.val_loader = None
self.test_loader = None
# Module load and save utility
self.device = self.configer.get("device")
self.model_utility = ModuleUtilizer(
self.configer
) #: Model utility for load, save and update optimizer
self.net = None
self.lr = None
# Training procedure
self.optimizer = None
self.iters = None
self.epoch = 0
self.train_transforms = None
self.val_transforms = None
self.loss = None
# Tensorboard and Metrics
self.tbx_summary = SummaryWriter(
str(
Path(configer.get('checkpoints',
'tb_path')) #: Summary Writer plot
/ configer.get("dataset") #: data with TensorboardX
/ configer.get('checkpoints', 'save_name')))
self.tbx_summary.add_text('parameters',
str(self.configer).replace("\n", "\n\n"))
self.save_iters = self.configer.get('checkpoints',
'save_iters') #: int: Saving ratio
# Other useful data
self.backbone = self.configer.get("network",
"backbone") #: str: Backbone type
self.in_planes = None #: int: Input channels
self.clip_length = self.configer.get(
"data", "n_frames") #: int: Number of frames per sequence
self.n_classes = self.configer.get(
"data", "n_classes") #: int: Total number of classes for dataset
self.data_type = self.configer.get(
"data", "type") #: str: Type of data (rgb, depth, ir, leapmotion)
self.dataset = self.configer.get(
"dataset").lower() #: str: Type of dataset
self.optical_flow = self.configer.get("data", "optical_flow")
if self.optical_flow is None:
self.optical_flow = True
self.scheduler = None
def init_model(self):
"""Initialize model and other data for procedure"""
if self.optical_flow is True:
self.in_planes = 2
elif self.data_type in ["depth", "ir"]:
self.in_planes = 1
else:
self.in_planes = 3
self.loss = nn.CrossEntropyLoss().to(self.device)
# Selecting correct model and normalization variable based on type variable
self.net = GestureTransoformer(
self.backbone,
self.in_planes,
self.n_classes,
pretrained=self.configer.get("network", "pretrained"),
n_head=self.configer.get("network", "n_head"),
dropout_backbone=self.configer.get("network", "dropout2d"),
dropout_transformer=self.configer.get("network", "dropout1d"),
dff=self.configer.get("network", "ff_size"),
n_module=self.configer.get("network", "n_module"))
# Initializing training
self.iters = 0
self.epoch = None
phase = self.configer.get('phase')
# Starting or resuming procedure
if phase == 'train':
self.net, self.iters, self.epoch, optim_dict = self.model_utility.load_net(
self.net)
else:
raise ValueError('Phase: {} is not valid.'.format(phase))
if self.epoch is None:
self.epoch = 0
# ToDo Restore optimizer and scheduler from checkpoint
self.optimizer, self.lr = self.model_utility.update_optimizer(
self.net, self.iters)
self.scheduler = MultiStepLR(self.optimizer,
self.configer["solver", "decay_steps"],
gamma=0.1)
# Resuming training, restoring optimizer value
if optim_dict is not None:
print("Resuming training from epoch {}.".format(self.epoch))
self.optimizer.load_state_dict(optim_dict)
# Selecting Dataset and DataLoader
if self.dataset == "briareo":
Dataset = Briareo
self.train_transforms = iaa.Sequential([
iaa.Resize((0.85, 1.15)),
iaa.CropToFixedSize(width=190, height=190),
iaa.Rotate((-15, 15))
])
self.val_transforms = iaa.CenterCropToFixedSize(200, 200)
elif self.dataset == "nvgestures":
Dataset = NVGesture
self.train_transforms = iaa.Sequential([
iaa.Resize((0.8, 1.2)),
iaa.CropToFixedSize(width=256, height=192),
iaa.Rotate((-15, 15))
])
self.val_transforms = iaa.CenterCropToFixedSize(256, 192)
else:
raise NotImplementedError(
f"Dataset not supported: {self.configer.get('dataset')}")
# Setting Dataloaders
self.train_loader = DataLoader(
Dataset(self.configer,
self.data_path,
split="train",
data_type=self.data_type,
transforms=self.train_transforms,
n_frames=self.clip_length,
optical_flow=self.optical_flow),
batch_size=self.configer.get('data', 'batch_size'),
shuffle=True,
drop_last=True,
num_workers=self.configer.get('solver', 'workers'),
pin_memory=True,
worker_init_fn=worker_init_fn)
self.val_loader = DataLoader(
Dataset(self.configer,
self.data_path,
split="val",
data_type=self.data_type,
transforms=self.val_transforms,
n_frames=self.clip_length,
optical_flow=self.optical_flow),
batch_size=self.configer.get('data', 'batch_size'),
shuffle=False,
drop_last=True,
num_workers=self.configer.get('solver', 'workers'),
pin_memory=True,
worker_init_fn=worker_init_fn)
if self.dataset == "nvgestures":
self.test_loader = None
else:
self.test_loader = DataLoader(
Dataset(self.configer,
self.data_path,
split="test",
data_type=self.data_type,
transforms=self.val_transforms,
n_frames=self.clip_length,
optical_flow=self.optical_flow),
batch_size=1,
shuffle=False,
drop_last=True,
num_workers=self.configer.get('solver', 'workers'),
pin_memory=True,
worker_init_fn=worker_init_fn)
def __train(self):
"""Train function for every epoch."""
self.net.train()
for data_tuple in tqdm(self.train_loader, desc="Train"):
"""
input, gt
"""
inputs = data_tuple[0].to(self.device)
gt = data_tuple[1].to(self.device)
output = self.net(inputs)
self.optimizer.zero_grad()
loss = self.loss(output, gt.squeeze(dim=1))
loss.backward()
torch.nn.utils.clip_grad_norm_(self.net.parameters(), max_norm=1)
self.optimizer.step()
predicted = torch.argmax(output.detach(), dim=1)
correct = gt.detach().squeeze(dim=1)
self.iters += 1
self.update_metrics(
"train", loss.item(), inputs.size(0),
float((predicted == correct).sum()) / len(correct))
def __val(self):
"""Validation function."""
self.net.eval()
with torch.no_grad():
# for i, data_tuple in enumerate(tqdm(self.val_loader, desc="Val", postfix=str(self.accuracy["val"].avg))):
for i, data_tuple in enumerate(
tqdm(self.val_loader,
desc="Val",
postfix="" + str(np.random.randint(200)))):
"""
input, gt
"""
inputs = data_tuple[0].to(self.device)
gt = data_tuple[1].to(self.device)
output = self.net(inputs)
loss = self.loss(output, gt.squeeze(dim=1))
predicted = torch.argmax(output.detach(), dim=1)
correct = gt.detach().squeeze(dim=1)
self.iters += 1
self.update_metrics(
"val", loss.item(), inputs.size(0),
float((predicted == correct).sum()) / len(correct))
self.tbx_summary.add_scalar('val_loss', self.losses["val"].avg,
self.epoch + 1)
self.tbx_summary.add_scalar('val_accuracy', self.accuracy["val"].avg,
self.epoch + 1)
accuracy = self.accuracy["val"].avg
self.accuracy["val"].reset()
self.losses["val"].reset()
ret = self.model_utility.save(accuracy, self.net, self.optimizer,
self.iters, self.epoch + 1)
if ret < 0:
return -1
elif ret > 0 and self.test_loader is not None:
self.__test()
return ret
def __test(self):
"""Testing function."""
self.net.eval()
with torch.no_grad():
for i, data_tuple in enumerate(
tqdm(self.test_loader,
desc="Test",
postfix=str(self.accuracy["test"].avg))):
"""
input, gt
"""
inputs = data_tuple[0].to(self.device)
gt = data_tuple[1].to(self.device)
output = self.net(inputs)
loss = self.loss(output, gt.squeeze(dim=1))
predicted = torch.argmax(output.detach(), dim=1)
correct = gt.detach().squeeze(dim=1)
self.iters += 1
self.update_metrics(
"test", loss.item(), inputs.size(0),
float((predicted == correct).sum()) / len(correct))
self.tbx_summary.add_scalar('test_loss', self.losses["test"].avg,
self.epoch + 1)
self.tbx_summary.add_scalar('test_accuracy', self.accuracy["test"].avg,
self.epoch + 1)
self.losses["test"].reset()
self.accuracy["test"].reset()
def train(self):
for n in range(self.configer.get("epochs")):
print("Starting epoch {}".format(self.epoch + 1))
self.__train()
ret = self.__val()
if ret < 0:
print("Got no improvement for {} epochs, current epoch is {}.".
format(self.configer.get("checkpoints", "early_stop"),
n))
break
self.epoch += 1
def update_metrics(self, split: str, loss, bs, accuracy=None):
self.losses[split].update(loss, bs)
if accuracy is not None:
self.accuracy[split].update(accuracy, bs)
if split == "train" and self.iters % self.save_iters == 0:
self.tbx_summary.add_scalar('{}_loss'.format(split),
self.losses[split].avg, self.iters)
self.tbx_summary.add_scalar('{}_accuracy'.format(split),
self.accuracy[split].avg, self.iters)
self.losses[split].reset()
self.accuracy[split].reset()
class GestureTest(object):
"""Gesture Recognition Test class
Attributes:
configer (Configer): Configer object, contains procedure configuration.
train_loader (torch.utils.data.DataLoader): Train data loader variable
val_loader (torch.utils.data.DataLoader): Val data loader variable
test_loader (torch.utils.data.DataLoader): Test data loader variable
net (torch.nn.Module): Network used for the current procedure
lr (int): Learning rate value
optimizer (torch.nn.optim.optimizer): Optimizer for training procedure
iters (int): Starting iteration number, not zero if resuming training
epoch (int): Starting epoch number, not zero if resuming training
scheduler (torch.optim.lr_scheduler): Scheduler to utilize during training
"""
def __init__(self, configer):
self.configer = configer
self.data_path = configer.get(
"data", "data_path") #: str: Path to data directory
# Train val and test accuracy
self.accuracy = AverageMeter()
# DataLoaders
self.data_loader = None
# Module load and save utility
self.device = self.configer.get("device")
self.model_utility = ModuleUtilizer(
self.configer
) #: Model utility for load, save and update optimizer
self.net = None
# Training procedure
self.transforms = None
# Other useful data
self.backbone = self.configer.get("network",
"backbone") #: str: Backbone type
self.in_planes = None #: int: Input channels
self.clip_length = self.configer.get(
"data", "n_frames") #: int: Number of frames per sequence
self.n_classes = self.configer.get(
"data", "n_classes") #: int: Total number of classes for dataset
self.data_type = self.configer.get(
"data", "type") #: str: Type of data (rgb, depth, ir, leapmotion)
self.dataset = self.configer.get(
"dataset").lower() #: str: Type of dataset
self.optical_flow = self.configer.get("data", "optical_flow")
if self.optical_flow is None:
self.optical_flow = True
def init_model(self):
"""Initialize model and other data for procedure"""
if self.optical_flow is True:
self.in_planes = 2
elif self.data_type in ["depth", "ir"]:
self.in_planes = 1
else:
self.in_planes = 3
# Selecting correct model and normalization variable based on type variable
self.net = GestureTransoformer(
self.backbone,
self.in_planes,
self.n_classes,
pretrained=self.configer.get("network", "pretrained"),
n_head=self.configer.get("network", "n_head"),
dropout_backbone=self.configer.get("network", "dropout2d"),
dropout_transformer=self.configer.get("network", "dropout1d"),
dff=self.configer.get("network", "ff_size"),
n_module=self.configer.get("network", "n_module"))
self.net, _, _, _ = self.model_utility.load_net(self.net)
# Selecting Dataset and DataLoader
if self.dataset == "briareo":
Dataset = Briareo
self.transforms = iaa.CenterCropToFixedSize(200, 200)
elif self.dataset == "nvgestures":
Dataset = NVGesture
self.transforms = iaa.CenterCropToFixedSize(256, 192)
else:
raise NotImplementedError(
f"Dataset not supported: {self.configer.get('dataset')}")
# Setting Dataloaders
self.data_loader = DataLoader(Dataset(self.configer,
self.data_path,
split="test",
data_type=self.data_type,
transforms=self.transforms,
n_frames=self.clip_length,
optical_flow=self.optical_flow),
batch_size=1,
shuffle=False,
drop_last=True,
num_workers=self.configer.get(
'solver', 'workers'),
pin_memory=True,
worker_init_fn=worker_init_fn)
def __test(self):
"""Testing function."""
self.net.eval()
c = 0
tot = 0
with torch.no_grad():
for i, data_tuple in enumerate(tqdm(self.data_loader,
desc="Test")):
"""
input, gt
"""
inputs = data_tuple[0].to(self.device)
gt = data_tuple[1].to(self.device)
output = self.net(inputs)
predicted = torch.argmax(output.detach(), dim=1)
correct = gt.detach().squeeze(dim=1)
if predicted == correct:
c += 1
tot += 1
accuracy = c / tot
print("Accuracy: {}".format(accuracy))
def test(self):
self.__test()
def update_metrics(self, split: str, loss, bs, accuracy=None):
self.losses[split].update(loss, bs)
if accuracy is not None:
self.accuracy[split].update(accuracy, bs)
if split == "train" and self.iters % self.save_iters == 0:
self.tbx_summary.add_scalar('{}_loss'.format(split),
self.losses[split].avg, self.iters)
self.tbx_summary.add_scalar('{}_accuracy'.format(split),
self.accuracy[split].avg, self.iters)
self.losses[split].reset()
self.accuracy[split].reset()