class ConvPoseMachine(object):
"""
The class for Pose Estimation. Include train, val, val & predict.
"""
def __init__(self, configer):
self.configer = configer
self.batch_time = AverageMeter()
self.data_time = AverageMeter()
self.train_losses = AverageMeter()
self.val_losses = AverageMeter()
self.pose_visualizer = PoseVisualizer(configer)
self.pose_loss_manager = PoseLossManager(configer)
self.pose_model_manager = PoseModelManager(configer)
self.pose_data_loader = PoseDataLoader(configer)
self.module_utilizer = ModuleUtilizer(configer)
self.optim_scheduler = OptimScheduler(configer)
self.data_transformer = DataTransformer(configer)
self.heatmap_generator = HeatmapGenerator(configer)
self.pose_net = None
self.train_loader = None
self.val_loader = None
self.optimizer = None
self.scheduler = None
self._init_model()
def _init_model(self):
self.pose_net = self.pose_model_manager.single_pose_detector()
self.pose_net = self.module_utilizer.load_net(self.pose_net)
self.optimizer, self.scheduler = self.optim_scheduler.init_optimizer(self._get_parameters())
self.train_loader = self.pose_data_loader.get_trainloader()
self.val_loader = self.pose_data_loader.get_valloader()
self.mse_loss = self.pose_loss_manager.get_pose_loss('mse_loss')
def _get_parameters(self):
return self.pose_net.parameters()
def __train(self):
"""
Train function of every epoch during train phase.
"""
self.pose_net.train()
start_time = time.time()
# Adjust the learning rate after every epoch.
self.configer.plus_one('epoch')
self.scheduler.step(self.configer.get('epoch'))
# data_tuple: (inputs, heatmap, maskmap, tagmap, num_objects)
for i, data_dict in enumerate(self.train_loader):
inputs = data_dict['img']
input_size = [inputs.size(3), inputs.size(2)]
heatmap = self.heatmap_generator(data_dict['kpts'], input_size)
self.data_time.update(time.time() - start_time)
# Change the data type.
inputs, heatmap = self.module_utilizer.to_device(inputs, heatmap)
# self.pose_visualizer.vis_peaks(heatmap[0], inputs[0], name='cpm')
# Forward pass.
outputs = self.pose_net(inputs)
# Compute the loss of the train batch & backward.
loss = self.mse_loss(outputs, heatmap, maskmap)
self.train_losses.update(loss.item(), inputs.size(0))
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# Update the vars of the train phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.configer.plus_one('iters')
# Print the log info & reset the states.
if self.configer.get('iters') % self.configer.get('solver', 'display_iter') == 0:
Log.info('Train Epoch: {0}\tTrain Iteration: {1}\t'
'Time {batch_time.sum:.3f}s / {2}iters, ({batch_time.avg:.3f})\t'
'Data load {data_time.sum:.3f}s / {2}iters, ({data_time.avg:3f})\n'
'Learning rate = {3}\tLoss = {loss.val:.8f} (ave = {loss.avg:.8f})\n'.format(
self.configer.get('epoch'), self.configer.get('iters'),
self.configer.get('solver', 'display_iter'),
self.scheduler.get_lr(), batch_time=self.batch_time,
data_time=self.data_time, loss=self.train_losses))
self.batch_time.reset()
self.data_time.reset()
self.train_losses.reset()
# Check to val the current model.
if self.val_loader is not None and \
self.configer.get('iters') % self.configer.get('solver', 'test_interval') == 0:
self.__val()
def __val(self):
"""
Validation function during the train phase.
"""
self.pose_net.eval()
start_time = time.time()
with torch.no_grad():
for j, data_dict in enumerate(self.val_loader):
inputs = data_dict['img']
input_size = [inputs.size(3), inputs.size(2)]
heatmap = self.heatmap_generator(data_dict['kpts'], input_size)
# Change the data type.
inputs, heatmap = self.module_utilizer.to_device(inputs, heatmap)
# Forward pass.
outputs = self.pose_net(inputs)
# Compute the loss of the val batch.
loss = self.mse_loss(outputs[-1], heatmap)
self.val_losses.update(loss.item(), inputs.size(0))
# Update the vars of the val phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.module_utilizer.save_net(self.pose_net, save_mode='iters')
# Print the log info & reset the states.
Log.info(
'Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t'
'Loss {loss.avg:.8f}\n'.format(
batch_time=self.batch_time, loss=self.val_losses))
self.batch_time.reset()
self.val_losses.reset()
self.pose_net.train()
def train(self):
cudnn.benchmark = True
if self.configer.get('network', 'resume') is not None and self.configer.get('network', 'resume_val'):
self.__val()
while self.configer.get('epoch') < self.configer.get('solver', 'max_epoch'):
self.__train()
if self.configer.get('epoch') == self.configer.get('solver', 'max_epoch'):
break
class RPNPose(object):
"""
The class for Pose Estimation. Include train, val, test & predict.
"""
def __init__(self, configer):
self.configer = configer
self.batch_time = AverageMeter()
self.data_time = AverageMeter()
self.train_losses = AverageMeter()
self.val_losses = AverageMeter()
self.val_loss_heatmap = AverageMeter()
self.val_loss_associate = AverageMeter()
self.pose_visualizer = PoseVisualizer(configer)
self.pose_loss_manager = PoseLossManager(configer)
self.pose_model_manager = PoseModelManager(configer)
self.pose_data_loader = PoseDataLoader(configer)
self.module_utilizer = ModuleUtilizer(configer)
self.optim_scheduler = OptimScheduler(configer)
self.pose_net = None
self.train_loader = None
self.val_loader = None
self.optimizer = None
self.scheduler = None
def init_model(self):
self.pose_net = self.pose_model_manager.multi_pose_detector()
self.pose_net = self.module_utilizer.load_net(self.pose_net)
self.optimizer, self.scheduler = self.optim_scheduler.init_optimizer(
self._get_parameters())
self.train_loader = self.pose_data_loader.get_trainloader()
self.val_loader = self.pose_data_loader.get_valloader()
self.mse_loss = self.pose_loss_manager.get_pose_loss('mse_loss')
self.embeding_loss = self.pose_loss_manager.get_pose_loss(
'embedding_loss')
def _get_parameters(self):
return self.pose_net.parameters()
def __train(self):
"""
Train function of every epoch during train phase.
"""
self.pose_net.train()
start_time = time.time()
# Adjust the learning rate after every epoch.
self.configer.plus_one('epoch')
self.scheduler.step(self.configer.get('epoch'))
# data_tuple: (inputs, heatmap, maskmap, vecmap)
for i, (inputs, label, heatmap, maskmap, vecmap, tagmap,
num_objects) in enumerate(self.train_loader):
self.data_time.update(time.time() - start_time)
# Change the data type.
inputs, label, heatmap, maskmap, vecmap, tagmap = self.module_utilizer.to_device(
inputs, label, heatmap, maskmap, vecmap, tagmap)
# Forward pass.
paf_out, heatmap_out, embed_out = self.pose_net(inputs)
# Compute the loss of the train batch & backward.
loss_label = self.mse_loss(embed_out.sum(1).squeeze(), label)
loss_heatmap = self.mse_loss(heatmap_out, heatmap, maskmap)
loss_paf = self.mse_loss(paf_out, vecmap, maskmap)
loss_associate = self.embeding_loss(embed_out, tagmap, num_objects)
loss = loss_label + loss_heatmap + loss_paf + loss_associate
self.train_losses.update(loss.item(), inputs.size(0))
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# Update the vars of the train phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.configer.plus_one('iters')
# Print the log info & reset the states.
if self.configer.get('iters') % self.configer.get(
'solver', 'display_iter') == 0:
Log.info(
'Train Epoch: {0}\tTrain Iteration: {1}\t'
'Time {batch_time.sum:.3f}s / {2}iters, ({batch_time.avg:.3f})\t'
'Data load {data_time.sum:.3f}s / {2}iters, ({data_time.avg:3f})\n'
'Learning rate = {3}\tLoss = {loss.val:.8f} (ave = {loss.avg:.8f})\n'
.format(self.configer.get('epoch'),
self.configer.get('iters'),
self.configer.get('solver', 'display_iter'),
self.scheduler.get_lr(),
batch_time=self.batch_time,
data_time=self.data_time,
loss=self.train_losses))
self.batch_time.reset()
self.data_time.reset()
self.train_losses.reset()
# Check to val the current model.
if self.val_loader is not None and \
self.configer.get('iters') % self.configer.get('solver', 'test_interval') == 0:
self.__val()
def __val(self):
"""
Validation function during the train phase.
"""
self.pose_net.eval()
start_time = time.time()
with torch.no_grad():
for j, (inputs, label, heatmap, maskmap, vecmap, tagmap,
num_objects) in enumerate(self.val_loader):
# Change the data type.
inputs, label, heatmap, maskmap, vecmap, tagmap = self.module_utilizer.to_device(
inputs, label, heatmap, maskmap, vecmap, tagmap)
# Forward pass.
paf_out, heatmap_out, embed_out = self.pose_net(inputs)
# Compute the loss of the val batch.
loss_label = self.mse_loss(embed_out, label)
loss_heatmap = self.mse_loss(heatmap_out, heatmap, maskmap)
loss_paf = self.mse_loss(paf_out, vecmap, maskmap)
loss_associate = self.embeding_loss(embed_out, tagmap,
num_objects)
loss = loss_label + loss_heatmap + loss_paf + loss_associate
self.val_losses.update(loss.item(), inputs.size(0))
self.val_loss_heatmap.update(loss_heatmap.item(),
inputs.size(0))
self.val_loss_associate.update(loss_associate.item(),
inputs.size(0))
# Update the vars of the val phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.module_utilizer.save_net(self.pose_net, metric='iters')
Log.info('Loss Heatmap:{}, Loss Asso: {}'.format(
self.val_loss_heatmap.avg, self.val_loss_associate.avg))
# Print the log info & reset the states.
Log.info(
'Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t'
'Loss {loss.avg:.8f}\n'.format(batch_time=self.batch_time,
loss=self.val_losses))
self.batch_time.reset()
self.val_losses.reset()
self.pose_net.train()
def train(self):
cudnn.benchmark = True
while self.configer.get('epoch') < self.configer.get(
'solver', 'max_epoch'):
self.__train()
if self.configer.get('epoch') == self.configer.get(
'solver', 'max_epoch'):
break
class OpenPose(object):
"""
The class for Pose Estimation. Include train, val, test & predict.
"""
def __init__(self, configer):
self.configer = configer
self.batch_time = AverageMeter()
self.data_time = AverageMeter()
self.train_losses = AverageMeter()
self.train_loss_heatmap = AverageMeter()
self.train_loss_associate = AverageMeter()
self.val_losses = AverageMeter()
self.val_loss_heatmap = AverageMeter()
self.val_loss_associate = AverageMeter()
self.pose_visualizer = PoseVisualizer(configer)
self.pose_loss_manager = PoseLossManager(configer)
self.pose_model_manager = PoseModelManager(configer)
self.pose_data_loader = PoseDataLoader(configer)
self.module_utilizer = ModuleUtilizer(configer)
self.optim_scheduler = OptimScheduler(configer)
self.heatmap_generator = HeatmapGenerator(configer)
self.paf_generator = PafGenerator(configer)
self.data_transformer = DataTransformer(configer)
self.pose_net = None
self.train_loader = None
self.val_loader = None
self.optimizer = None
self.scheduler = None
self._init_model()
def _init_model(self):
self.pose_net = self.pose_model_manager.multi_pose_detector()
self.pose_net = self.module_utilizer.load_net(self.pose_net)
self.optimizer, self.scheduler = self.optim_scheduler.init_optimizer(
self._get_parameters())
self.train_loader = self.pose_data_loader.get_trainloader()
self.val_loader = self.pose_data_loader.get_valloader()
self.weights = self.configer.get('network', 'loss_weights')
self.mse_loss = self.pose_loss_manager.get_pose_loss('mse_loss')
def _get_parameters(self):
lr_1 = []
lr_2 = []
lr_4 = []
lr_8 = []
params_dict = dict(self.pose_net.named_parameters())
for key, value in params_dict.items():
if ('model1_' not in key) and ('model0.'
not in key) and ('backbone.'
not in key):
if key[-4:] == 'bias':
lr_8.append(value)
else:
lr_4.append(value)
elif key[-4:] == 'bias':
lr_2.append(value)
else:
lr_1.append(value)
params = [{
'params': lr_1,
'lr': self.configer.get('lr', 'base_lr')
}, {
'params': lr_2,
'lr': self.configer.get('lr', 'base_lr') * 2.,
'weight_decay': 0.0
}, {
'params': lr_4,
'lr': self.configer.get('lr', 'base_lr') * 4.
}, {
'params': lr_8,
'lr': self.configer.get('lr', 'base_lr') * 8.,
'weight_decay': 0.0
}]
return params
def __train(self):
"""
Train function of every epoch during train phase.
"""
self.pose_net.train()
start_time = time.time()
# Adjust the learning rate after every epoch.
self.configer.plus_one('epoch')
self.scheduler.step(self.configer.get('epoch'))
# data_tuple: (inputs, heatmap, maskmap, vecmap)
for i, data_dict in enumerate(self.train_loader):
inputs = data_dict['img']
maskmap = data_dict['maskmap']
input_size = [inputs.size(3), inputs.size(2)]
heatmap = self.heatmap_generator(data_dict['kpts'],
input_size,
maskmap=maskmap)
vecmap = self.paf_generator(data_dict['kpts'],
input_size,
maskmap=maskmap)
self.data_time.update(time.time() - start_time)
# Change the data type.
inputs, heatmap, maskmap, vecmap = self.module_utilizer.to_device(
inputs, heatmap, maskmap, vecmap)
# Forward pass.
paf_out, heatmap_out = self.pose_net(inputs)
# Compute the loss of the train batch & backward.
loss_heatmap = self.mse_loss(heatmap_out,
heatmap,
mask=maskmap,
weights=self.weights)
loss_associate = self.mse_loss(paf_out,
vecmap,
mask=maskmap,
weights=self.weights)
loss = loss_heatmap + loss_associate
self.train_losses.update(loss.item(), inputs.size(0))
self.train_loss_heatmap.update(loss_heatmap.item(), inputs.size(0))
self.train_loss_associate.update(loss_associate.item(),
inputs.size(0))
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# Update the vars of the train phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.configer.plus_one('iters')
# Print the log info & reset the states.
if self.configer.get('iters') % self.configer.get(
'solver', 'display_iter') == 0:
Log.info('Loss Heatmap:{}, Loss Asso: {}'.format(
self.train_loss_heatmap.avg,
self.train_loss_associate.avg))
Log.info(
'Train Epoch: {0}\tTrain Iteration: {1}\t'
'Time {batch_time.sum:.3f}s / {2}iters, ({batch_time.avg:.3f})\t'
'Data load {data_time.sum:.3f}s / {2}iters, ({data_time.avg:3f})\n'
'Learning rate = {3}\tLoss = {loss.val:.8f} (ave = {loss.avg:.8f})\n'
.format(self.configer.get('epoch'),
self.configer.get('iters'),
self.configer.get('solver', 'display_iter'),
self.scheduler.get_lr(),
batch_time=self.batch_time,
data_time=self.data_time,
loss=self.train_losses))
self.batch_time.reset()
self.data_time.reset()
self.train_losses.reset()
self.train_loss_heatmap.reset()
self.train_loss_associate.reset()
# Check to val the current model.
if self.val_loader is not None and \
self.configer.get('iters') % self.configer.get('solver', 'test_interval') == 0:
self.__val()
def __val(self):
"""
Validation function during the train phase.
"""
self.pose_net.eval()
start_time = time.time()
with torch.no_grad():
for i, data_dict in enumerate(self.val_loader):
inputs = data_dict['img']
maskmap = data_dict['maskmap']
input_size = [inputs.size(3), inputs.size(2)]
heatmap = self.heatmap_generator(data_dict['kpts'],
input_size,
maskmap=maskmap)
vecmap = self.paf_generator(data_dict['kpts'],
input_size,
maskmap=maskmap)
# Change the data type.
inputs, heatmap, maskmap, vecmap = self.module_utilizer.to_device(
inputs, heatmap, maskmap, vecmap)
# Forward pass.
paf_out, heatmap_out = self.pose_net(inputs)
# Compute the loss of the val batch.
loss_heatmap = self.mse_loss(heatmap_out[-1], heatmap, maskmap)
loss_associate = self.mse_loss(paf_out[-1], vecmap, maskmap)
loss = loss_heatmap + loss_associate
self.val_losses.update(loss.item(), inputs.size(0))
self.val_loss_heatmap.update(loss_heatmap.item(),
inputs.size(0))
self.val_loss_associate.update(loss_associate.item(),
inputs.size(0))
# Update the vars of the val phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.module_utilizer.save_net(self.pose_net, save_mode='iters')
Log.info('Loss Heatmap:{}, Loss Asso: {}'.format(
self.val_loss_heatmap.avg, self.val_loss_associate.avg))
# Print the log info & reset the states.
Log.info(
'Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t'
'Loss {loss.avg:.8f}\n'.format(batch_time=self.batch_time,
loss=self.val_losses))
self.batch_time.reset()
self.val_losses.reset()
self.val_loss_heatmap.reset()
self.val_loss_associate.reset()
self.pose_net.train()
def train(self):
cudnn.benchmark = True
if self.configer.get('network',
'resume') is not None and self.configer.get(
'network', 'resume_val'):
self.__val()
while self.configer.get('epoch') < self.configer.get(
'solver', 'max_epoch'):
self.__train()
if self.configer.get('epoch') == self.configer.get(
'solver', 'max_epoch'):
break
class ConvPoseMachine(object):
"""
The class for Pose Estimation. Include train, val, val & predict.
"""
def __init__(self, configer):
self.configer = configer
self.batch_time = AverageMeter()
self.data_time = AverageMeter()
self.train_losses = AverageMeter()
self.val_losses = AverageMeter()
self.pose_visualizer = PoseVisualizer(configer)
self.loss_manager = PoseLossManager(configer)
self.model_manager = PoseModelManager(configer)
self.train_utilizer = ModuleUtilizer(configer)
self.pose_net = None
self.train_loader = None
self.val_loader = None
self.optimizer = None
self.best_model_loss = None
self.is_best = None
self.lr = None
self.iters = None
def init_model(self, train_loader=None, val_loader=None):
self.pose_net = self.model_manager.pose_detector()
self.pose_net, self.iters = self.train_utilizer.load_net(self.pose_net)
self.optimizer = self.train_utilizer.update_optimizer(self.pose_net, self.iters)
self.train_loader = train_loader
self.val_loader = val_loader
self.heatmap_loss = self.loss_manager.get_pose_loss('heatmap_loss')
def __train(self):
"""
Train function of every epoch during train phase.
"""
self.pose_net.train()
start_time = time.time()
# data_tuple: (inputs, heatmap, maskmap, tagmap, num_objects)
for i, data_tuple in enumerate(self.train_loader):
self.data_time.update(time.time() - start_time)
# Change the data type.
if len(data_tuple) < 2:
Log.error('Train Loader Error!')
exit(0)
inputs = Variable(data_tuple[0].cuda(async=True))
heatmap = Variable(data_tuple[1].cuda(async=True))
maskmap = None
if len(data_tuple) > 2:
maskmap = Variable(data_tuple[2].cuda(async=True))
self.pose_visualizer.vis_tensor(heatmap, name='heatmap')
self.pose_visualizer.vis_tensor((inputs*256+128)/255, name='image')
# Forward pass.
outputs = self.pose_net(inputs)
self.pose_visualizer.vis_tensor(outputs, name='output')
self.pose_visualizer.vis_peaks(inputs, outputs, name='peak')
# Compute the loss of the train batch & backward.
loss_heatmap = self.heatmap_loss(outputs, heatmap, maskmap)
loss = loss_heatmap
self.train_losses.update(loss.data[0], inputs.size(0))
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# Update the vars of the train phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.iters += 1
# Print the log info & reset the states.
if self.iters % self.configer.get('solver', 'display_iter') == 0:
Log.info('Train Iteration: {0}\t'
'Time {batch_time.sum:.3f}s / {1}iters, ({batch_time.avg:.3f})\t'
'Data load {data_time.sum:.3f}s / {1}iters, ({data_time.avg:3f})\n'
'Learning rate = {2}\n'
'Loss = {loss.val:.8f} (ave = {loss.avg:.8f})\n'.format(
self.iters, self.configer.get('solver', 'display_iter'), self.lr, batch_time=self.batch_time,
data_time=self.data_time, loss=self.train_losses))
self.batch_time.reset()
self.data_time.reset()
self.train_losses.reset()
# Check to val the current model.
if self.val_loader is not None and \
self.iters % self.configer.get('solver', 'test_interval') == 0:
self.__val()
self.optimizer = self.train_utilizer.update_optimizer(self.pose_net, self.iters)
def __val(self):
"""
Validation function during the train phase.
"""
self.pose_net.eval()
start_time = time.time()
for j, data_tuple in enumerate(self.val_loader):
# Change the data type.
inputs = Variable(data_tuple[0].cuda(async=True), volatile=True)
heatmap = Variable(data_tuple[1].cuda(async=True), volatile=True)
maskmap = None
if len(data_tuple) > 2:
maskmap = Variable(data_tuple[2].cuda(async=True), volatile=True)
# Forward pass.
outputs = self.pose_net(inputs)
self.pose_visualizer.vis_peaks(inputs, outputs, name='peak_val')
# Compute the loss of the val batch.
loss_heatmap = self.heatmap_loss(outputs, heatmap, maskmap)
loss = loss_heatmap
self.val_losses.update(loss.data[0], inputs.size(0))
# Update the vars of the val phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
# Print the log info & reset the states.
Log.info(
'Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t'
'Loss {loss.avg:.8f}\n'.format(
batch_time=self.batch_time, loss=self.val_losses))
self.batch_time.reset()
self.val_losses.reset()
self.pose_net.train()
def train(self):
cudnn.benchmark = True
while self.iters < self.configer.get('solver', 'max_iter'):
self.__train()
if self.iters == self.configer.get('solver', 'max_iter'):
break
def test(self, img_path=None, img_dir=None):
if img_path is not None and os.path.exists(img_path):
image = Image.open(img_path).convert('RGB')
class OpenPose(object):
"""
The class for Pose Estimation. Include train, val, test & predict.
"""
def __init__(self, configer):
self.configer = configer
self.batch_time = AverageMeter()
self.data_time = AverageMeter()
self.train_losses = AverageMeter()
self.val_losses = AverageMeter()
self.vis = PoseVisualizer(configer)
self.loss_manager = PoseLossManager(configer)
self.model_manager = PoseModelManager(configer)
self.data_loader = PoseDataLoader(configer)
self.module_utilizer = ModuleUtilizer(configer)
self.pose_net = None
self.train_loader = None
self.val_loader = None
self.optimizer = None
self.lr = None
self.iters = None
def init_model(self):
self.pose_net = self.model_manager.pose_detector()
self.iters = 0
self.pose_net, _ = self.module_utilizer.load_net(self.pose_net)
self.optimizer, self.lr = self.module_utilizer.update_optimizer(
self.pose_net, self.iters)
if self.configer.get('dataset') == 'coco':
self.train_loader = self.data_loader.get_trainloader(OPCocoLoader)
self.val_loader = self.data_loader.get_valloader(OPCocoLoader)
else:
Log.error('Dataset: {} is not valid!'.format(
self.configer.get('dataset')))
exit(1)
self.mse_loss = self.loss_manager.get_pose_loss('mse_loss')
def __train(self):
"""
Train function of every epoch during train phase.
"""
self.pose_net.train()
start_time = time.time()
# data_tuple: (inputs, heatmap, maskmap, vecmap)
for i, data_tuple in enumerate(self.train_loader):
self.data_time.update(time.time() - start_time)
# Change the data type.
if len(data_tuple) < 2:
Log.error('Train Loader Error!')
exit(0)
inputs = Variable(data_tuple[0].cuda(async=True))
heatmap = Variable(data_tuple[1].cuda(async=True))
maskmap = None
if len(data_tuple) > 2:
maskmap = Variable(data_tuple[2].cuda(async=True))
# Forward pass.
paf_out, heatmap_out = self.pose_net(inputs)
self.vis.vis_paf(paf_out,
inputs.data.cpu().squeeze().numpy().transpose(
1, 2, 0),
name='paf_out')
# Compute the loss of the train batch & backward.
loss_heatmap = self.mse_loss(heatmap_out, heatmap, maskmap)
loss = loss_heatmap
if len(data_tuple) > 3:
vecmap = Variable(data_tuple[3].cuda(async=True))
self.vis.vis_paf(vecmap,
inputs.data.cpu().squeeze().numpy().transpose(
1, 2, 0),
name='paf')
loss_associate = self.mse_loss(paf_out, vecmap, maskmap)
loss += loss_associate
self.train_losses.update(loss.data[0], inputs.size(0))
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# Update the vars of the train phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.iters += 1
# Print the log info & reset the states.
if self.iters % self.configer.get('solver', 'display_iter') == 0:
Log.info(
'Train Iteration: {0}\t'
'Time {batch_time.sum:.3f}s / {1}iters, ({batch_time.avg:.3f})\t'
'Data load {data_time.sum:.3f}s / {1}iters, ({data_time.avg:3f})\n'
'Learning rate = {2}\n'
'Loss = {loss.val:.8f} (ave = {loss.avg:.8f})\n'.format(
self.iters,
self.configer.get('solver', 'display_iter'),
self.lr,
batch_time=self.batch_time,
data_time=self.data_time,
loss=self.train_losses))
self.batch_time.reset()
self.data_time.reset()
self.train_losses.reset()
# Check to val the current model.
if self.val_loader is not None and \
self.iters % self.configer.get('solver', 'test_interval') == 0:
self.__val()
# Adjust the learning rate after every iteration.
self.optimizer, self.lr = self.module_utilizer.update_optimizer(
self.pose_net, self.iters)
def __val(self):
"""
Validation function during the train phase.
"""
self.pose_net.eval()
start_time = time.time()
for j, data_tuple in enumerate(self.val_loader):
# Change the data type.
inputs = Variable(data_tuple[0].cuda(async=True), volatile=True)
heatmap = Variable(data_tuple[1].cuda(async=True), volatile=True)
maskmap = None
if len(data_tuple) > 2:
maskmap = Variable(data_tuple[2].cuda(async=True),
volatile=True)
# Forward pass.
paf_out, heatmap_out = self.pose_net(inputs)
# Compute the loss of the val batch.
loss_heatmap = self.mse_loss(heatmap_out, heatmap, maskmap)
loss = loss_heatmap
if len(data_tuple) > 3:
vecmap = Variable(data_tuple[3].cuda(async=True),
volatile=True)
loss_associate = self.mse_loss(paf_out, vecmap, maskmap)
loss = loss_heatmap + loss_associate
self.val_losses.update(loss.data[0], inputs.size(0))
# Update the vars of the val phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.module_utilizer.save_net(self.pose_net, self.iters)
# Print the log info & reset the states.
Log.info('Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t'
'Loss {loss.avg:.8f}\n'.format(batch_time=self.batch_time,
loss=self.val_losses))
self.batch_time.reset()
self.val_losses.reset()
self.pose_net.train()
def train(self):
cudnn.benchmark = True
while self.iters < self.configer.get('solver', 'max_iter'):
self.__train()
if self.iters == self.configer.get('solver', 'max_iter'):
break
class ConvPoseMachine(object):
"""
The class for Pose Estimation. Include train, val, val & predict.
"""
def __init__(self, configer):
self.configer = configer
self.batch_time = AverageMeter()
self.data_time = AverageMeter()
self.train_losses = AverageMeter()
self.val_losses = AverageMeter()
self.pose_visualizer = PoseVisualizer(configer)
self.loss_manager = PoseLossManager(configer)
self.model_manager = PoseModelManager(configer)
self.train_utilizer = ModuleUtilizer(configer)
self.pose_net = None
self.train_loader = None
self.val_loader = None
self.optimizer = None
self.best_model_loss = None
self.is_best = None
self.lr = None
self.iters = None
def init_model(self, train_loader=None, val_loader=None):
self.pose_net = self.model_manager.pose_detector()
self.pose_net, self.iters = self.train_utilizer.load_net(self.pose_net)
self.optimizer = self.train_utilizer.update_optimizer(
self.pose_net, self.iters)
self.train_loader = train_loader
self.val_loader = val_loader
self.heatmap_loss = self.loss_manager.get_pose_loss('heatmap_loss')
def __train(self):
"""
Train function of every epoch during train phase.
"""
self.pose_net.train()
start_time = time.time()
# data_tuple: (inputs, heatmap, maskmap, tagmap, num_objects)
for i, data_tuple in enumerate(self.train_loader):
self.data_time.update(time.time() - start_time)
# Change the data type.
if len(data_tuple) < 2:
Log.error('Train Loader Error!')
exit(0)
inputs = Variable(data_tuple[0].cuda(async=True))
heatmap = Variable(data_tuple[1].cuda(async=True))
maskmap = None
if len(data_tuple) > 2:
maskmap = Variable(data_tuple[2].cuda(async=True))
self.pose_visualizer.vis_tensor(heatmap, name='heatmap')
self.pose_visualizer.vis_tensor((inputs * 256 + 128) / 255,
name='image')
# Forward pass.
outputs = self.pose_net(inputs)
self.pose_visualizer.vis_tensor(outputs, name='output')
self.pose_visualizer.vis_peaks(inputs, outputs, name='peak')
# Compute the loss of the train batch & backward.
loss_heatmap = self.heatmap_loss(outputs, heatmap, maskmap)
loss = loss_heatmap
self.train_losses.update(loss.data[0], inputs.size(0))
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# Update the vars of the train phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.iters += 1
# Print the log info & reset the states.
if self.iters % self.configer.get('solver', 'display_iter') == 0:
Log.info(
'Train Iteration: {0}\t'
'Time {batch_time.sum:.3f}s / {1}iters, ({batch_time.avg:.3f})\t'
'Data load {data_time.sum:.3f}s / {1}iters, ({data_time.avg:3f})\n'
'Learning rate = {2}\n'
'Loss = {loss.val:.8f} (ave = {loss.avg:.8f})\n'.format(
self.iters,
self.configer.get('solver', 'display_iter'),
self.lr,
batch_time=self.batch_time,
data_time=self.data_time,
loss=self.train_losses))
self.batch_time.reset()
self.data_time.reset()
self.train_losses.reset()
# Check to val the current model.
if self.val_loader is not None and \
self.iters % self.configer.get('solver', 'test_interval') == 0:
self.__val()
self.optimizer = self.train_utilizer.update_optimizer(
self.pose_net, self.iters)
def __val(self):
"""
Validation function during the train phase.
"""
self.pose_net.eval()
start_time = time.time()
for j, data_tuple in enumerate(self.val_loader):
# Change the data type.
inputs = Variable(data_tuple[0].cuda(async=True), volatile=True)
heatmap = Variable(data_tuple[1].cuda(async=True), volatile=True)
maskmap = None
if len(data_tuple) > 2:
maskmap = Variable(data_tuple[2].cuda(async=True),
volatile=True)
# Forward pass.
outputs = self.pose_net(inputs)
self.pose_visualizer.vis_peaks(inputs, outputs, name='peak_val')
# Compute the loss of the val batch.
loss_heatmap = self.heatmap_loss(outputs, heatmap, maskmap)
loss = loss_heatmap
self.val_losses.update(loss.data[0], inputs.size(0))
# Update the vars of the val phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
# Print the log info & reset the states.
Log.info('Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t'
'Loss {loss.avg:.8f}\n'.format(batch_time=self.batch_time,
loss=self.val_losses))
self.batch_time.reset()
self.val_losses.reset()
self.pose_net.train()
def train(self):
cudnn.benchmark = True
while self.iters < self.configer.get('solver', 'max_iter'):
self.__train()
if self.iters == self.configer.get('solver', 'max_iter'):
break
def test(self, img_path=None, img_dir=None):
if img_path is not None and os.path.exists(img_path):
image = Image.open(img_path).convert('RGB')
class OpenPose(object):
"""
The class for Pose Estimation. Include train, val, test & predict.
"""
def __init__(self, configer):
self.configer = configer
self.batch_time = AverageMeter()
self.data_time = AverageMeter()
self.train_losses = AverageMeter()
self.val_losses = AverageMeter()
self.vis = PoseVisualizer(configer)
self.loss_manager = PoseLossManager(configer)
self.model_manager = PoseModelManager(configer)
self.data_loader = PoseDataLoader(configer)
self.module_utilizer = ModuleUtilizer(configer)
self.pose_net = None
self.train_loader = None
self.val_loader = None
self.optimizer = None
self.lr = None
self.iters = None
def init_model(self):
self.pose_net = self.model_manager.pose_detector()
self.iters = 0
self.pose_net, _ = self.module_utilizer.load_net(self.pose_net)
self.optimizer, self.lr = self.module_utilizer.update_optimizer(self.pose_net, self.iters)
if self.configer.get('dataset') == 'coco':
self.train_loader = self.data_loader.get_trainloader(OPCocoLoader)
self.val_loader = self.data_loader.get_valloader(OPCocoLoader)
else:
Log.error('Dataset: {} is not valid!'.format(self.configer.get('dataset')))
exit(1)
self.mse_loss = self.loss_manager.get_pose_loss('mse_loss')
def __train(self):
"""
Train function of every epoch during train phase.
"""
self.pose_net.train()
start_time = time.time()
# data_tuple: (inputs, heatmap, maskmap, vecmap)
for i, data_tuple in enumerate(self.train_loader):
self.data_time.update(time.time() - start_time)
# Change the data type.
if len(data_tuple) < 2:
Log.error('Train Loader Error!')
exit(0)
inputs = Variable(data_tuple[0].cuda(async=True))
heatmap = Variable(data_tuple[1].cuda(async=True))
maskmap = None
if len(data_tuple) > 2:
maskmap = Variable(data_tuple[2].cuda(async=True))
# Forward pass.
paf_out, heatmap_out = self.pose_net(inputs)
self.vis.vis_paf(paf_out, inputs.data.cpu().squeeze().numpy().transpose(1, 2, 0), name='paf_out')
# Compute the loss of the train batch & backward.
loss_heatmap = self.mse_loss(heatmap_out, heatmap, maskmap)
loss = loss_heatmap
if len(data_tuple) > 3:
vecmap = Variable(data_tuple[3].cuda(async=True))
self.vis.vis_paf(vecmap, inputs.data.cpu().squeeze().numpy().transpose(1, 2, 0), name='paf')
loss_associate = self.mse_loss(paf_out, vecmap, maskmap)
loss += loss_associate
self.train_losses.update(loss.data[0], inputs.size(0))
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# Update the vars of the train phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.iters += 1
# Print the log info & reset the states.
if self.iters % self.configer.get('solver', 'display_iter') == 0:
Log.info('Train Iteration: {0}\t'
'Time {batch_time.sum:.3f}s / {1}iters, ({batch_time.avg:.3f})\t'
'Data load {data_time.sum:.3f}s / {1}iters, ({data_time.avg:3f})\n'
'Learning rate = {2}\n'
'Loss = {loss.val:.8f} (ave = {loss.avg:.8f})\n'.format(
self.iters, self.configer.get('solver', 'display_iter'), self.lr, batch_time=self.batch_time,
data_time=self.data_time, loss=self.train_losses))
self.batch_time.reset()
self.data_time.reset()
self.train_losses.reset()
# Check to val the current model.
if self.val_loader is not None and \
self.iters % self.configer.get('solver', 'test_interval') == 0:
self.__val()
# Adjust the learning rate after every iteration.
self.optimizer, self.lr = self.module_utilizer.update_optimizer(self.pose_net, self.iters)
def __val(self):
"""
Validation function during the train phase.
"""
self.pose_net.eval()
start_time = time.time()
for j, data_tuple in enumerate(self.val_loader):
# Change the data type.
inputs = Variable(data_tuple[0].cuda(async=True), volatile=True)
heatmap = Variable(data_tuple[1].cuda(async=True), volatile=True)
maskmap = None
if len(data_tuple) > 2:
maskmap = Variable(data_tuple[2].cuda(async=True), volatile=True)
# Forward pass.
paf_out, heatmap_out = self.pose_net(inputs)
# Compute the loss of the val batch.
loss_heatmap = self.mse_loss(heatmap_out, heatmap, maskmap)
loss = loss_heatmap
if len(data_tuple) > 3:
vecmap = Variable(data_tuple[3].cuda(async=True), volatile=True)
loss_associate = self.mse_loss(paf_out, vecmap, maskmap)
loss = loss_heatmap + loss_associate
self.val_losses.update(loss.data[0], inputs.size(0))
# Update the vars of the val phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.module_utilizer.save_net(self.pose_net, self.iters)
# Print the log info & reset the states.
Log.info(
'Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t'
'Loss {loss.avg:.8f}\n'.format(
batch_time=self.batch_time, loss=self.val_losses))
self.batch_time.reset()
self.val_losses.reset()
self.pose_net.train()
def train(self):
cudnn.benchmark = True
while self.iters < self.configer.get('solver', 'max_iter'):
self.__train()
if self.iters == self.configer.get('solver', 'max_iter'):
break