def run(self, loss_current_iteration=0, use_pretrained=False):
"""
Runs the training
:return:
:rtype:
"""
start_iteration = copy.copy(loss_current_iteration)
DCE = DenseCorrespondenceEvaluation
self.setup()
self.save_configs()
if not use_pretrained:
# create new network and optimizer
self._dcn = self.build_network()
self._optimizer = self._construct_optimizer(self._dcn.parameters())
else:
logging.info("using pretrained model")
if (self._dcn is None):
raise ValueError("you must set self._dcn if use_pretrained=True")
if (self._optimizer is None):
raise ValueError("you must set self._optimizer if use_pretrained=True")
# make sure network is using cuda and is in train mode
dcn = self._dcn
dcn.cuda()
dcn.train()
optimizer = self._optimizer
batch_size = self._data_loader.batch_size
pixelwise_contrastive_loss = PixelwiseContrastiveLoss(image_shape=dcn.image_shape, config=self._config['loss_function'])
pixelwise_contrastive_loss.debug = True
loss = match_loss = non_match_loss = 0
max_num_iterations = self._config['training']['num_iterations'] + start_iteration
logging_rate = self._config['training']['logging_rate']
save_rate = self._config['training']['save_rate']
compute_test_loss_rate = self._config['training']['compute_test_loss_rate']
# logging
self._logging_dict = dict()
self._logging_dict['train'] = {"iteration": [], "loss": [], "match_loss": [],
"masked_non_match_loss": [],
"background_non_match_loss": [],
"blind_non_match_loss": [],
"learning_rate": [],
"different_object_non_match_loss": []}
self._logging_dict['test'] = {"iteration": [], "loss": [], "match_loss": [],
"non_match_loss": []}
# save network before starting
if not use_pretrained:
self.save_network(dcn, optimizer, 0)
for epoch in range(50): # loop over the dataset multiple times
for i, data in enumerate(self._data_loader, 0):
loss_current_iteration += 1
start_iter = time.time()
match_type, \
img_a, img_b, \
matches_a, matches_b, \
masked_non_matches_a, masked_non_matches_b, \
background_non_matches_a, background_non_matches_b, \
blind_non_matches_a, blind_non_matches_b, \
metadata = data
if (match_type == -1).all():
print "\n empty data, continuing \n"
continue
data_type = metadata["type"][0]
img_a = Variable(img_a.cuda(), requires_grad=False)
img_b = Variable(img_b.cuda(), requires_grad=False)
matches_a = Variable(matches_a.cuda().squeeze(0), requires_grad=False)
matches_b = Variable(matches_b.cuda().squeeze(0), requires_grad=False)
masked_non_matches_a = Variable(masked_non_matches_a.cuda().squeeze(0), requires_grad=False)
masked_non_matches_b = Variable(masked_non_matches_b.cuda().squeeze(0), requires_grad=False)
background_non_matches_a = Variable(background_non_matches_a.cuda().squeeze(0), requires_grad=False)
background_non_matches_b = Variable(background_non_matches_b.cuda().squeeze(0), requires_grad=False)
blind_non_matches_a = Variable(blind_non_matches_a.cuda().squeeze(0), requires_grad=False)
blind_non_matches_b = Variable(blind_non_matches_b.cuda().squeeze(0), requires_grad=False)
optimizer.zero_grad()
self.adjust_learning_rate(optimizer, loss_current_iteration)
# run both images through the network
image_a_pred = dcn.forward(img_a)
image_a_pred = dcn.process_network_output(image_a_pred, batch_size)
image_b_pred = dcn.forward(img_b)
image_b_pred = dcn.process_network_output(image_b_pred, batch_size)
# get loss
loss, match_loss, masked_non_match_loss, \
background_non_match_loss, blind_non_match_loss = loss_composer.get_loss(pixelwise_contrastive_loss, match_type,
image_a_pred, image_b_pred,
matches_a, matches_b,
masked_non_matches_a, masked_non_matches_b,
background_non_matches_a, background_non_matches_b,
blind_non_matches_a, blind_non_matches_b)
loss.backward()
optimizer.step()
elapsed = time.time() - start_iter
print "single iteration took %.3f seconds" %(elapsed)
def update_visdom_plots(loss, match_loss, masked_non_match_loss, background_non_match_loss, blind_non_match_loss):
"""
Updates the visdom plots with current loss function information
:return:
:rtype:
"""
learning_rate = DenseCorrespondenceTraining.get_learning_rate(optimizer)
self._logging_dict['train']['learning_rate'].append(learning_rate)
self._visdom_plots['learning_rate'].log(loss_current_iteration, learning_rate)
self._tensorboard_logger.log_value("learning rate", learning_rate, loss_current_iteration)
# Don't update any plots if the entry corresponding to that term
# is a zero loss
if not loss_composer.is_zero_loss(match_loss):
self._logging_dict['train']['match_loss'].append(match_loss.data[0])
self._visdom_plots['train']['match_loss'].log(loss_current_iteration, match_loss.data[0])
self._tensorboard_logger.log_value("train match loss", match_loss.data[0], loss_current_iteration)
if not loss_composer.is_zero_loss(masked_non_match_loss):
self._logging_dict['train']['masked_non_match_loss'].append(masked_non_match_loss.data[0])
self._visdom_plots['train']['masked_non_match_loss'].log(loss_current_iteration,
masked_non_match_loss.data[0])
self._tensorboard_logger.log_value("train masked non match loss", masked_non_match_loss.data[0], loss_current_iteration)
if not loss_composer.is_zero_loss(background_non_match_loss):
self._logging_dict['train']['background_non_match_loss'].append(background_non_match_loss.data[0])
self._visdom_plots['train']['background_non_match_loss'].log(loss_current_iteration,
background_non_match_loss.data[0])
self._tensorboard_logger.log_value("train background non match loss", background_non_match_loss.data[0], loss_current_iteration)
if not loss_composer.is_zero_loss(blind_non_match_loss):
if data_type == SpartanDatasetDataType.SINGLE_OBJECT_WITHIN_SCENE:
self._tensorboard_logger.log_value("train blind SINGLE_OBJECT_WITHIN_SCENE", blind_non_match_loss.data[0], loss_current_iteration)
if data_type == SpartanDatasetDataType.DIFFERENT_OBJECT:
self._tensorboard_logger.log_value("train blind DIFFERENT_OBJECT", blind_non_match_loss.data[0], loss_current_iteration)
# loss is never zero
if data_type == SpartanDatasetDataType.SINGLE_OBJECT_WITHIN_SCENE:
self._tensorboard_logger.log_value("train loss SINGLE_OBJECT_WITHIN_SCENE", loss.data[0], loss_current_iteration)
elif data_type == SpartanDatasetDataType.DIFFERENT_OBJECT:
self._tensorboard_logger.log_value("train loss DIFFERENT_OBJECT", loss.data[0], loss_current_iteration)
elif data_type == SpartanDatasetDataType.SINGLE_OBJECT_ACROSS_SCENE:
self._tensorboard_logger.log_value("train loss SINGLE_OBJECT_ACROSS_SCENE", loss.data[0], loss_current_iteration)
elif data_type == SpartanDatasetDataType.MULTI_OBJECT:
self._tensorboard_logger.log_value("train loss MULTI_OBJECT", loss.data[0], loss_current_iteration)
elif data_type == SpartanDatasetDataType.SYNTHETIC_MULTI_OBJECT:
self._tensorboard_logger.log_value("train loss SYNTHETIC_MULTI_OBJECT", loss.data[0], loss_current_iteration)
else:
raise ValueError("unknown data type")
if data_type == SpartanDatasetDataType.DIFFERENT_OBJECT:
self._tensorboard_logger.log_value("train different object", loss.data[0], loss_current_iteration)
# #non_match_type = metadata['non_match_type'][0]
# fraction_hard_negatives = pixelwise_contrastive_loss.debug_data['fraction_hard_negatives']
# if pixelwise_contrastive_loss.debug:
# if non_match_type == "masked":
# self._visdom_plots['masked_hard_negative_rate'].log(loss_current_iteration, fraction_hard_negatives)
# self._tensorboard_logger.log_value("masked hard negative rate", fraction_hard_negatives, loss_current_iteration)
# elif non_match_type == "non_masked":
# self._visdom_plots['non_masked_hard_negative_rate'].log(loss_current_iteration,
# fraction_hard_negatives)
# self._tensorboard_logger.log_value("non-masked hard negative rate", fraction_hard_negatives,
# loss_current_iteration)
# else:
# raise ValueError("uknown non_match_type %s" %(non_match_type))
# def update_visdom_test_loss_plots(test_loss, test_match_loss, test_non_match_loss):
# """
# Log data about test loss and update the visdom plots
# :return:
# :rtype:
# """
# self._logging_dict['test']['loss'].append(test_loss)
# self._logging_dict['test']['match_loss'].append(test_match_loss)
# self._logging_dict['test']['non_match_loss'].append(test_non_match_loss)
# self._logging_dict['test']['iteration'].append(loss_current_iteration)
# self._visdom_plots['test']['loss'].log(loss_current_iteration, test_loss)
# self._visdom_plots['test']['match_loss'].log(loss_current_iteration, test_match_loss)
# self._visdom_plots['test']['non_match_loss'].log(loss_current_iteration, test_non_match_loss)
# self._tensorboard_logger.log_value('test loss', test_loss, loss_current_iteration)
# self._tensorboard_logger.log_value('test match loss', test_match_loss, loss_current_iteration)
# self._tensorboard_logger.log_value('test non-match loss', test_non_match_loss, loss_current_iteration)
update_visdom_plots(loss, match_loss, masked_non_match_loss, background_non_match_loss, blind_non_match_loss)
if loss_current_iteration % save_rate == 0:
self.save_network(dcn, optimizer, loss_current_iteration, logging_dict=self._logging_dict)
if loss_current_iteration % logging_rate == 0:
logging.info("Training on iteration %d of %d" %(loss_current_iteration, max_num_iterations))
logging.info("single iteration took %.3f seconds" %(elapsed))
percent_complete = loss_current_iteration * 100.0/(max_num_iterations - start_iteration)
logging.info("Training is %d percent complete\n" %(percent_complete))
# don't compute the test loss on the first few times through the loop
if self._config["training"]["compute_test_loss"] and (loss_current_iteration % compute_test_loss_rate == 0) and loss_current_iteration > 5:
logging.info("Computing test loss")
# delete the loss, match_loss, non_match_loss variables so that
# pytorch can use that GPU memory
del loss, match_loss, masked_non_match_loss, background_non_match_loss, blind_non_match_loss
gc.collect()
dcn.eval()
test_loss, test_match_loss, test_non_match_loss = DCE.compute_loss_on_dataset(dcn,
self._data_loader_test, self._config['loss_function'], num_iterations=self._config['training']['test_loss_num_iterations'])
update_visdom_test_loss_plots(test_loss, test_match_loss, test_non_match_loss)
# delete these variables so we can free GPU memory
del test_loss, test_match_loss, test_non_match_loss
# make sure to set the network back to train mode
dcn.train()
if loss_current_iteration % self._config['training']['garbage_collect_rate'] == 0:
logging.debug("running garbage collection")
gc_start = time.time()
gc.collect()
gc_elapsed = time.time() - gc_start
logging.debug("garbage collection took %.2d seconds" %(gc_elapsed))
if loss_current_iteration > max_num_iterations:
logging.info("Finished testing after %d iterations" % (max_num_iterations))
self.save_network(dcn, optimizer, loss_current_iteration, logging_dict=self._logging_dict)
return
def run(self, loss_current_iteration=0, use_pretrained=False):
"""
Runs the training
:return:
:rtype:
"""
start_iteration = copy.copy(loss_current_iteration)
DCE = DenseCorrespondenceEvaluation
self.setup()
self.save_configs()
if not use_pretrained:
# create new network and optimizer
self._dcn = self.build_network()
self._optimizer = self._construct_optimizer(self._dcn.parameters())
else:
logging.info("using pretrained model")
if (self._dcn is None):
raise ValueError(
"you must set self._dcn if use_pretrained=True")
if (self._optimizer is None):
raise ValueError(
"you must set self._optimizer if use_pretrained=True")
# make sure network is using cuda and is in train mode
dcn = self._dcn
dcn.cuda()
dcn.train()
optimizer = self._optimizer
batch_size = self._data_loader.batch_size
pixelwise_contrastive_loss = PixelwiseContrastiveLoss(
image_shape=dcn.image_shape, config=self._config['loss_function'])
pixelwise_contrastive_loss.debug = True
loss = match_loss = non_match_loss = 0
max_num_iterations = self._config['training'][
'num_iterations'] + start_iteration
logging_rate = self._config['training']['logging_rate']
save_rate = self._config['training']['save_rate']
compute_test_loss_rate = self._config['training'][
'compute_test_loss_rate']
# logging
self._logging_dict = dict()
self._logging_dict['train'] = {
"iteration": [],
"loss": [],
"match_loss": [],
"masked_non_match_loss": [],
"background_non_match_loss": [],
"blind_non_match_loss": [],
"learning_rate": [],
"different_object_non_match_loss": []
}
self._logging_dict['test'] = {
"iteration": [],
"loss": [],
"match_loss": [],
"non_match_loss": []
}
# save network before starting
if not use_pretrained:
self.save_network(dcn, optimizer, 0)
# from training_progress_visualizer import TrainingProgressVisualizer
# TPV = TrainingProgressVisualizer()
for epoch in range(50): # loop over the dataset multiple times
for i, data in enumerate(self._data_loader, 0):
loss_current_iteration += 1
start_iter = time.time()
match_type, \
img_a, img_b, \
matches_a, matches_b, \
masked_non_matches_a, masked_non_matches_b, \
background_non_matches_a, background_non_matches_b, \
blind_non_matches_a, blind_non_matches_b, \
metadata = data
if (match_type == -1).all():
print("\n empty data, continuing \n")
continue
data_type = metadata["type"][0]
img_a = Variable(img_a.cuda(), requires_grad=False)
img_b = Variable(img_b.cuda(), requires_grad=False)
matches_a = Variable(matches_a.cuda().squeeze(0),
requires_grad=False)
matches_b = Variable(matches_b.cuda().squeeze(0),
requires_grad=False)
masked_non_matches_a = Variable(
masked_non_matches_a.cuda().squeeze(0),
requires_grad=False)
masked_non_matches_b = Variable(
masked_non_matches_b.cuda().squeeze(0),
requires_grad=False)
background_non_matches_a = Variable(
background_non_matches_a.cuda().squeeze(0),
requires_grad=False)
background_non_matches_b = Variable(
background_non_matches_b.cuda().squeeze(0),
requires_grad=False)
blind_non_matches_a = Variable(
blind_non_matches_a.cuda().squeeze(0), requires_grad=False)
blind_non_matches_b = Variable(
blind_non_matches_b.cuda().squeeze(0), requires_grad=False)
optimizer.zero_grad()
self.adjust_learning_rate(optimizer, loss_current_iteration)
# run both images through the network
image_a_pred = dcn.forward(img_a)
image_a_pred = dcn.process_network_output(
image_a_pred, batch_size)
image_b_pred = dcn.forward(img_b)
image_b_pred = dcn.process_network_output(
image_b_pred, batch_size)
# get loss
loss, match_loss, masked_non_match_loss, \
background_non_match_loss, blind_non_match_loss = loss_composer.get_loss(pixelwise_contrastive_loss, match_type,
image_a_pred, image_b_pred,
matches_a, matches_b,
masked_non_matches_a, masked_non_matches_b,
background_non_matches_a, background_non_matches_b,
blind_non_matches_a, blind_non_matches_b)
loss.backward()
optimizer.step()
#if i % 10 == 0:
# TPV.update(self._dataset, dcn, loss_current_iteration, now_training_object_id=metadata["object_id"])
elapsed = time.time() - start_iter
def update_plots(loss, match_loss, masked_non_match_loss,
background_non_match_loss,
blind_non_match_loss):
"""
Updates the tensorboard plots with current loss function information
:return:
:rtype:
"""
learning_rate = DenseCorrespondenceTraining.get_learning_rate(
optimizer)
self._logging_dict['train']['learning_rate'].append(
learning_rate)
self._tensorboard_logger.log_value("learning rate",
learning_rate,
loss_current_iteration)
# Don't update any plots if the entry corresponding to that term
# is a zero loss
if not loss_composer.is_zero_loss(match_loss):
self._logging_dict['train']['match_loss'].append(
match_loss.item())
self._tensorboard_logger.log_value(
"train match loss", match_loss.item(),
loss_current_iteration)
if not loss_composer.is_zero_loss(masked_non_match_loss):
self._logging_dict['train'][
'masked_non_match_loss'].append(
masked_non_match_loss.item())
self._tensorboard_logger.log_value(
"train masked non match loss",
masked_non_match_loss.item(),
loss_current_iteration)
if not loss_composer.is_zero_loss(
background_non_match_loss):
self._logging_dict['train'][
'background_non_match_loss'].append(
background_non_match_loss.item())
self._tensorboard_logger.log_value(
"train background non match loss",
background_non_match_loss.item(),
loss_current_iteration)
if not loss_composer.is_zero_loss(blind_non_match_loss):
if data_type == SpartanDatasetDataType.SINGLE_OBJECT_WITHIN_SCENE:
self._tensorboard_logger.log_value(
"train blind SINGLE_OBJECT_WITHIN_SCENE",
blind_non_match_loss.item(),
loss_current_iteration)
if data_type == SpartanDatasetDataType.DIFFERENT_OBJECT:
self._tensorboard_logger.log_value(
"train blind DIFFERENT_OBJECT",
blind_non_match_loss.item(),
loss_current_iteration)
# loss is never zero
if data_type == SpartanDatasetDataType.SINGLE_OBJECT_WITHIN_SCENE:
self._tensorboard_logger.log_value(
"train loss SINGLE_OBJECT_WITHIN_SCENE",
loss.item(), loss_current_iteration)
elif data_type == SpartanDatasetDataType.DIFFERENT_OBJECT:
self._tensorboard_logger.log_value(
"train loss DIFFERENT_OBJECT", loss.item(),
loss_current_iteration)
elif data_type == SpartanDatasetDataType.SINGLE_OBJECT_ACROSS_SCENE:
self._tensorboard_logger.log_value(
"train loss SINGLE_OBJECT_ACROSS_SCENE",
loss.item(), loss_current_iteration)
elif data_type == SpartanDatasetDataType.MULTI_OBJECT:
self._tensorboard_logger.log_value(
"train loss MULTI_OBJECT", loss.item(),
loss_current_iteration)
elif data_type == SpartanDatasetDataType.SYNTHETIC_MULTI_OBJECT:
self._tensorboard_logger.log_value(
"train loss SYNTHETIC_MULTI_OBJECT", loss.item(),
loss_current_iteration)
else:
raise ValueError("unknown data type")
if data_type == SpartanDatasetDataType.DIFFERENT_OBJECT:
self._tensorboard_logger.log_value(
"train different object", loss.item(),
loss_current_iteration)
update_plots(loss, match_loss, masked_non_match_loss,
background_non_match_loss, blind_non_match_loss)
if loss_current_iteration % save_rate == 0:
self.save_network(dcn,
optimizer,
loss_current_iteration,
logging_dict=self._logging_dict)
if loss_current_iteration % logging_rate == 0:
logging.info("Training on iteration %d of %d" %
(loss_current_iteration, max_num_iterations))
logging.info("single iteration took %.3f seconds" %
(elapsed))
percent_complete = loss_current_iteration * 100.0 / (
max_num_iterations - start_iteration)
logging.info("Training is %d percent complete\n" %
(percent_complete))
# don't compute the test loss on the first few times through the loop
if self._config["training"]["compute_test_loss"] and (
loss_current_iteration % compute_test_loss_rate
== 0) and loss_current_iteration > 5:
logging.info("Computing test loss")
# delete the loss, match_loss, non_match_loss variables so that
# pytorch can use that GPU memory
del loss, match_loss, masked_non_match_loss, background_non_match_loss, blind_non_match_loss
gc.collect()
dcn.eval()
test_loss, test_match_loss, test_non_match_loss = DCE.compute_loss_on_dataset(
dcn,
self._data_loader_test,
self._config['loss_function'],
num_iterations=self._config['training']
['test_loss_num_iterations'])
# delete these variables so we can free GPU memory
del test_loss, test_match_loss, test_non_match_loss
# make sure to set the network back to train mode
dcn.train()
if loss_current_iteration % self._config['training'][
'garbage_collect_rate'] == 0:
logging.debug("running garbage collection")
gc_start = time.time()
gc.collect()
gc_elapsed = time.time() - gc_start
logging.debug("garbage collection took %.2d seconds" %
(gc_elapsed))
if loss_current_iteration > max_num_iterations:
logging.info("Finished testing after %d iterations" %
(max_num_iterations))
self.save_network(dcn,
optimizer,
loss_current_iteration,
logging_dict=self._logging_dict)
return
def run(self, loss_current_iteration=0, use_pretrained=False):
"""
Runs the training
:return:
:rtype:
"""
start_iteration = copy.copy(loss_current_iteration)
DCE = DenseCorrespondenceEvaluation
self.setup()
self.save_configs()
if not use_pretrained:
# create new network and optimizer
self._dcn = self.build_network()
self._optimizer = self._construct_optimizer(self._dcn.parameters())
else:
logging.info("using pretrained model")
if (self._dcn is None):
raise ValueError("you must set self._dcn if use_pretrained=True")
if (self._optimizer is None):
raise ValueError("you must set self._optimizer if use_pretrained=True")
# make sure network is using cuda and is in train mode
dcn = self._dcn
dcn.cuda()
dcn.train()
optimizer = self._optimizer
batch_size = self._data_loader.batch_size
pixelwise_contrastive_loss = PixelwiseContrastiveLoss(image_shape=dcn.image_shape, config=self._config['loss_function'])
pixelwise_contrastive_loss.debug = True
# Repeat M for background and masked
pixelwise_contrastive_loss._config['M_background'] = pixelwise_contrastive_loss._config['M_descriptor']
pixelwise_contrastive_loss._config['M_masked'] = pixelwise_contrastive_loss._config['M_descriptor']
loss = match_loss = non_match_loss = 0
num_epochs = self._config['training']['num_epochs']
logging_rate = self._config['training']['logging_rate']
save_rate = self._config['training']['save_rate']
compute_test_loss_rate = self._config['training']['compute_test_loss_rate']
# logging
self._logging_dict = dict()
self._logging_dict['train'] = {"iteration": [], "loss": [], "match_loss": [],
"masked_non_match_loss": [],
"background_non_match_loss": [],
"blind_non_match_loss": [],
"learning_rate": [],
"different_object_non_match_loss": []}
self._logging_dict['test'] = {"iteration": [], "loss": [], "match_loss": [],
"non_match_loss": []}
# save network before starting
if not use_pretrained:
self.save_network(dcn, optimizer, 0)
t_start = time.time()
loss_vec = []
match_loss_vec = []
non_match_loss_vec = []
for epoch in range(num_epochs): # loop over the dataset multiple times
for i, data in enumerate(self._data_loader, 0):
loss_current_iteration += 1
start_iter = time.time()
match_type, img_a, img_b, matches_a, matches_b, non_matches_a, non_matches_b = data
img_a = Variable(img_a.cuda(), requires_grad=False)
img_b = Variable(img_b.cuda(), requires_grad=False)
# Note: repeat non_matches for both masked and background, and fake blind nonmatches using empty tensor, for compatibility in loss computation
matches_a = Variable(matches_a.cuda().squeeze(0), requires_grad=False)
matches_b = Variable(matches_b.cuda().squeeze(0), requires_grad=False)
non_matches_a = Variable(non_matches_a.cuda().squeeze(0), requires_grad=False)
non_matches_b = Variable(non_matches_b.cuda().squeeze(0), requires_grad=False)
blind_non_matches_a = Variable(SpartanDataset.empty_tensor().cuda().squeeze(0), requires_grad=False)
blind_non_matches_b = Variable(SpartanDataset.empty_tensor().cuda().squeeze(0), requires_grad=False)
optimizer.zero_grad()
self.adjust_learning_rate(optimizer, loss_current_iteration)
# run both images through the network
image_a_pred = dcn.forward(img_a)
image_a_pred = dcn.process_network_output(image_a_pred, batch_size)
image_b_pred = dcn.forward(img_b)
image_b_pred = dcn.process_network_output(image_b_pred, batch_size)
# get loss.
loss, match_loss, non_match_loss, masked_non_match_loss, background_non_match_loss, blind_non_match_loss \
= loss_composer.get_loss(pixelwise_contrastive_loss, match_type,
image_a_pred, image_b_pred,
matches_a, matches_b,
non_matches_a, non_matches_b,
non_matches_a, non_matches_b,
blind_non_matches_a, blind_non_matches_b)
loss.backward()
optimizer.step()
elapsed = time.time() - start_iter
# print "single iteration took %.3f seconds" %(elapsed)
if loss_current_iteration % save_rate == 0:
self.save_network(dcn, optimizer, loss_current_iteration, logging_dict=self._logging_dict)
sys.stdout.write('Epoch %d/%d, Image %d/%d, total_itr: %d, loss: %.4f, match_loss: %.4f, non_match_loss: %.4f, total_time: %s \r' % \
(epoch+1, num_epochs, i+1, len(self._dataset), loss_current_iteration, loss.data[0], match_loss.data[0], non_match_loss.data[0], str(timedelta(seconds=time.time()-t_start))[:-4])); sys.stdout.flush()
loss_vec.append(loss.data[0])
match_loss_vec.append(match_loss.data[0])
non_match_loss_vec.append(non_match_loss.data[0])
if self._config["training"]["compute_test_loss"] and (loss_current_iteration % compute_test_loss_rate == 0):
print
# logging.info("Computing test loss")
# delete the loss, match_loss, non_match_loss variables so that
# pytorch can use that GPU memory
del loss, match_loss, non_match_loss, masked_non_match_loss, background_non_match_loss, blind_non_match_loss
gc.collect()
print '\tTraining average:loss: %.4f, match_loss: %.4f, non_match_loss: %.4f' % \
(np.mean(loss_vec), np.mean(match_loss_vec), np.mean(non_match_loss_vec))
loss_vec = []
match_loss_vec = []
non_match_loss_vec = []
dcn.eval()
test_loss, test_match_loss, test_non_match_loss = DCE.compute_loss_on_salad_dataset(dcn,
self._data_loader_test, self._config['loss_function'], num_iterations=self._config['training']['test_loss_num_iterations'])
print '\tTesting results: loss: %.4f, match_loss: %.4f, non_match_loss: %.4f' % \
(test_loss, test_match_loss, test_non_match_loss)
# delete these variables so we can free GPU memory
del test_loss, test_match_loss, test_non_match_loss
# make sure to set the network back to train mode
dcn.train()
if loss_current_iteration % self._config['training']['garbage_collect_rate'] == 0:
logging.debug("running garbage collection")
gc_start = time.time()
gc.collect()
gc_elapsed = time.time() - gc_start
logging.debug("garbage collection took %.2d seconds" %(gc_elapsed))
logging.info("Finished training.")
self.save_network(dcn, optimizer, loss_current_iteration, logging_dict=self._logging_dict)
return
