def train(self):
train_timer = Timer()
load_timer = Timer()
for step in xrange(1, self.max_iter + 1):
load_timer.tic()
images, labels = self.data.get()
load_timer.toc()
feed_dict = {self.net.x: images, self.net.labels: labels}
train_timer.tic()
# summary_str, loss = self.sess.run([self.summary_op, self.net.loss], feed_dict=feed_dict)
_, summary_str, loss = self.sess.run(
[self.train_op, self.summary_op, self.net.loss],
feed_dict=feed_dict)
# _, loss = self.sess.run([self.train_op, self.net.loss], feed_dict=feed_dict)
train_timer.toc()
log_str = (
'{} Epoch: {}, Step: {},'
' Loss: {:5.3f} Speed: {:.3f}s/iter,'
' Load: {:.3f}s/iter, Remain: {}'
).format(
datetime.datetime.now().strftime('%m/%d %H:%M:%S'),
self.data.epoch,
int(step),
# round(self.initial_learning_rate.eval(session=self.sess), 6),
loss,
train_timer.average_time,
load_timer.average_time,
train_timer.remain(step, self.max_iter))
print log_str
self.log_file.write(log_str + "\n")
self.writer.add_summary(summary_str, step)
if step % self.save_iter == 0:
log_str = '{} Saving checkpoint file to: {}'.format(
datetime.datetime.now().strftime('%m/%d %H:%M:%S'),
self.output_dir)
print log_str
self.log_file.write(log_str + "\n")
path_where_variables_saved = self.saver.save(
self.sess,
self.ckpt_file,
global_step=self.global_step,
write_meta_graph=False)
log_str = '{} Session variables successfully saved to: {}'.format(
datetime.datetime.now().strftime('%m/%d %H:%M:%S'),
path_where_variables_saved)
print log_str
self.log_file.write(log_str + "\n")
def camera_detector(self, cap, wait=10):
detect_timer = Timer()
ret, _ = cap.read()
while ret:
ret, frame = cap.read()
detect_timer.tic()
result = self.detect(frame)
detect_timer.toc()
print 'Average detecting time: {:.3f}s'.format(
detect_timer.average_time)
self.draw_result(frame, result)
cv2.imshow('Camera', frame)
cv2.waitKey(wait)
def image_detector(self, imname, wait=0, resize=False):
if not os.path.isfile(imname):
raise ValueError("No such file is present: {}".format(imname))
detect_timer = Timer()
image = cv2.imread(imname)
if resize:
image = cv2.resize(image, (640, 360))
detect_timer.tic()
result = self.detect(image)
detect_timer.toc()
print result
print 'Average detecting time: {:.3f}s'.format(
detect_timer.average_time)
self.draw_result(image, result)
cv2.imshow('Image', image)
cv2.waitKey(wait)
class DataSummary(object):
"""
DataSummary class. Used to store all kind of summaries during ytaining
"""
def __init__(self, log_dir, start_iteration=0):
"""
:param log_dir: folder to store the summaries
:param start_iteration: iteration where the training is starting
"""
self.loss_per_iteration = []
self.iteration = start_iteration
self.error_counter = 0
self.timer = Timer()
self.timer.tic()
self.log_dir = log_dir
self.writer = SummaryWriter(
log_dir=self.log_dir
)
def add_summaries(self, loss_value, n_steps=1):
"""
add summaries to object
:param loss_value: loss value of one forward pass or mean loss value
:param n_steps: number of past iterations
:return:
"""
self.loss_per_iteration.append(loss_value)
self.iteration += n_steps
def give_summary(self, reset=True):
"""
Store summaries and print them to std
:param reset: reset summaries
:return:
"""
mean_los = np.mean(self.loss_per_iteration)
logging.info('Iteration: {}, Mean loss: {:.4f}, Mean run time: {:.4f}'.format(self.iteration, mean_los, self.timer.toc(average=False)))
self.writer.add_scalar('training/mean_loss', mean_los, self.iteration)
if reset:
self.resert_summaries()
self.timer.tic()
def resert_summaries(self):
"""
Reset summaries in the model
:return:
"""
self.loss_per_iteration = []
def error(self, e):
"""
write error
:return:
"""
self.error_counter += 1
error_msg = 'Error in iteration: {}, Error: {}'.format(self.iteration, e)
logging.info(error_msg)
self.writer.add_text('Error', error_msg, self.iteration)
def store_histogram(self, name, param):
"""
store model weight histograms
:param name: layer name
:param param: model param
:return:
"""
self.writer.add_histogram(name, param.clone().cpu().data.numpy(), self.iteration)
