def run(self, plot=True, resume=False):
os.chdir(self.workdir)
if resume:
last_sn = self.get_last_snapshot()
if last_sn is None:
resume = False
else:
logging.info(
'Resuming training from iteration {}'.format(last_sn))
logging.info('Training on ' + self.train.get_name() +
' while validating on ' +
', '.join([str(v) for v in self.val]) + ' ...')
if os.path.exists(self.LOG_FILE) and not resume:
os.remove(self.LOG_FILE)
cmd = [
'/opt/caffe/build/tools/caffe', 'train', '-gpu', '0', '-solver',
'solver.prototxt'
]
if resume:
cmd += [
'-snapshot', '{}/snapshot_iter_{}.solverstate'.format(
self.SNAPSHOTS_DIR, last_sn)
]
elif self.model.infmt.pretrain is not None:
cmd += ['-weights', os.path.basename(self.model.infmt.pretrain)]
caffe = subprocess.Popen(cmd, stderr=subprocess.PIPE)
dst = subprocess.PIPE if plot else open(os.devnull, 'wb')
tee = subprocess.Popen(['tee', '-a', self.LOG_FILE],
stdin=caffe.stderr,
stdout=dst)
def handler(sig, frame):
# propagate SIGINT down, and wait
os.kill(caffe.pid, signal.SIGHUP)
os.kill(caffe.pid, sig)
caffe.wait()
signal.signal(signal.SIGINT, handler)
if plot:
line_iter = iter(tee.stdout.readline, '')
live_plot = pyffe.LivePlot(title=self.long_name())
pyffe.LogParser(line_iter).parse(live_plot)
tee.wait()
# print something in case of error
if caffe.returncode != 0:
os.system('tail -n 20 {}'.format(self.LOG_FILE))
for it in its:
if it not in self.it_vlines:
self.it_vlines.append(it)
self.axup.axvline(it, color='k')
h1, l1 = self.axup.get_legend_handles_labels()
h2, l2 = self.axup2.get_legend_handles_labels()
self.axup2.legend(h1 + h2, l1 + l2, loc='center right', fancybox=True, shadow=True, fontsize='small')
self.axdown.legend(loc='upper right', fancybox=True, shadow=True, fontsize='small')
if __name__ == "__main__":
import os
import sys
# print sys.path
import pyffe
# sys.stdin.read()
if len(sys.argv) < 2:
print "Usage: {} log.caffelog".format(os.path.basename(sys.argv[0]))
sys.exit(1)
with open(sys.argv[1], "r") as f:
lp = LivePlot()
line_iter = iter(f.readline, '')
data = pyffe.LogParser(line_iter).parse()
lp(data)
lp.waitclose()
def get_log_data(self):
p = os.path.join(self.workdir, self.LOG_FILE)
line_iter = iter(open(p).readline, '')
return pyffe.LogParser(line_iter).parse()
def get_log_data(self):
line_iter = iter(open(self.workdir + '/' + self.LOG_FILE).readline, '')
return pyffe.LogParser(line_iter).parse()