def run(self,
train_ops,
train_sess,
test_ops=None,
test_sess=None,
test_iter=100):
iter = 0
for epoch in range(1):
while not train_sess.should_stop():
values = train_sess.run(train_ops)
if values is None:
break
loss, acc, aux_loss, _ = values
add_metrics("loss", loss)
add_metrics(
"time",
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
iter += 1
if (iter % test_iter) == 0:
self.run_test(test_ops, test_sess)
if (iter % test_iter) == 0: #add timeline
run_option = xdl.RunOption()
run_option.perf = True
run_statistic = xdl.RunStatistic()
_ = train_sess.run(train_ops, run_option, run_statistic)
xdl.Timeline(run_statistic.perf_result).save(
'../ckpt/timeline.json-' + str(iter))
print('======print the timeline =====')
iter += 1
train_sess._finish = False
def run(self, train_ops, train_sess, test_ops=None, test_sess=None, test_iter=100):
iter = 0
for epoch in range(1):
while not train_sess.should_stop():
values = train_sess.run(train_ops)
if values is None:
break
loss, acc, aux_loss, _ = values
add_metrics("loss", loss)
add_metrics("time", datetime.datetime.now(
).strftime('%Y-%m-%d %H:%M:%S'))
iter += 1
if (iter % test_iter) == 0:
self.run_test(test_ops, test_sess)
train_sess._finish = False
def run(self, train_ops, train_sess, test_ops=None, test_sess=None, test_iter=100):
iter = 0
for epoch in range(1):
while not train_sess.should_stop():
values = train_sess.run(train_ops)
if values is None:
break
loss, acc, aux_loss, _ = values
add_metrics("loss", loss)
add_metrics("time", datetime.datetime.now(
).strftime('%Y-%m-%d %H:%M:%S'))
iter += 1
if (iter % test_iter) == 0:
self.run_test(test_ops, test_sess)
train_sess._finish = False
def after_run(self, v):
if isinstance(v, list) and len(v) == 0:
return
add_metrics(self._name, str(v))
def after_run(self, v):
self._lstep = self._lstep + 1
self._gstep = v
if len(self._lsteps) < self._interval:
self._lsteps.append(self._lstep)
else:
self._lsteps.pop(0)
self._lsteps.append(self._lstep)
if len(self._gsteps) < self._interval:
self._gsteps.append(self._gstep)
else:
self._gsteps.pop(0)
self._gsteps.append(self._gstep)
if len(self._times) < self._interval:
dt = datetime.datetime.now()
self._times.append(dt.microsecond + dt.second * 1000000)
else:
self._times.pop(0)
dt = datetime.datetime.now()
self._times.append(dt.microsecond + dt.second * 1000000)
add_metrics("lstep", self._lstep)
add_metrics("gstep", self._gstep)
if len(self._times) > 1:
interval = self._times[-1] - self._times[0]
if interval == 0:
interval = 1
gsteps = self._gsteps[-1] - self._gsteps[0]
lsteps = self._lsteps[-1] - self._lsteps[0]
gqps = int(gsteps * 1000000.0 / interval)
lqps = int(lsteps * 1000000.0 / interval)
add_metrics("gqps", str(gqps))
add_metrics("lqps", str(lqps))
else:
add_metrics("gqps", "0")
add_metrics("lqps", "0")
def after_run(self, v):
self._lstep = self._lstep + 1
self._gstep = v
if len(self._lsteps) < self._interval:
self._lsteps.append(self._lstep)
else:
self._lsteps.pop(0)
self._lsteps.append(self._lstep)
if len(self._gsteps) < self._interval:
self._gsteps.append(self._gstep)
else:
self._gsteps.pop(0)
self._gsteps.append(self._gstep)
if len(self._times) < self._interval:
self._times.append(int(round(time.time() * 1000)))
else:
self._times.pop(0)
self._times.append(int(round(time.time() * 1000)))
add_metrics("lstep", self._lstep)
add_metrics("gstep", self._gstep)
if len(self._times) > 1:
interval = self._times[-1] - self._times[0]
if interval == 0:
interval = 1
gsteps = self._gsteps[-1] - self._gsteps[0]
lsteps = self._lsteps[-1] - self._lsteps[0]
gqps = gsteps * 1000.0 / interval
lqps = lsteps * 1000.0 / interval
add_metrics("gqps", '%.1f' % gqps)
add_metrics("lqps", '%.1f' % lqps)
else:
add_metrics("gqps", "0")
add_metrics("lqps", "0")