def run(self, train_ops, train_sess, test_ops=None, test_sess=None, test_iter=100): iter = 0 for epoch in range(1): while not train_sess.should_stop(): values = train_sess.run(train_ops) if values is None: break loss, acc, aux_loss, _ = values add_metrics("loss", loss) add_metrics( "time", datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) iter += 1 if (iter % test_iter) == 0: self.run_test(test_ops, test_sess) if (iter % test_iter) == 0: #add timeline run_option = xdl.RunOption() run_option.perf = True run_statistic = xdl.RunStatistic() _ = train_sess.run(train_ops, run_option, run_statistic) xdl.Timeline(run_statistic.perf_result).save( '../ckpt/timeline.json-' + str(iter)) print('======print the timeline =====') iter += 1 train_sess._finish = False
def run(self, train_ops, train_sess, test_ops=None, test_sess=None, test_iter=100): iter = 0 for epoch in range(1): while not train_sess.should_stop(): values = train_sess.run(train_ops) if values is None: break loss, acc, aux_loss, _ = values add_metrics("loss", loss) add_metrics("time", datetime.datetime.now( ).strftime('%Y-%m-%d %H:%M:%S')) iter += 1 if (iter % test_iter) == 0: self.run_test(test_ops, test_sess) train_sess._finish = False
def after_run(self, v): if isinstance(v, list) and len(v) == 0: return add_metrics(self._name, str(v))
def after_run(self, v): self._lstep = self._lstep + 1 self._gstep = v if len(self._lsteps) < self._interval: self._lsteps.append(self._lstep) else: self._lsteps.pop(0) self._lsteps.append(self._lstep) if len(self._gsteps) < self._interval: self._gsteps.append(self._gstep) else: self._gsteps.pop(0) self._gsteps.append(self._gstep) if len(self._times) < self._interval: dt = datetime.datetime.now() self._times.append(dt.microsecond + dt.second * 1000000) else: self._times.pop(0) dt = datetime.datetime.now() self._times.append(dt.microsecond + dt.second * 1000000) add_metrics("lstep", self._lstep) add_metrics("gstep", self._gstep) if len(self._times) > 1: interval = self._times[-1] - self._times[0] if interval == 0: interval = 1 gsteps = self._gsteps[-1] - self._gsteps[0] lsteps = self._lsteps[-1] - self._lsteps[0] gqps = int(gsteps * 1000000.0 / interval) lqps = int(lsteps * 1000000.0 / interval) add_metrics("gqps", str(gqps)) add_metrics("lqps", str(lqps)) else: add_metrics("gqps", "0") add_metrics("lqps", "0")
def after_run(self, v): self._lstep = self._lstep + 1 self._gstep = v if len(self._lsteps) < self._interval: self._lsteps.append(self._lstep) else: self._lsteps.pop(0) self._lsteps.append(self._lstep) if len(self._gsteps) < self._interval: self._gsteps.append(self._gstep) else: self._gsteps.pop(0) self._gsteps.append(self._gstep) if len(self._times) < self._interval: self._times.append(int(round(time.time() * 1000))) else: self._times.pop(0) self._times.append(int(round(time.time() * 1000))) add_metrics("lstep", self._lstep) add_metrics("gstep", self._gstep) if len(self._times) > 1: interval = self._times[-1] - self._times[0] if interval == 0: interval = 1 gsteps = self._gsteps[-1] - self._gsteps[0] lsteps = self._lsteps[-1] - self._lsteps[0] gqps = gsteps * 1000.0 / interval lqps = lsteps * 1000.0 / interval add_metrics("gqps", '%.1f' % gqps) add_metrics("lqps", '%.1f' % lqps) else: add_metrics("gqps", "0") add_metrics("lqps", "0")