def run(dataloader, exe, program, fetchs, epoch=0, mode='train'): """ Feed data to the model and fetch the measures and loss Args: dataloader(fluid dataloader): exe(): program(): fetchs(dict): dict of measures and the loss epoch(int): epoch of training or validation model(str): log only Returns: """ fetch_list = [f[0] for f in fetchs.values()] metric_list = [f[1] for f in fetchs.values()] for m in metric_list: m.reset() batch_time = AverageMeter('cost', ':6.3f') tic = time.time() trainer_id = int(os.getenv("PADDLE_TRAINER_ID", 0)) for idx, batch in enumerate(dataloader()): metrics = exe.run(program=program, feed=batch, fetch_list=fetch_list) batch_time.update(time.time() - tic) tic = time.time() for i, m in enumerate(metrics): metric_list[i].update(m[0], len(batch[0])) fetchs_str = ''.join([str(m) for m in metric_list] + [str(batch_time)]) if trainer_id == 0: logger.info("[epoch:%3d][%s][step:%4d]%s" % (epoch, mode, idx, fetchs_str)) if trainer_id == 0: logger.info("END [epoch:%3d][%s]%s" % (epoch, mode, fetchs_str))
def run(dataloader, exe, program, fetchs, epoch=0, mode='train'): """ Feed data to the model and fetch the measures and loss Args: dataloader(fluid dataloader): exe(): program(): fetchs(dict): dict of measures and the loss epoch(int): epoch of training or validation model(str): log only Returns: """ fetch_list = [f[0] for f in fetchs.values()] metric_list = [f[1] for f in fetchs.values()] for m in metric_list: m.reset() batch_time = AverageMeter('cost', '.3f') tic = time.time() for idx, batch in enumerate(dataloader()): metrics = exe.run(program=program, feed=batch, fetch_list=fetch_list) batch_time.update(time.time() - tic) tic = time.time() for i, m in enumerate(metrics): metric_list[i].update(m[0], len(batch[0])) fetchs_str = ''.join([m.value for m in metric_list] + [batch_time.value]) logger.info("[epoch:{:3d}][{:s}][step:{:4d}]{:s}".format( epoch, mode, idx, fetchs_str)) end_str = ''.join([m.mean for m in metric_list] + [batch_time.total]) logger.info("END [epoch:{:3d}][{:s}]{:s}".format(epoch, mode, end_str))
def run(dataloader, exe, program, fetchs, epoch=0, mode='train', vdl_writer=None): """ Feed data to the model and fetch the measures and loss Args: dataloader(fluid dataloader): exe(): program(): fetchs(dict): dict of measures and the loss epoch(int): epoch of training or validation model(str): log only Returns: """ fetch_list = [f[0] for f in fetchs.values()] metric_list = [f[1] for f in fetchs.values()] for m in metric_list: m.reset() batch_time = AverageMeter('elapse', '.3f') tic = time.time() for idx, batch in enumerate(dataloader()): metrics = exe.run(program=program, feed=batch, fetch_list=fetch_list) batch_time.update(time.time() - tic) tic = time.time() for i, m in enumerate(metrics): metric_list[i].update(m[0], len(batch[0])) fetchs_str = ''.join([str(m.value) + ' ' for m in metric_list] + [batch_time.value]) + 's' if vdl_writer: global total_step logger.scaler('loss', metrics[0][0], total_step, vdl_writer) total_step += 1 if mode == 'eval': logger.info("{:s} step:{:<4d} {:s}s".format(mode, idx, fetchs_str)) else: epoch_str = "epoch:{:<3d}".format(epoch) step_str = "{:s} step:{:<4d}".format(mode, idx) logger.info("{:s} {:s} {:s}".format( logger.coloring(epoch_str, "HEADER") if idx == 0 else epoch_str, logger.coloring(step_str, "PURPLE"), logger.coloring(fetchs_str, 'OKGREEN'))) end_str = ''.join([str(m.mean) + ' ' for m in metric_list] + [batch_time.total]) + 's' if mode == 'eval': logger.info("END {:s} {:s}s".format(mode, end_str)) else: end_epoch_str = "END epoch:{:<3d}".format(epoch) logger.info("{:s} {:s} {:s}".format( logger.coloring(end_epoch_str, "RED"), logger.coloring(mode, "PURPLE"), logger.coloring(end_str, "OKGREEN"))) # return top1_acc in order to save the best model if mode == 'valid': return fetchs["top1"][1].avg
def run(dataloader, exe, program, fetchs, epoch=0, mode='train'): """ Feed data to the model and fetch the measures and loss Args: dataloader(fluid dataloader): exe(): program(): fetchs(dict): dict of measures and the loss epoch(int): epoch of training or validation model(str): log only Returns: """ fetch_list = [f[0] for f in fetchs.values()] metric_list = [f[1] for f in fetchs.values()] for m in metric_list: m.reset() batch_time = AverageMeter('elapse', '.3f') tic = time.time() for idx, batch in enumerate(dataloader()): metrics = exe.run(program=program, feed=batch, fetch_list=fetch_list) batch_time.update(time.time() - tic) tic = time.time() for i, m in enumerate(metrics): metric_list[i].update(m[0], len(batch[0])) fetchs_str = ''.join([str(m.value) + ' ' for m in metric_list] + [batch_time.value]) if mode == 'eval': logger.info("{:s} step:{:<4d} {:s}s".format(mode, idx, fetchs_str)) else: logger.info("epoch:{:<3d} {:s} step:{:<4d} {:s}s".format( epoch, mode, idx, fetchs_str)) end_str = ''.join([str(m.mean) + ' ' for m in metric_list] + [batch_time.total]) if mode == 'eval': logger.info("END {:s} {:s}s".format(mode, end_str)) else: logger.info("END epoch:{:<3d} {:s} {:s}s".format(epoch, mode, end_str)) # return top1_acc in order to save the best model if mode == 'valid': return fetchs["top1"][1].avg
def run(dataloader, exe, program, fetchs, epoch=0, mode='train', config=None, vdl_writer=None): """ Feed data to the model and fetch the measures and loss Args: dataloader(fluid dataloader): exe(): program(): fetchs(dict): dict of measures and the loss epoch(int): epoch of training or validation model(str): log only Returns: """ fetch_list = [f[0] for f in fetchs.values()] metric_list = [f[1] for f in fetchs.values()] for m in metric_list: m.reset() batch_time = AverageMeter('elapse', '.5f', need_avg=True) tic = time.time() dataloader = dataloader if config.get('use_dali') else dataloader()() for idx, batch in enumerate(dataloader): if idx == 10: for m in metric_list: m.reset() batch_time.reset() batch_size = batch[0]["feed_image"].shape()[0] metrics = exe.run(program=program, feed=batch, fetch_list=fetch_list) batch_time.update(time.time() - tic) for i, m in enumerate(metrics): metric_list[i].update(np.mean(m), batch_size) fetchs_str = ''.join([str(m.value) + ' ' for m in metric_list] + [batch_time.mean]) + 's' ips_info = " ips: {:.5f} images/sec.".format(batch_size / batch_time.avg) fetchs_str += ips_info if vdl_writer: global total_step logger.scaler('loss', metrics[0][0], total_step, vdl_writer) total_step += 1 if mode == 'eval': if idx % config.get('print_interval', 10) == 0: logger.info("{:s} step:{:<4d} {:s}".format( mode, idx, fetchs_str)) else: epoch_str = "epoch:{:<3d}".format(epoch) step_str = "{:s} step:{:<4d}".format(mode, idx) if idx % config.get('print_interval', 10) == 0: logger.info("{:s} {:s} {:s}".format( epoch_str if idx == 0 else epoch_str, step_str, fetchs_str)) tic = time.time() if config.get('use_dali'): dataloader.reset() end_str = ''.join([str(m.mean) + ' ' for m in metric_list] + [batch_time.total]) + 's' ips_info = "ips: {:.5f} images/sec.".format(batch_size * batch_time.count / batch_time.sum) if mode == 'eval': logger.info("END {:s} {:s}s {:s}".format(mode, end_str, ips_info)) else: end_epoch_str = "END epoch:{:<3d}".format(epoch) logger.info("{:s} {:s} {:s} {:s}".format(end_epoch_str, mode, end_str, ips_info)) # return top1_acc in order to save the best model if mode == 'valid': return fetchs["top1"][1].avg
def run(dataloader, exe, program, feeds, fetchs, epoch=0, mode='train', config=None, vdl_writer=None, lr_scheduler=None): """ Feed data to the model and fetch the measures and loss Args: dataloader(paddle io dataloader): exe(): program(): fetchs(dict): dict of measures and the loss epoch(int): epoch of training or validation model(str): log only Returns: """ fetch_list = [f[0] for f in fetchs.values()] metric_list = [f[1] for f in fetchs.values()] if mode == "train": metric_list.append(AverageMeter('lr', 'f', need_avg=False)) for m in metric_list: m.reset() batch_time = AverageMeter('elapse', '.3f') use_dali = config.get('use_dali', False) dataloader = dataloader if use_dali else dataloader() tic = time.time() for idx, batch in enumerate(dataloader): # ignore the warmup iters if idx == 5: batch_time.reset() if use_dali: batch_size = batch[0]["feed_image"].shape()[0] feed_dict = batch[0] else: batch_size = batch[0].shape()[0] feed_dict = { key.name: batch[idx] for idx, key in enumerate(feeds.values()) } metrics = exe.run(program=program, feed=feed_dict, fetch_list=fetch_list) batch_time.update(time.time() - tic) for i, m in enumerate(metrics): metric_list[i].update(np.mean(m), batch_size) if mode == "train": metric_list[-1].update(lr_scheduler.get_lr()) fetchs_str = ''.join([str(m.value) + ' ' for m in metric_list] + [batch_time.mean]) + 's' ips_info = " ips: {:.5f} images/sec.".format(batch_size / batch_time.avg) fetchs_str += ips_info if lr_scheduler is not None: if lr_scheduler.update_specified: curr_global_counter = lr_scheduler.step_each_epoch * epoch + idx update = max( 0, curr_global_counter - lr_scheduler.update_start_step ) % lr_scheduler.update_step_interval == 0 if update: lr_scheduler.step() else: lr_scheduler.step() if vdl_writer: global total_step logger.scaler('loss', metrics[0][0], total_step, vdl_writer) total_step += 1 if mode == 'valid': if idx % config.get('print_interval', 10) == 0: logger.info("{:s} step:{:<4d} {:s}".format( mode, idx, fetchs_str)) else: epoch_str = "epoch:{:<3d}".format(epoch) step_str = "{:s} step:{:<4d}".format(mode, idx) if idx % config.get('print_interval', 10) == 0: logger.info("{:s} {:s} {:s}".format( logger.coloring(epoch_str, "HEADER") if idx == 0 else epoch_str, logger.coloring(step_str, "PURPLE"), logger.coloring(fetchs_str, 'OKGREEN'))) tic = time.time() end_str = ''.join([str(m.mean) + ' ' for m in metric_list] + [batch_time.total]) + 's' ips_info = "ips: {:.5f} images/sec.".format(batch_size * batch_time.count / batch_time.sum) if mode == 'valid': logger.info("END {:s} {:s}s {:s}".format(mode, end_str, ips_info)) else: end_epoch_str = "END epoch:{:<3d}".format(epoch) logger.info("{:s} {:s} {:s} {:s}".format(end_epoch_str, mode, end_str, ips_info)) if use_dali: dataloader.reset() # return top1_acc in order to save the best model if mode == 'valid': return fetchs["top1"][1].avg