def __init__(self, vocab_size, num_conv_layers, num_rnn_layers, rnn_layer_size, use_gru=False, share_rnn_weights=True, place=fluid.CPUPlace(), init_from_pretrained_model=None, output_model_dir=None, is_infer=False, error_rate_type='cer', vocab_list=None): self._vocab_size = vocab_size self._num_conv_layers = num_conv_layers self._num_rnn_layers = num_rnn_layers self._rnn_layer_size = rnn_layer_size self._use_gru = use_gru self._share_rnn_weights = share_rnn_weights self._place = place self._init_from_pretrained_model = init_from_pretrained_model self._output_model_dir = output_model_dir self._ext_scorer = None self.logger = logging.getLogger("") self.logger.setLevel(level=logging.INFO) self.writer = LogWriter(logdir='log') self.error_rate_type = error_rate_type self.vocab_list = vocab_list # 预测相关的参数 self.infer_program = None self.infer_feeder = None self.infer_log_probs = None self.infer_exe = None if is_infer: self.init_infer_program()
def __init__(self, hparams): # Use data distributed self.infer_network = hparams.infer_network self.dataset = hparams.dataset self.num_epochs = hparams.num_epochs self.batch_size = hparams.batch_size self.enable_ce = hparams.enable_ce self.logger = hparams.logger self.cpu_num = hparams.cpu_num self.cuda_devices = hparams.cuda_devices self.multi_card = hparams.multi_card self.num_class = 10 # default is cifar-10 if self.logger: from visualdl import LogWriter self.log_writer = LogWriter(self.logger, sync_cycle=20) # Create two ScalarWriter instances, whose mode is set to be "train" with self.log_writer.mode("train") as logger: self.train_cost = logger.scalar("cost") self.train_acc = logger.scalar("acc") # Create a ScalarWriter instance, whose mode is set to be "test" with self.log_writer.mode("test") as logger: self.test_loss = logger.scalar("loss") self.test_acc = logger.scalar("acc") if self.dataset is "cifar-100": self.num_class = 100
def __init__(self, model: paddle.nn.Layer, optimizer: paddle.optimizer.Optimizer, use_vdl: bool = True, checkpoint_dir: str = None, compare_metrics: Callable = None): self.nranks = paddle.distributed.get_world_size() self.local_rank = paddle.distributed.get_rank() self.model = model self.optimizer = optimizer self.checkpoint_dir = checkpoint_dir if checkpoint_dir else 'ckpt_{}'.format(time.time()) if not isinstance(self.model, paddle.nn.Layer): raise TypeError('The model {} is not a `paddle.nn.Layer` object.'.format(self.model.__name__)) if self.local_rank == 0 and not os.path.exists(self.checkpoint_dir): os.makedirs(self.checkpoint_dir) self.use_vdl = use_vdl if self.local_rank == 0 and self.use_vdl: vdl_dir = os.path.join(self.checkpoint_dir, 'visualization') self.log_writer = LogWriter(vdl_dir) self.current_epoch = 0 self.best_metrics = defaultdict(int) if self.nranks > 1: paddle.distributed.init_parallel_env() strategy = paddle.distributed.prepare_context() self.model = paddle.DataParallel(self.model, strategy) self.compare_metrics = self._compare_metrics if not compare_metrics else compare_metrics self._load_checkpoint()
def __init__(self, logdir, experimental_name, *, save_model_with_input=None): """ :param logdir: :param experimental_name: :param save_model_with_input: 默认不存储模型结构,当开启该参数时, """ from pyxllib.prog.pupil import check_install_package check_install_package('visualdl') from visualdl import LogWriter super().__init__() # 这样奇怪地加后缀,是为了字典序后,每个实验的train显示在eval之前 d = XlPath(logdir) / (experimental_name + '_train') # if d.exists(): shutil.rmtree(d) self.write = LogWriter(logdir=str(d)) d = XlPath(logdir) / (experimental_name + '_val') # if d.exists(): shutil.rmtree(d) self.eval_writer = LogWriter(logdir=str(d)) self.eval_times = 0 self.save_model_with_input = save_model_with_input
def run_begin(self, trainer): rank = dist.get_rank() if rank != 0: return logdir = os.path.join(trainer.output_dir, 'visual_dl') if not os.path.exists(logdir): os.makedirs(logdir) self.writer = LogWriter(logdir=logdir)
def train(config): use_gpu = config.use_gpu # Get the rank of the current training process. rank = dg.parallel.Env().local_rank nranks = dg.parallel.Env().nranks parallel = nranks > 1 if rank == 0: # Print the whole config setting. pprint(vars(config)) # Make checkpoint directory. run_dir = os.path.join("runs", config.model, config.name) checkpoint_dir = os.path.join(run_dir, "checkpoint") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) # Create tensorboard logger. vdl = LogWriter(os.path.join(run_dir, "logs")) \ if rank == 0 else None # Configurate device place = fluid.CUDAPlace(rank) if use_gpu else fluid.CPUPlace() with dg.guard(place): # Fix random seed. seed = config.seed random.seed(seed) np.random.seed(seed) fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed print("Random Seed: ", seed) # Build model. model = WaveFlow(config, checkpoint_dir, parallel, rank, nranks, vdl) iteration = model.build() while iteration < config.max_iterations: # Run one single training step. model.train_step(iteration) iteration += 1 if iteration % config.test_every == 0: # Run validation step. model.valid_step(iteration) if rank == 0 and iteration % config.save_every == 0: # Save parameters. model.save(iteration) # Close TensorBoard. if rank == 0: vdl.close()
def __init__(self): """ Initialize the Linear Model """ self.learning_rate = 0.001 self.epoches = 10000 self.loss_function = torch.nn.MSELoss() self.create_model() logdir = os.path.join('./runs', datetime.now().strftime('%b%d_%H-%M-%S')+'_line_reg') print("save to ", logdir) self.writer = LogWriter(logdir=logdir) print("initialize done")
def call(*args, **kwargs): global _writer if _writer is None: logdir = logger.get_dir() if logdir is None: logdir = logger.auto_set_dir(action='d') logger.warning( "[VisualDL] logdir is None, will save VisualDL files to {}\nView the data using: visualdl --logdir=./{} --host={}" .format(logdir, logdir, get_ip_address())) _writer = LogWriter(logdir=logger.get_dir()) func = getattr(_writer, func_name) func(*args, **kwargs) _writer.flush()
def test_modes(self): store = LogWriter(self.dir, sync_cycle=1) scalars = [] for i in range(10): with store.mode("mode-%d" % i) as writer: scalar = writer.scalar("add/scalar0") scalars.append(scalar) for scalar in scalars[:-1]: for i in range(10): scalar.add_record(i, float(i))
class VisualDL(tf.keras.callbacks.Callback): def __init__(self, logdir, save_per_epoch): super(VisualDL, self).__init__() self.logdir = logdir self.save_per_epoch = save_per_epoch def on_train_begin(self, logs=None): self.writer_train = LogWriter(self.logdir + "/train") self.writer_val = LogWriter(self.logdir + "/val") def on_batch_end(self, batch, logs=None): pass def on_epoch_end(self, epoch, logs=None): self.writer_train.add_scalar("loss/output_1_loss", logs["output_1_loss"], step=epoch + 1) self.writer_train.add_scalar("metric/F1", logs["output_2_F1"], step=epoch + 1) self.writer_train.add_scalar("metric/Precision", logs["output_2_Precision"], step=epoch + 1) self.writer_train.add_scalar("metric/Recall", logs["output_2_Recall"], step=epoch + 1)
def __init__(self, model): super(VisualDLWriter, self).__init__(model) assert six.PY3, "VisualDL requires Python >= 3.5" try: from visualdl import LogWriter except Exception as e: logger.error('visualdl not found, plaese install visualdl. ' 'for example: `pip install visualdl`.') raise e self.vdl_writer = LogWriter(model.cfg.vdl_log_dir) self.vdl_loss_step = 0 self.vdl_mAP_step = 0 self.vdl_image_step = 0 self.vdl_image_frame = 0
def validate(self): # testingzX print('Validate: ') eval_loss = 0 eval_acc = 0 idx = 0 self.net.eval() with LogWriter( logdir=os.path.join('./log', self.str_config, 'val')) as witer: for batch_id, data in enumerate(self.test_loader()): x = data[0] # y = paddle.to_tensor(np.where(data[1].numpy() > 5, 1, 0)) y = data[1] predicts = self.net(x) eval_loss = self.criterion(predicts, y) eval_acc = paddle.metric.accuracy(predicts, y) # 打印信息 if (batch_id + 1) % 100 == 0: print("batch_id: {}, loss is: {}, acc is: {}".format( batch_id + 1, eval_loss.numpy(), eval_acc.numpy())) witer.add_scalar(tag='acc', step=idx, value=eval_acc.numpy()) witer.add_scalar(tag='loss', step=idx, value=eval_loss.numpy()) idx += 1
def __init__( self, evaluator, cudas=["0"], popsize=5, output_dir=None, ): self._num_thread = len(cudas) self._popsize = popsize self.cudas = cudas self.is_cuda_free = {"free": [], "busy": []} self.is_cuda_free["free"] = cudas self._round = 0 self.evaluator = evaluator self.init_input = evaluator.get_init_params() self.num_hparam = len(self.init_input) self.best_hparams_all_pop = [] self.best_reward_all_pop = INF self.current_hparams = [[0] * self.num_hparam] * self._popsize self.hparams_name_list = [ param["name"] for param in evaluator.params['param_list'] ] if output_dir is None: now = int(time.time()) time_str = time.strftime("%Y%m%d%H%M%S", time.localtime(now)) self._output_dir = "output_" + time_str else: self._output_dir = output_dir # record the information for the whole auto finetune self.writer = LogWriter(logdir=self._output_dir + '/visualization') # record the information for per population in all round self.writer_pop_trails = [] for i in range(self.popsize): writer_pop_trail = LogWriter( logdir=self._output_dir + '/visualization/pop_{}'.format(i)) self.writer_pop_trails.append(writer_pop_trail) # for parallel on mpi self.mpi = MPIHelper() if self.mpi.multi_machine: print("Autofinetune multimachine mode: running on {}".format( self.mpi.gather(self.mpi.name)))
def __init__(self, config: Config, processor: DataProcessor, classifier) -> None: self.config = config self.set_device() self.tokenizer: PretrainedTokenizer = ppnlp.transformers.BertTokenizer.from_pretrained( config.pretrained_model) # 2. build data related objects self.train_dataset = processor.get_train_dataset() self.dev_dataset = processor.get_dev_dataset() self.test_dataset = processor.get_test_dataset() self.train_dataloader = create_dataloader( self.train_dataset, batch_size=config.batch_size, collate_fn=lambda examples: self.collate_fn( examples, self.train_dataset.label2idx), ) self.dev_dataloader = create_dataloader( self.dev_dataset, batch_size=config.batch_size, collate_fn=lambda examples: self.collate_fn( examples, self.dev_dataset.label2idx), ) self.test_dataloader = create_dataloader( self.test_dataset, batch_size=config.batch_size, collate_fn=lambda examples: self.collate_fn( examples, self.test_dataset.label2idx), ) # 3. init model related self.model = classifier self.lr_scheduler: LRScheduler = LinearDecayWithWarmup( config.learning_rate, total_steps=len(self.train_dataloader) * config.epochs, warmup=config.warmup_proportion) if config.init_from_ckpt and os.path.isfile(config.init_from_ckpt): state_dict = paddle.load(config.init_from_ckpt) self.model.set_dict(state_dict) # Generate parameter names needed to perform weight decay. # All bias and LayerNorm parameters are excluded. decay_params = [ p.name for n, p in self.model.named_parameters() if not any(nd in n for nd in ["bias", "norm"]) ] self.optimizer: Optimizer = paddle.optimizer.AdamW( learning_rate=self.lr_scheduler, parameters=self.model.parameters(), weight_decay=config.weight_decay, apply_decay_param_fun=lambda x: x in decay_params) self.criterion = paddle.nn.loss.CrossEntropyLoss() self.metric: Metric = paddle.metric.Accuracy() self.context_data = ContextContainer() self._init_output_dir() self.writer: LogWriter = LogWriter(logdir=config.output_dir)
class VisualDLWriter(Callback): """ Use VisualDL to log data or image """ def __init__(self, model): super(VisualDLWriter, self).__init__(model) assert six.PY3, "VisualDL requires Python >= 3.5" try: from visualdl import LogWriter except Exception as e: logger.error('visualdl not found, plaese install visualdl. ' 'for example: `pip install visualdl`.') raise e self.vdl_writer = LogWriter( model.cfg.get('vdl_log_dir', 'vdl_log_dir/scalar')) self.vdl_loss_step = 0 self.vdl_mAP_step = 0 self.vdl_image_step = 0 self.vdl_image_frame = 0 def on_step_end(self, status): mode = status['mode'] if dist.get_world_size() < 2 or dist.get_rank() == 0: if mode == 'train': training_staus = status['training_staus'] for loss_name, loss_value in training_staus.get().items(): self.vdl_writer.add_scalar(loss_name, loss_value, self.vdl_loss_step) self.vdl_loss_step += 1 elif mode == 'test': ori_image = status['original_image'] result_image = status['result_image'] self.vdl_writer.add_image( "original/frame_{}".format(self.vdl_image_frame), ori_image, self.vdl_image_step) self.vdl_writer.add_image( "result/frame_{}".format(self.vdl_image_frame), result_image, self.vdl_image_step) self.vdl_image_step += 1 # each frame can display ten pictures at most. if self.vdl_image_step % 10 == 0: self.vdl_image_step = 0 self.vdl_image_frame += 1 def on_epoch_end(self, status): mode = status['mode'] if dist.get_world_size() < 2 or dist.get_rank() == 0: if mode == 'eval': for metric in self.model._metrics: for key, map_value in metric.get_results().items(): self.vdl_writer.add_scalar("{}-mAP".format(key), map_value[0], self.vdl_mAP_step) self.vdl_mAP_step += 1
def _get_summary_writer(path): summary_writer = None try: from visualdl import LogWriter if distribution.status.is_master: summary_writer = LogWriter(os.path.join(path)) except ImportError: log.warning('VisualDL not installed, will not log to VisualDL') return summary_writer
def __init__(self, model, to_tensor, hparams, logger=None): # Use data distributed if hparams.use_data_distributed: strategy = parallel.prepare_context() if strategy is not None: parallel_model = parallel.DataParallel(model, strategy) model.before_backward_fn = parallel_model.scale_loss model.after_backward_fn = parallel_model.apply_collective_grads model = parallel_model self.model = model self.to_tensor = to_tensor self.is_decreased_valid_metric = hparams.valid_metric_name[0] == "-" self.valid_metric_name = hparams.valid_metric_name[1:] self.num_epochs = hparams.num_epochs self.save_dir = hparams.save_dir self.log_steps = hparams.log_steps self.valid_steps = hparams.valid_steps self.save_checkpoint = hparams.save_checkpoint self.save_summary = hparams.save_summary if not os.path.exists(self.save_dir): os.makedirs(self.save_dir) self.logger = logger or get_logger( os.path.join(self.save_dir, "trainer.log"), "trainer") if self.save_summary: from visualdl import LogWriter self.summary_logger = LogWriter(os.path.join( self.save_dir, "summary"), sync_cycle=10000) self.train_summary = {} self.valid_summary = {} self.batch_metrics_tracker = MetricsTracker() self.token_metrics_tracker = MetricsTracker() self.best_valid_metric = float( "inf" if self.is_decreased_valid_metric else "-inf") self.epoch = 0 self.batch_num = 0
def vdl_writer(self) -> LogWriter: ''' get vdl_writer for visualization. ''' if not os.path.exists(self.config.checkpoint_dir): os.mkdir(self.config.checkpoint_dir) tb_log_dir = os.path.join(self.config.checkpoint_dir, 'visualization') if not self._vdl_writer: self._vdl_writer = LogWriter(tb_log_dir) return self._vdl_writer
def __init__(self, solver_prototxt, log_dir, pretrained_model=None): self.solver = caffe.SGDSolver(solver_prototxt) if pretrained_model is not None: print('Loading pretrained model weights from {:s}'.format(pretrained_model)) self.solver.net.copy_from(pretrained_model) self.solver_param = caffe_pb2.SolverParameter() with open(solver_prototxt, 'rt') as f: pb2.text_format.Merge(f.read(), self.solver_param) self.cur_epoch = 0 self.test_interval = 500 #用来替代self.solver_param.test_interval #self.test_interval = 2000 #用来替代self.solver_param.test_interval self.logw = LogWriter(log_dir, sync_cycle=100) with self.logw.mode('train') as logger: self.sc_train_acc = logger.scalar("Accuracy") self.sc_train_lr = logger.scalar("learning_rate") with self.logw.mode('val') as logger: self.sc_val_acc = logger.scalar("Accuracy") self.sc_val_lr = logger.scalar("learning_rate")
def vdl_writer(self): """ get vdl_writer for visualization. """ if not os.path.exists(self.config.checkpoint_dir): mkdir(self.config.checkpoint_dir) tb_log_dir = os.path.join(self.config.checkpoint_dir, "visualization") if not self._vdl_writer: self._vdl_writer = LogWriter(tb_log_dir) return self._vdl_writer
def preprocess(is_train=False): FLAGS = ArgsParser().parse_args() profiler_options = FLAGS.profiler_options config = load_config(FLAGS.config) merge_config(FLAGS.opt) profile_dic = {"profiler_options": FLAGS.profiler_options} merge_config(profile_dic) if is_train: # save_config save_model_dir = config['Global']['save_model_dir'] os.makedirs(save_model_dir, exist_ok=True) with open(os.path.join(save_model_dir, 'config.yml'), 'w') as f: yaml.dump(dict(config), f, default_flow_style=False, sort_keys=False) log_file = '{}/train.log'.format(save_model_dir) else: log_file = None logger = get_logger(name='root', log_file=log_file) # check if set use_gpu=True in paddlepaddle cpu version use_gpu = config['Global']['use_gpu'] check_gpu(use_gpu) alg = config['Architecture']['algorithm'] assert alg in [ 'EAST', 'DB', 'SAST', 'Rosetta', 'CRNN', 'STARNet', 'RARE', 'SRN', 'CLS', 'PGNet', 'Distillation', 'NRTR', 'TableAttn', 'SAR', 'PSE', 'SEED', 'SDMGR' ] windows_not_support_list = ['PSE'] if platform.system() == "Windows" and alg in windows_not_support_list: logger.warning('{} is not support in Windows now'.format( windows_not_support_list)) sys.exit() device = 'gpu:{}'.format(dist.ParallelEnv().dev_id) if use_gpu else 'cpu' device = paddle.set_device(device) config['Global']['distributed'] = dist.get_world_size() != 1 if config['Global']['use_visualdl']: from visualdl import LogWriter save_model_dir = config['Global']['save_model_dir'] vdl_writer_path = '{}/vdl/'.format(save_model_dir) os.makedirs(vdl_writer_path, exist_ok=True) vdl_writer = LogWriter(logdir=vdl_writer_path) else: vdl_writer = None print_dict(config, logger) logger.info('train with paddle {} and device {}'.format( paddle.__version__, device)) return config, device, logger, vdl_writer
def __init__(self, solver_prototxt, log_dir, pretrained_model=None): self.solver = caffe.SGDSolver(solver_prototxt) if pretrained_model is not None: print('Loading pretrained model weights from {:s}'.format(pretrained_model)) self.solver.net.copy_from(pretrained_model) self.solver_param = caffe_pb2.SolverParameter() with open(solver_prototxt, 'rt') as f: pb2.text_format.Merge(f.read(), self.solver_param) self.cur_epoch = 0 self.test_interval = 30 #用来替代self.solver_param.test_interval self.logw = LogWriter(log_dir, sync_cycle=10) with self.logw.mode('train') as logger: self.sc_train_loss = logger.scalar("loss") self.sc_train_acc = logger.scalar("Accuracy") with self.logw.mode('val') as logger: self.sc_val_acc = logger.scalar("Accuracy(acc)") self.sc_val_auc = logger.scalar("Area Under Roc Curve(auc)") self.sc_val_ap = logger.scalar("Average Precision(ap)") self.sc_val_se = logger.scalar("Sensitivity(se)") self.sc_val_sp = logger.scalar("Specificity(sp)")
def get_writer(args): if args.writer_type == "visualdl": from visualdl import LogWriter writer = LogWriter(logdir=args.logdir) elif args.writer_type == "tensorboard": from tensorboardX import SummaryWriter writer = SummaryWriter(logdir=args.logdir) else: raise ValueError("writer_type must be in ['visualdl', 'tensorboard']") return writer
class MyLog(): def __init__(self,mode="train",logDir="../log"): self.mode=mode self.varDic={} self.log_writer = LogWriter(logDir, sync_cycle=10) def add_scalar(self,tag,scalar_value,global_step): if tag not in self.varDic: with self.log_writer.mode(self.mode) as writer: self.varDic[tag]=writer.scalar(tag) self.varDic[tag].add_record(global_step,scalar_value)
def __init__(self, data_loader, epochs, save_epoch, model_path, numTransform, numRef): self.data_loader = data_loader self.epochs = epochs self.model_path = model_path self.save_epoch = save_epoch self.numTransform = numTransform self.numRef = numRef self.G = Generator(numTransform, numRef) self.D = Discriminator(numTransform, numRef) self.G_optim = optim.SGD(self.G.parameters(), lr=1e-3, momentum=0.9) self.D_optim = optim.SGD(self.D.parameters(), lr=1e-3, momentum=0.9) if self.gpu_mode: self.G.cuda() self.D.cuda() self.BCE_loss = nn.BCELoss().cuda() self.L1_Loss = nn.L1Loss().cuda() else: self.BCE_loss = nn.BCELoss() self.L1_Loss = nn.L1Loss() self.save_path = model_path + '/model_%d.weights' logdir = model_path + "/tmp" logger = LogWriter(logdir, sync_cycle=10000) with logger.mode("train"): self.log_D_real_loss = logger.scalar("D/real_loss") self.log_D_fake_loss = logger.scalar("D/fake_loss") self.log_D_total_loss = logger.scalar("D/total_loss") self.log_G_D_loss = logger.scalar("G/D_Loss") self.log_G_L1_loss = logger.scalar("G/L1_Loss") self.log_G_total_loss = logger.scalar("G/total_Loss") with logger.mode("test"): self.log_test_loss = logger.scalar("test/loss")
def train(self): idx = 0 print(self.str_config) with LogWriter(logdir=os.path.join('./log', self.str_config, 'train')) as writer: for epoch in range(self.epochs): train_loss = 0 train_acc = 0 self.net.train() for batch_id, data in enumerate(self.train_loader()): x_data = data[0] # print('Type: ', type(data[1])) # y_data = paddle.to_tensor(np.where(data[1].numpy() > 5, 1, 0)) y_data = paddle.to_tensor(data[1].numpy().astype(np.int64)) predicts = self.net(x_data) train_loss = self.criterion(predicts, y_data) train_acc = paddle.metric.accuracy(predicts, y_data) train_loss.backward() if (batch_id + 1) % 100 == 0: print( "epoch: {}, batch_id: {}, loss is: {}, acc is: {}". format(epoch + 1, batch_id + 1, train_loss.numpy(), train_acc.numpy())) writer.add_scalar(tag='loss', step=idx, value=train_loss.numpy()) writer.add_scalar(tag='acc', step=idx, value=train_acc.numpy()) idx += 1 # self.losses.append(train_loss.numpy()) # self.acces.append(train_acc.numpy()) self.optm.step() self.optm.clear_grad()
class MyLog(): ''' 本类用于适配PaddleHub在AIStudio中VisualDL的使用 使用方式: # 创建 LogWriter 对象 log_writer = MyLog(mode="role2") seq_label_task._tb_writer=log_writer ''' def __init__(self, mode="train", logDir="../log"): self.mode = mode self.varDic = {} self.log_writer = LogWriter(logDir, sync_cycle=10) def add_scalar(self, tag, scalar_value, global_step): if not tag in self.varDic: with self.log_writer.mode(self.mode) as writer: self.varDic[tag] = writer.scalar(tag) self.varDic[tag].add_record(global_step, scalar_value)
class VisualDL(tf.keras.callbacks.Callback): def __init__(self,logdir,validation_every_times): super(VisualDL,self).__init__() self.logdir = logdir self.validation_every_times = validation_every_times def on_train_begin(self, logs=None): self.writer_train = LogWriter(self.logdir+"/train") self.writer_val = LogWriter(self.logdir+"/val") def on_batch_end(self, batch, logs=None): pass def on_epoch_end(self, epoch, logs=None): self.writer_train.add_scalar("loss/output_1_loss",logs["output_1_loss"],step=epoch) self.writer_train.add_scalar("metric/F1",logs["output_4_F1"],step=epoch) self.total_loss = 0.0 if ((epoch+1) % self.validation_every_times)==0: self.writer_val.add_scalar("loss/output_1_loss",logs["val_output_1_loss"],step=epoch) self.writer_val.add_scalar("metric/F1", logs["val_output_4_F1"], step=epoch)
def preprocess(is_train=False): FLAGS = ArgsParser().parse_args() config = load_config(FLAGS.config) merge_config(FLAGS.opt) # check if set use_gpu=True in paddlepaddle cpu version use_gpu = config['Global']['use_gpu'] check_gpu(use_gpu) alg = config['Architecture']['algorithm'] assert alg in [ 'EAST', 'DB', 'SAST', 'Rosetta', 'CRNN', 'STARNet', 'RARE', 'SRN', 'CLS' ] device = 'gpu:{}'.format(dist.ParallelEnv().dev_id) if use_gpu else 'cpu' device = paddle.set_device(device) config['Global']['distributed'] = dist.get_world_size() != 1 if is_train: # save_config save_model_dir = config['Global']['save_model_dir'] os.makedirs(save_model_dir, exist_ok=True) with open(os.path.join(save_model_dir, 'config.yml'), 'w') as f: yaml.dump(dict(config), f, default_flow_style=False, sort_keys=False) log_file = '{}/train.log'.format(save_model_dir) else: log_file = None logger = get_logger(name='root', log_file=log_file) if config['Global']['use_visualdl']: from visualdl import LogWriter save_model_dir = config['Global']['save_model_dir'] vdl_writer_path = '{}/vdl/'.format(save_model_dir) os.makedirs(vdl_writer_path, exist_ok=True) vdl_writer = LogWriter(logdir=vdl_writer_path) else: vdl_writer = None print_dict(config, logger) logger.info('train with paddle {} and device {}'.format( paddle.__version__, device)) return config, device, logger, vdl_writer
def visualize(dataset, img_count=3, save_dir='vdl_output'): '''对数据预处理/增强中间结果进行可视化。 可使用VisualDL查看中间结果: 1. VisualDL启动方式: visualdl --logdir vdl_output --port 8001 2. 浏览器打开 https://0.0.0.0:8001即可, 其中0.0.0.0为本机访问,如为远程服务, 改成相应机器IP Args: dataset (paddlex.datasets): 数据集读取器。 img_count (int): 需要进行数据预处理/增强的图像数目。默认为3。 save_dir (str): 日志保存的路径。默认为'vdl_output'。 ''' if dataset.num_samples < img_count: img_count = dataset.num_samples transforms = dataset.transforms if not osp.isdir(save_dir): if osp.exists(save_dir): os.remove(save_dir) os.makedirs(save_dir) from visualdl import LogWriter vdl_save_dir = osp.join(save_dir, 'image_transforms') vdl_writer = LogWriter(vdl_save_dir) for i, data in enumerate(dataset.iterator()): if i == img_count: break data.append(transforms.transforms) data.append(vdl_writer) data.append(i) if isinstance(transforms, ClsTransform): cls_compose(*data) elif isinstance(transforms, DetTransform): labels = dataset.labels color_map = get_color_map_list(len(labels) + 1) catid2color = {} for catid in range(len(labels)): catid2color[catid] = color_map[catid + 1] data.append(labels) data.append(catid2color) det_compose(*data) elif isinstance(transforms, SegTransform): seg_compose(*data) else: raise Exception('The transform must the subclass of \ ClsTransform or DetTransform or SegTransform!')