def __init__(self,
vocab_size,
num_conv_layers,
num_rnn_layers,
rnn_layer_size,
use_gru=False,
share_rnn_weights=True,
place=fluid.CPUPlace(),
init_from_pretrained_model=None,
output_model_dir=None,
is_infer=False,
error_rate_type='cer',
vocab_list=None):
self._vocab_size = vocab_size
self._num_conv_layers = num_conv_layers
self._num_rnn_layers = num_rnn_layers
self._rnn_layer_size = rnn_layer_size
self._use_gru = use_gru
self._share_rnn_weights = share_rnn_weights
self._place = place
self._init_from_pretrained_model = init_from_pretrained_model
self._output_model_dir = output_model_dir
self._ext_scorer = None
self.logger = logging.getLogger("")
self.logger.setLevel(level=logging.INFO)
self.writer = LogWriter(logdir='log')
self.error_rate_type = error_rate_type
self.vocab_list = vocab_list
# 预测相关的参数
self.infer_program = None
self.infer_feeder = None
self.infer_log_probs = None
self.infer_exe = None
if is_infer:
self.init_infer_program()
def __init__(self, hparams):
# Use data distributed
self.infer_network = hparams.infer_network
self.dataset = hparams.dataset
self.num_epochs = hparams.num_epochs
self.batch_size = hparams.batch_size
self.enable_ce = hparams.enable_ce
self.logger = hparams.logger
self.cpu_num = hparams.cpu_num
self.cuda_devices = hparams.cuda_devices
self.multi_card = hparams.multi_card
self.num_class = 10 # default is cifar-10
if self.logger:
from visualdl import LogWriter
self.log_writer = LogWriter(self.logger, sync_cycle=20)
# Create two ScalarWriter instances, whose mode is set to be "train"
with self.log_writer.mode("train") as logger:
self.train_cost = logger.scalar("cost")
self.train_acc = logger.scalar("acc")
# Create a ScalarWriter instance, whose mode is set to be "test"
with self.log_writer.mode("test") as logger:
self.test_loss = logger.scalar("loss")
self.test_acc = logger.scalar("acc")
if self.dataset is "cifar-100":
self.num_class = 100
def __init__(self,
model: paddle.nn.Layer,
optimizer: paddle.optimizer.Optimizer,
use_vdl: bool = True,
checkpoint_dir: str = None,
compare_metrics: Callable = None):
self.nranks = paddle.distributed.get_world_size()
self.local_rank = paddle.distributed.get_rank()
self.model = model
self.optimizer = optimizer
self.checkpoint_dir = checkpoint_dir if checkpoint_dir else 'ckpt_{}'.format(time.time())
if not isinstance(self.model, paddle.nn.Layer):
raise TypeError('The model {} is not a `paddle.nn.Layer` object.'.format(self.model.__name__))
if self.local_rank == 0 and not os.path.exists(self.checkpoint_dir):
os.makedirs(self.checkpoint_dir)
self.use_vdl = use_vdl
if self.local_rank == 0 and self.use_vdl:
vdl_dir = os.path.join(self.checkpoint_dir, 'visualization')
self.log_writer = LogWriter(vdl_dir)
self.current_epoch = 0
self.best_metrics = defaultdict(int)
if self.nranks > 1:
paddle.distributed.init_parallel_env()
strategy = paddle.distributed.prepare_context()
self.model = paddle.DataParallel(self.model, strategy)
self.compare_metrics = self._compare_metrics if not compare_metrics else compare_metrics
self._load_checkpoint()
def __init__(self,
logdir,
experimental_name,
*,
save_model_with_input=None):
"""
:param logdir:
:param experimental_name:
:param save_model_with_input: 默认不存储模型结构，当开启该参数时，
"""
from pyxllib.prog.pupil import check_install_package
check_install_package('visualdl')
from visualdl import LogWriter
super().__init__()
# 这样奇怪地加后缀，是为了字典序后，每个实验的train显示在eval之前
d = XlPath(logdir) / (experimental_name + '_train')
# if d.exists(): shutil.rmtree(d)
self.write = LogWriter(logdir=str(d))
d = XlPath(logdir) / (experimental_name + '_val')
# if d.exists(): shutil.rmtree(d)
self.eval_writer = LogWriter(logdir=str(d))
self.eval_times = 0
self.save_model_with_input = save_model_with_input
def run_begin(self, trainer):
rank = dist.get_rank()
if rank != 0:
return
logdir = os.path.join(trainer.output_dir, 'visual_dl')
if not os.path.exists(logdir):
os.makedirs(logdir)
self.writer = LogWriter(logdir=logdir)
def train(config):
use_gpu = config.use_gpu
# Get the rank of the current training process.
rank = dg.parallel.Env().local_rank
nranks = dg.parallel.Env().nranks
parallel = nranks > 1
if rank == 0:
# Print the whole config setting.
pprint(vars(config))
# Make checkpoint directory.
run_dir = os.path.join("runs", config.model, config.name)
checkpoint_dir = os.path.join(run_dir, "checkpoint")
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
# Create tensorboard logger.
vdl = LogWriter(os.path.join(run_dir, "logs")) \
if rank == 0 else None
# Configurate device
place = fluid.CUDAPlace(rank) if use_gpu else fluid.CPUPlace()
with dg.guard(place):
# Fix random seed.
seed = config.seed
random.seed(seed)
np.random.seed(seed)
fluid.default_startup_program().random_seed = seed
fluid.default_main_program().random_seed = seed
print("Random Seed: ", seed)
# Build model.
model = WaveFlow(config, checkpoint_dir, parallel, rank, nranks, vdl)
iteration = model.build()
while iteration < config.max_iterations:
# Run one single training step.
model.train_step(iteration)
iteration += 1
if iteration % config.test_every == 0:
# Run validation step.
model.valid_step(iteration)
if rank == 0 and iteration % config.save_every == 0:
# Save parameters.
model.save(iteration)
# Close TensorBoard.
if rank == 0:
vdl.close()
def __init__(self):
"""
Initialize the Linear Model
"""
self.learning_rate = 0.001
self.epoches = 10000
self.loss_function = torch.nn.MSELoss()
self.create_model()
logdir = os.path.join('./runs', datetime.now().strftime('%b%d_%H-%M-%S')+'_line_reg')
print("save to ", logdir)
self.writer = LogWriter(logdir=logdir)
print("initialize done")
def call(*args, **kwargs):
global _writer
if _writer is None:
logdir = logger.get_dir()
if logdir is None:
logdir = logger.auto_set_dir(action='d')
logger.warning(
"[VisualDL] logdir is None, will save VisualDL files to {}\nView the data using: visualdl --logdir=./{} --host={}"
.format(logdir, logdir, get_ip_address()))
_writer = LogWriter(logdir=logger.get_dir())
func = getattr(_writer, func_name)
func(*args, **kwargs)
_writer.flush()
def test_modes(self):
store = LogWriter(self.dir, sync_cycle=1)
scalars = []
for i in range(10):
with store.mode("mode-%d" % i) as writer:
scalar = writer.scalar("add/scalar0")
scalars.append(scalar)
for scalar in scalars[:-1]:
for i in range(10):
scalar.add_record(i, float(i))
class VisualDL(tf.keras.callbacks.Callback):
def __init__(self, logdir, save_per_epoch):
super(VisualDL, self).__init__()
self.logdir = logdir
self.save_per_epoch = save_per_epoch
def on_train_begin(self, logs=None):
self.writer_train = LogWriter(self.logdir + "/train")
self.writer_val = LogWriter(self.logdir + "/val")
def on_batch_end(self, batch, logs=None):
pass
def on_epoch_end(self, epoch, logs=None):
self.writer_train.add_scalar("loss/output_1_loss",
logs["output_1_loss"],
step=epoch + 1)
self.writer_train.add_scalar("metric/F1",
logs["output_2_F1"],
step=epoch + 1)
self.writer_train.add_scalar("metric/Precision",
logs["output_2_Precision"],
step=epoch + 1)
self.writer_train.add_scalar("metric/Recall",
logs["output_2_Recall"],
step=epoch + 1)
def __init__(self, model):
super(VisualDLWriter, self).__init__(model)
assert six.PY3, "VisualDL requires Python >= 3.5"
try:
from visualdl import LogWriter
except Exception as e:
logger.error('visualdl not found, plaese install visualdl. '
'for example: `pip install visualdl`.')
raise e
self.vdl_writer = LogWriter(model.cfg.vdl_log_dir)
self.vdl_loss_step = 0
self.vdl_mAP_step = 0
self.vdl_image_step = 0
self.vdl_image_frame = 0
def validate(self):
# testingzX
print('Validate: ')
eval_loss = 0
eval_acc = 0
idx = 0
self.net.eval()
with LogWriter(
logdir=os.path.join('./log', self.str_config, 'val')) as witer:
for batch_id, data in enumerate(self.test_loader()):
x = data[0]
# y = paddle.to_tensor(np.where(data[1].numpy() > 5, 1, 0))
y = data[1]
predicts = self.net(x)
eval_loss = self.criterion(predicts, y)
eval_acc = paddle.metric.accuracy(predicts, y)
# 打印信息
if (batch_id + 1) % 100 == 0:
print("batch_id: {}, loss is: {}, acc is: {}".format(
batch_id + 1, eval_loss.numpy(), eval_acc.numpy()))
witer.add_scalar(tag='acc',
step=idx,
value=eval_acc.numpy())
witer.add_scalar(tag='loss',
step=idx,
value=eval_loss.numpy())
idx += 1
def __init__(
self,
evaluator,
cudas=["0"],
popsize=5,
output_dir=None,
):
self._num_thread = len(cudas)
self._popsize = popsize
self.cudas = cudas
self.is_cuda_free = {"free": [], "busy": []}
self.is_cuda_free["free"] = cudas
self._round = 0
self.evaluator = evaluator
self.init_input = evaluator.get_init_params()
self.num_hparam = len(self.init_input)
self.best_hparams_all_pop = []
self.best_reward_all_pop = INF
self.current_hparams = [[0] * self.num_hparam] * self._popsize
self.hparams_name_list = [
param["name"] for param in evaluator.params['param_list']
]
if output_dir is None:
now = int(time.time())
time_str = time.strftime("%Y%m%d%H%M%S", time.localtime(now))
self._output_dir = "output_" + time_str
else:
self._output_dir = output_dir
# record the information for the whole auto finetune
self.writer = LogWriter(logdir=self._output_dir + '/visualization')
# record the information for per population in all round
self.writer_pop_trails = []
for i in range(self.popsize):
writer_pop_trail = LogWriter(
logdir=self._output_dir + '/visualization/pop_{}'.format(i))
self.writer_pop_trails.append(writer_pop_trail)
# for parallel on mpi
self.mpi = MPIHelper()
if self.mpi.multi_machine:
print("Autofinetune multimachine mode: running on {}".format(
self.mpi.gather(self.mpi.name)))
def __init__(self, config: Config, processor: DataProcessor,
classifier) -> None:
self.config = config
self.set_device()
self.tokenizer: PretrainedTokenizer = ppnlp.transformers.BertTokenizer.from_pretrained(
config.pretrained_model)
# 2. build data related objects
self.train_dataset = processor.get_train_dataset()
self.dev_dataset = processor.get_dev_dataset()
self.test_dataset = processor.get_test_dataset()
self.train_dataloader = create_dataloader(
self.train_dataset,
batch_size=config.batch_size,
collate_fn=lambda examples: self.collate_fn(
examples, self.train_dataset.label2idx),
)
self.dev_dataloader = create_dataloader(
self.dev_dataset,
batch_size=config.batch_size,
collate_fn=lambda examples: self.collate_fn(
examples, self.dev_dataset.label2idx),
)
self.test_dataloader = create_dataloader(
self.test_dataset,
batch_size=config.batch_size,
collate_fn=lambda examples: self.collate_fn(
examples, self.test_dataset.label2idx),
)
# 3. init model related
self.model = classifier
self.lr_scheduler: LRScheduler = LinearDecayWithWarmup(
config.learning_rate,
total_steps=len(self.train_dataloader) * config.epochs,
warmup=config.warmup_proportion)
if config.init_from_ckpt and os.path.isfile(config.init_from_ckpt):
state_dict = paddle.load(config.init_from_ckpt)
self.model.set_dict(state_dict)
# Generate parameter names needed to perform weight decay.
# All bias and LayerNorm parameters are excluded.
decay_params = [
p.name for n, p in self.model.named_parameters()
if not any(nd in n for nd in ["bias", "norm"])
]
self.optimizer: Optimizer = paddle.optimizer.AdamW(
learning_rate=self.lr_scheduler,
parameters=self.model.parameters(),
weight_decay=config.weight_decay,
apply_decay_param_fun=lambda x: x in decay_params)
self.criterion = paddle.nn.loss.CrossEntropyLoss()
self.metric: Metric = paddle.metric.Accuracy()
self.context_data = ContextContainer()
self._init_output_dir()
self.writer: LogWriter = LogWriter(logdir=config.output_dir)
class VisualDLWriter(Callback):
"""
Use VisualDL to log data or image
"""
def __init__(self, model):
super(VisualDLWriter, self).__init__(model)
assert six.PY3, "VisualDL requires Python >= 3.5"
try:
from visualdl import LogWriter
except Exception as e:
logger.error('visualdl not found, plaese install visualdl. '
'for example: `pip install visualdl`.')
raise e
self.vdl_writer = LogWriter(
model.cfg.get('vdl_log_dir', 'vdl_log_dir/scalar'))
self.vdl_loss_step = 0
self.vdl_mAP_step = 0
self.vdl_image_step = 0
self.vdl_image_frame = 0
def on_step_end(self, status):
mode = status['mode']
if dist.get_world_size() < 2 or dist.get_rank() == 0:
if mode == 'train':
training_staus = status['training_staus']
for loss_name, loss_value in training_staus.get().items():
self.vdl_writer.add_scalar(loss_name, loss_value,
self.vdl_loss_step)
self.vdl_loss_step += 1
elif mode == 'test':
ori_image = status['original_image']
result_image = status['result_image']
self.vdl_writer.add_image(
"original/frame_{}".format(self.vdl_image_frame),
ori_image, self.vdl_image_step)
self.vdl_writer.add_image(
"result/frame_{}".format(self.vdl_image_frame),
result_image, self.vdl_image_step)
self.vdl_image_step += 1
# each frame can display ten pictures at most.
if self.vdl_image_step % 10 == 0:
self.vdl_image_step = 0
self.vdl_image_frame += 1
def on_epoch_end(self, status):
mode = status['mode']
if dist.get_world_size() < 2 or dist.get_rank() == 0:
if mode == 'eval':
for metric in self.model._metrics:
for key, map_value in metric.get_results().items():
self.vdl_writer.add_scalar("{}-mAP".format(key),
map_value[0],
self.vdl_mAP_step)
self.vdl_mAP_step += 1
def _get_summary_writer(path):
summary_writer = None
try:
from visualdl import LogWriter
if distribution.status.is_master:
summary_writer = LogWriter(os.path.join(path))
except ImportError:
log.warning('VisualDL not installed, will not log to VisualDL')
return summary_writer
def __init__(self, model, to_tensor, hparams, logger=None):
# Use data distributed
if hparams.use_data_distributed:
strategy = parallel.prepare_context()
if strategy is not None:
parallel_model = parallel.DataParallel(model, strategy)
model.before_backward_fn = parallel_model.scale_loss
model.after_backward_fn = parallel_model.apply_collective_grads
model = parallel_model
self.model = model
self.to_tensor = to_tensor
self.is_decreased_valid_metric = hparams.valid_metric_name[0] == "-"
self.valid_metric_name = hparams.valid_metric_name[1:]
self.num_epochs = hparams.num_epochs
self.save_dir = hparams.save_dir
self.log_steps = hparams.log_steps
self.valid_steps = hparams.valid_steps
self.save_checkpoint = hparams.save_checkpoint
self.save_summary = hparams.save_summary
if not os.path.exists(self.save_dir):
os.makedirs(self.save_dir)
self.logger = logger or get_logger(
os.path.join(self.save_dir, "trainer.log"), "trainer")
if self.save_summary:
from visualdl import LogWriter
self.summary_logger = LogWriter(os.path.join(
self.save_dir, "summary"),
sync_cycle=10000)
self.train_summary = {}
self.valid_summary = {}
self.batch_metrics_tracker = MetricsTracker()
self.token_metrics_tracker = MetricsTracker()
self.best_valid_metric = float(
"inf" if self.is_decreased_valid_metric else "-inf")
self.epoch = 0
self.batch_num = 0
def vdl_writer(self) -> LogWriter:
'''
get vdl_writer for visualization.
'''
if not os.path.exists(self.config.checkpoint_dir):
os.mkdir(self.config.checkpoint_dir)
tb_log_dir = os.path.join(self.config.checkpoint_dir, 'visualization')
if not self._vdl_writer:
self._vdl_writer = LogWriter(tb_log_dir)
return self._vdl_writer
def __init__(self, solver_prototxt, log_dir, pretrained_model=None):
self.solver = caffe.SGDSolver(solver_prototxt)
if pretrained_model is not None:
print('Loading pretrained model weights from {:s}'.format(pretrained_model))
self.solver.net.copy_from(pretrained_model)
self.solver_param = caffe_pb2.SolverParameter()
with open(solver_prototxt, 'rt') as f:
pb2.text_format.Merge(f.read(), self.solver_param)
self.cur_epoch = 0
self.test_interval = 500 #用来替代self.solver_param.test_interval
#self.test_interval = 2000 #用来替代self.solver_param.test_interval
self.logw = LogWriter(log_dir, sync_cycle=100)
with self.logw.mode('train') as logger:
self.sc_train_acc = logger.scalar("Accuracy")
self.sc_train_lr = logger.scalar("learning_rate")
with self.logw.mode('val') as logger:
self.sc_val_acc = logger.scalar("Accuracy")
self.sc_val_lr = logger.scalar("learning_rate")
def vdl_writer(self):
"""
get vdl_writer for visualization.
"""
if not os.path.exists(self.config.checkpoint_dir):
mkdir(self.config.checkpoint_dir)
tb_log_dir = os.path.join(self.config.checkpoint_dir, "visualization")
if not self._vdl_writer:
self._vdl_writer = LogWriter(tb_log_dir)
return self._vdl_writer
def preprocess(is_train=False):
FLAGS = ArgsParser().parse_args()
profiler_options = FLAGS.profiler_options
config = load_config(FLAGS.config)
merge_config(FLAGS.opt)
profile_dic = {"profiler_options": FLAGS.profiler_options}
merge_config(profile_dic)
if is_train:
# save_config
save_model_dir = config['Global']['save_model_dir']
os.makedirs(save_model_dir, exist_ok=True)
with open(os.path.join(save_model_dir, 'config.yml'), 'w') as f:
yaml.dump(dict(config),
f,
default_flow_style=False,
sort_keys=False)
log_file = '{}/train.log'.format(save_model_dir)
else:
log_file = None
logger = get_logger(name='root', log_file=log_file)
# check if set use_gpu=True in paddlepaddle cpu version
use_gpu = config['Global']['use_gpu']
check_gpu(use_gpu)
alg = config['Architecture']['algorithm']
assert alg in [
'EAST', 'DB', 'SAST', 'Rosetta', 'CRNN', 'STARNet', 'RARE', 'SRN',
'CLS', 'PGNet', 'Distillation', 'NRTR', 'TableAttn', 'SAR', 'PSE',
'SEED', 'SDMGR'
]
windows_not_support_list = ['PSE']
if platform.system() == "Windows" and alg in windows_not_support_list:
logger.warning('{} is not support in Windows now'.format(
windows_not_support_list))
sys.exit()
device = 'gpu:{}'.format(dist.ParallelEnv().dev_id) if use_gpu else 'cpu'
device = paddle.set_device(device)
config['Global']['distributed'] = dist.get_world_size() != 1
if config['Global']['use_visualdl']:
from visualdl import LogWriter
save_model_dir = config['Global']['save_model_dir']
vdl_writer_path = '{}/vdl/'.format(save_model_dir)
os.makedirs(vdl_writer_path, exist_ok=True)
vdl_writer = LogWriter(logdir=vdl_writer_path)
else:
vdl_writer = None
print_dict(config, logger)
logger.info('train with paddle {} and device {}'.format(
paddle.__version__, device))
return config, device, logger, vdl_writer
def __init__(self, solver_prototxt, log_dir, pretrained_model=None):
self.solver = caffe.SGDSolver(solver_prototxt)
if pretrained_model is not None:
print('Loading pretrained model weights from {:s}'.format(pretrained_model))
self.solver.net.copy_from(pretrained_model)
self.solver_param = caffe_pb2.SolverParameter()
with open(solver_prototxt, 'rt') as f:
pb2.text_format.Merge(f.read(), self.solver_param)
self.cur_epoch = 0
self.test_interval = 30 #用来替代self.solver_param.test_interval
self.logw = LogWriter(log_dir, sync_cycle=10)
with self.logw.mode('train') as logger:
self.sc_train_loss = logger.scalar("loss")
self.sc_train_acc = logger.scalar("Accuracy")
with self.logw.mode('val') as logger:
self.sc_val_acc = logger.scalar("Accuracy(acc)")
self.sc_val_auc = logger.scalar("Area Under Roc Curve(auc)")
self.sc_val_ap = logger.scalar("Average Precision(ap)")
self.sc_val_se = logger.scalar("Sensitivity(se)")
self.sc_val_sp = logger.scalar("Specificity(sp)")
def get_writer(args):
if args.writer_type == "visualdl":
from visualdl import LogWriter
writer = LogWriter(logdir=args.logdir)
elif args.writer_type == "tensorboard":
from tensorboardX import SummaryWriter
writer = SummaryWriter(logdir=args.logdir)
else:
raise ValueError("writer_type must be in ['visualdl', 'tensorboard']")
return writer
class MyLog():
def __init__(self,mode="train",logDir="../log"):
self.mode=mode
self.varDic={}
self.log_writer = LogWriter(logDir, sync_cycle=10)
def add_scalar(self,tag,scalar_value,global_step):
if tag not in self.varDic:
with self.log_writer.mode(self.mode) as writer:
self.varDic[tag]=writer.scalar(tag)
self.varDic[tag].add_record(global_step,scalar_value)
def __init__(self, data_loader, epochs, save_epoch, model_path, numTransform, numRef):
self.data_loader = data_loader
self.epochs = epochs
self.model_path = model_path
self.save_epoch = save_epoch
self.numTransform = numTransform
self.numRef = numRef
self.G = Generator(numTransform, numRef)
self.D = Discriminator(numTransform, numRef)
self.G_optim = optim.SGD(self.G.parameters(), lr=1e-3, momentum=0.9)
self.D_optim = optim.SGD(self.D.parameters(), lr=1e-3, momentum=0.9)
if self.gpu_mode:
self.G.cuda()
self.D.cuda()
self.BCE_loss = nn.BCELoss().cuda()
self.L1_Loss = nn.L1Loss().cuda()
else:
self.BCE_loss = nn.BCELoss()
self.L1_Loss = nn.L1Loss()
self.save_path = model_path + '/model_%d.weights'
logdir = model_path + "/tmp"
logger = LogWriter(logdir, sync_cycle=10000)
with logger.mode("train"):
self.log_D_real_loss = logger.scalar("D/real_loss")
self.log_D_fake_loss = logger.scalar("D/fake_loss")
self.log_D_total_loss = logger.scalar("D/total_loss")
self.log_G_D_loss = logger.scalar("G/D_Loss")
self.log_G_L1_loss = logger.scalar("G/L1_Loss")
self.log_G_total_loss = logger.scalar("G/total_Loss")
with logger.mode("test"):
self.log_test_loss = logger.scalar("test/loss")
def train(self):
idx = 0
print(self.str_config)
with LogWriter(logdir=os.path.join('./log', self.str_config,
'train')) as writer:
for epoch in range(self.epochs):
train_loss = 0
train_acc = 0
self.net.train()
for batch_id, data in enumerate(self.train_loader()):
x_data = data[0]
# print('Type: ', type(data[1]))
# y_data = paddle.to_tensor(np.where(data[1].numpy() > 5, 1, 0))
y_data = paddle.to_tensor(data[1].numpy().astype(np.int64))
predicts = self.net(x_data)
train_loss = self.criterion(predicts, y_data)
train_acc = paddle.metric.accuracy(predicts, y_data)
train_loss.backward()
if (batch_id + 1) % 100 == 0:
print(
"epoch: {}, batch_id: {}, loss is: {}, acc is: {}".
format(epoch + 1, batch_id + 1, train_loss.numpy(),
train_acc.numpy()))
writer.add_scalar(tag='loss',
step=idx,
value=train_loss.numpy())
writer.add_scalar(tag='acc',
step=idx,
value=train_acc.numpy())
idx += 1
# self.losses.append(train_loss.numpy())
# self.acces.append(train_acc.numpy())
self.optm.step()
self.optm.clear_grad()
class MyLog():
'''
本类用于适配PaddleHub在AIStudio中VisualDL的使用
使用方式：
# 创建 LogWriter 对象
log_writer = MyLog(mode="role2")
seq_label_task._tb_writer=log_writer
'''
def __init__(self, mode="train", logDir="../log"):
self.mode = mode
self.varDic = {}
self.log_writer = LogWriter(logDir, sync_cycle=10)
def add_scalar(self, tag, scalar_value, global_step):
if not tag in self.varDic:
with self.log_writer.mode(self.mode) as writer:
self.varDic[tag] = writer.scalar(tag)
self.varDic[tag].add_record(global_step, scalar_value)
class VisualDL(tf.keras.callbacks.Callback):
def __init__(self,logdir,validation_every_times):
super(VisualDL,self).__init__()
self.logdir = logdir
self.validation_every_times = validation_every_times
def on_train_begin(self, logs=None):
self.writer_train = LogWriter(self.logdir+"/train")
self.writer_val = LogWriter(self.logdir+"/val")
def on_batch_end(self, batch, logs=None):
pass
def on_epoch_end(self, epoch, logs=None):
self.writer_train.add_scalar("loss/output_1_loss",logs["output_1_loss"],step=epoch)
self.writer_train.add_scalar("metric/F1",logs["output_4_F1"],step=epoch)
self.total_loss = 0.0
if ((epoch+1) % self.validation_every_times)==0:
self.writer_val.add_scalar("loss/output_1_loss",logs["val_output_1_loss"],step=epoch)
self.writer_val.add_scalar("metric/F1", logs["val_output_4_F1"], step=epoch)
def preprocess(is_train=False):
FLAGS = ArgsParser().parse_args()
config = load_config(FLAGS.config)
merge_config(FLAGS.opt)
# check if set use_gpu=True in paddlepaddle cpu version
use_gpu = config['Global']['use_gpu']
check_gpu(use_gpu)
alg = config['Architecture']['algorithm']
assert alg in [
'EAST', 'DB', 'SAST', 'Rosetta', 'CRNN', 'STARNet', 'RARE', 'SRN',
'CLS'
]
device = 'gpu:{}'.format(dist.ParallelEnv().dev_id) if use_gpu else 'cpu'
device = paddle.set_device(device)
config['Global']['distributed'] = dist.get_world_size() != 1
if is_train:
# save_config
save_model_dir = config['Global']['save_model_dir']
os.makedirs(save_model_dir, exist_ok=True)
with open(os.path.join(save_model_dir, 'config.yml'), 'w') as f:
yaml.dump(dict(config),
f,
default_flow_style=False,
sort_keys=False)
log_file = '{}/train.log'.format(save_model_dir)
else:
log_file = None
logger = get_logger(name='root', log_file=log_file)
if config['Global']['use_visualdl']:
from visualdl import LogWriter
save_model_dir = config['Global']['save_model_dir']
vdl_writer_path = '{}/vdl/'.format(save_model_dir)
os.makedirs(vdl_writer_path, exist_ok=True)
vdl_writer = LogWriter(logdir=vdl_writer_path)
else:
vdl_writer = None
print_dict(config, logger)
logger.info('train with paddle {} and device {}'.format(
paddle.__version__, device))
return config, device, logger, vdl_writer
def visualize(dataset, img_count=3, save_dir='vdl_output'):
'''对数据预处理/增强中间结果进行可视化。
可使用VisualDL查看中间结果：
1. VisualDL启动方式: visualdl --logdir vdl_output --port 8001
2. 浏览器打开 https://0.0.0.0:8001即可，
其中0.0.0.0为本机访问，如为远程服务, 改成相应机器IP
Args:
dataset (paddlex.datasets): 数据集读取器。
img_count (int): 需要进行数据预处理/增强的图像数目。默认为3。
save_dir (str): 日志保存的路径。默认为'vdl_output'。
'''
if dataset.num_samples < img_count:
img_count = dataset.num_samples
transforms = dataset.transforms
if not osp.isdir(save_dir):
if osp.exists(save_dir):
os.remove(save_dir)
os.makedirs(save_dir)
from visualdl import LogWriter
vdl_save_dir = osp.join(save_dir, 'image_transforms')
vdl_writer = LogWriter(vdl_save_dir)
for i, data in enumerate(dataset.iterator()):
if i == img_count:
break
data.append(transforms.transforms)
data.append(vdl_writer)
data.append(i)
if isinstance(transforms, ClsTransform):
cls_compose(*data)
elif isinstance(transforms, DetTransform):
labels = dataset.labels
color_map = get_color_map_list(len(labels) + 1)
catid2color = {}
for catid in range(len(labels)):
catid2color[catid] = color_map[catid + 1]
data.append(labels)
data.append(catid2color)
det_compose(*data)
elif isinstance(transforms, SegTransform):
seg_compose(*data)
else:
raise Exception('The transform must the subclass of \
ClsTransform or DetTransform or SegTransform!')
