def __init__(self, dir):
import datetime
import visualdl
self.now_time = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M')
self.dir = dir + '/' + self.now_time
self.train_vsdl = visualdl.LogWriter(logdir=self.dir + '/train')
self.test_vsdl = visualdl.LogWriter(logdir=self.dir + '/test')
self.flag = 1
def scalar_test():
log_dir = 'visual_test'
visualLogger = visualdl.LogWriter(log_dir, sync_cycle=20)
logging.basicConfig(format='[%(asctime)s][%(levelname)s] %(message)s')
logger = logging.getLogger()
logger.setLevel(logging.INFO)
try:
with Archive('proxy_test') as archive:
proxy = LoggingProxy(visualLogger, archive ,logger)
with proxy.in_run('train'):
hello = proxy.scalar('hello')
for i in range(1000):
if i == 768:
raise ValueError('intented')
hello(i, i)
except Exception as e:
if e.args[0] != 'intented':
raise e
with h5.File('proxy_test.hdf5', 'r') as f:
label = f['train/scalar/hello']
assert label.shape == (768,)
for i in range(768):
assert label[i] == i, 'content error'
def __init__(self, cfg):
# build train dataloader
self.train_dataloader = build_dataloader(cfg.dataset.train)
if 'lr_scheduler' in cfg.optimizer:
cfg.optimizer.lr_scheduler.step_per_epoch = len(
self.train_dataloader)
# build model
self.model = build_model(cfg)
# multiple gpus prepare
if ParallelEnv().nranks > 1:
self.distributed_data_parallel()
self.logger = logging.getLogger(__name__)
self.enable_visualdl = cfg.get('enable_visualdl', False)
if self.enable_visualdl:
import visualdl
self.vdl_logger = visualdl.LogWriter(logdir=cfg.output_dir)
# base config
self.output_dir = cfg.output_dir
self.epochs = cfg.epochs
self.start_epoch = 1
self.current_epoch = 1
self.batch_id = 0
self.global_steps = 0
self.weight_interval = cfg.snapshot_config.interval
self.log_interval = cfg.log_config.interval
self.visual_interval = cfg.log_config.visiual_interval
self.validate_interval = -1
if cfg.get('validate', None) is not None:
self.validate_interval = cfg.validate.get('interval', -1)
self.cfg = cfg
self.local_rank = ParallelEnv().local_rank
# time count
self.steps_per_epoch = len(self.train_dataloader)
self.total_steps = self.epochs * self.steps_per_epoch
self.time_count = {}
self.best_metric = {}
def __init__(self, cfg):
# base config
self.logger = logging.getLogger(__name__)
self.cfg = cfg
self.output_dir = cfg.output_dir
self.max_eval_steps = cfg.model.get('max_eval_steps', None)
self.local_rank = ParallelEnv().local_rank
self.log_interval = cfg.log_config.interval
self.visual_interval = cfg.log_config.visiual_interval
self.weight_interval = cfg.snapshot_config.interval
self.start_epoch = 1
self.current_epoch = 1
self.current_iter = 1
self.inner_iter = 1
self.batch_id = 0
self.global_steps = 0
# build model
self.model = build_model(cfg.model)
# multiple gpus prepare
if ParallelEnv().nranks > 1:
self.distributed_data_parallel()
# build metrics
self.metrics = None
validate_cfg = cfg.get('validate', None)
if validate_cfg and 'metrics' in validate_cfg:
self.metrics = self.model.setup_metrics(validate_cfg['metrics'])
self.enable_visualdl = cfg.get('enable_visualdl', False)
if self.enable_visualdl:
import visualdl
self.vdl_logger = visualdl.LogWriter(logdir=cfg.output_dir)
# evaluate only
if not cfg.is_train:
return
# build train dataloader
self.train_dataloader = build_dataloader(cfg.dataset.train)
self.iters_per_epoch = len(self.train_dataloader)
# build lr scheduler
# TODO: has a better way?
if 'lr_scheduler' in cfg and 'iters_per_epoch' in cfg.lr_scheduler:
cfg.lr_scheduler.iters_per_epoch = self.iters_per_epoch
self.lr_schedulers = self.model.setup_lr_schedulers(cfg.lr_scheduler)
# build optimizers
self.optimizers = self.model.setup_optimizers(self.lr_schedulers,
cfg.optimizer)
self.epochs = cfg.get('epochs', None)
if self.epochs:
self.total_iters = self.epochs * self.iters_per_epoch
self.by_epoch = True
else:
self.by_epoch = False
self.total_iters = cfg.total_iters
if self.by_epoch:
self.weight_interval *= self.iters_per_epoch
self.validate_interval = -1
if cfg.get('validate', None) is not None:
self.validate_interval = cfg.validate.get('interval', -1)
self.time_count = {}
self.best_metric = {}
self.model.set_total_iter(self.total_iters)
def parse_args_and_config():
parser = argparse.ArgumentParser(description=globals()["__doc__"])
parser.add_argument("--config",
type=str,
required=True,
help="Path to the config file")
parser.add_argument("--seed", type=int, default=1234, help="Random seed")
parser.add_argument("--exp",
type=str,
default="exp",
help="Path for saving running related data.")
parser.add_argument(
"--doc",
type=str,
required=True,
help="A string for documentation purpose. "
"Will be the name of the log folder.",
)
parser.add_argument("--comment",
type=str,
default="",
help="A string for experiment comment")
parser.add_argument(
"--verbose",
type=str,
default="info",
help="Verbose level: info | debug | warning | critical",
)
parser.add_argument("--test",
action="store_true",
help="Whether to test the model")
parser.add_argument(
"--sample",
action="store_true",
help="Whether to produce samples from the model",
)
parser.add_argument("--fid", action="store_true")
parser.add_argument("--interpolation", action="store_true")
parser.add_argument("--resume_training",
action="store_true",
help="Whether to resume training")
parser.add_argument(
"-i",
"--image_folder",
type=str,
default="images",
help="The folder name of samples",
)
parser.add_argument(
"--ni",
action="store_true",
help="No interaction. Suitable for Slurm Job launcher",
)
parser.add_argument("--use_pretrained", action="store_true")
parser.add_argument(
"--sample_type",
type=str,
default="generalized",
help="sampling approach (generalized or ddpm_noisy)",
)
parser.add_argument(
"--skip_type",
type=str,
default="uniform",
help="skip according to (uniform or quadratic)",
)
parser.add_argument("--timesteps",
type=int,
default=1000,
help="number of steps involved")
parser.add_argument(
"--eta",
type=float,
default=0.0,
help="eta used to control the variances of sigma",
)
parser.add_argument("--sequence", action="store_true")
args = parser.parse_args()
args.log_path = os.path.join(args.exp, "logs", args.doc)
# parse config file
with open(os.path.join("configs", args.config), "r") as f:
config = yaml.safe_load(f)
new_config = dict2namespace(config)
vdl_path = os.path.join(args.exp, "visualdl", args.doc)
if not args.test and not args.sample:
if not args.resume_training:
if os.path.exists(args.log_path):
overwrite = False
if args.ni:
overwrite = True
else:
response = input("Folder already exists. Overwrite? (Y/N)")
if response.upper() == "Y":
overwrite = True
if overwrite:
shutil.rmtree(args.log_path)
shutil.rmtree(vdl_path)
os.makedirs(args.log_path)
if os.path.exists(vdl_path):
shutil.rmtree(vdl_path)
else:
print("Folder exists. Program halted.")
sys.exit(0)
else:
os.makedirs(args.log_path)
with open(os.path.join(args.log_path, "config.yml"), "w") as f:
yaml.dump(new_config, f, default_flow_style=False)
new_config.vdl_logger = vdl.LogWriter(log_dir=vdl_path)
# setup logger
level = getattr(logging, args.verbose.upper(), None)
if not isinstance(level, int):
raise ValueError("level {} not supported".format(args.verbose))
handler1 = logging.StreamHandler()
handler2 = logging.FileHandler(
os.path.join(args.log_path, "stdout.txt"))
formatter = logging.Formatter(
"%(levelname)s - %(filename)s - %(asctime)s - %(message)s")
handler1.setFormatter(formatter)
handler2.setFormatter(formatter)
logger = logging.getLogger()
logger.addHandler(handler1)
logger.addHandler(handler2)
logger.setLevel(level)
else:
level = getattr(logging, args.verbose.upper(), None)
if not isinstance(level, int):
raise ValueError("level {} not supported".format(args.verbose))
handler1 = logging.StreamHandler()
formatter = logging.Formatter(
"%(levelname)s - %(filename)s - %(asctime)s - %(message)s")
handler1.setFormatter(formatter)
logger = logging.getLogger()
logger.addHandler(handler1)
logger.setLevel(level)
if args.sample:
os.makedirs(os.path.join(args.exp, "image_samples"), exist_ok=True)
args.image_folder = os.path.join(args.exp, "image_samples",
args.image_folder)
if not os.path.exists(args.image_folder):
os.makedirs(args.image_folder)
else:
if not (args.fid or args.interpolation):
overwrite = False
if args.ni:
overwrite = True
else:
response = input(
f"Image folder {args.image_folder} already exists. Overwrite? (Y/N)"
)
if response.upper() == "Y":
overwrite = True
if overwrite:
shutil.rmtree(args.image_folder)
os.makedirs(args.image_folder)
else:
print("Output image folder exists. Program halted.")
sys.exit(0)
# add device
device = paddle.get_device()
logging.info("Using device: {}".format(device))
new_config.device = device
# set random seed
paddle.seed(args.seed)
np.random.seed(args.seed)
return args, new_config
# @Time: 2020/7/17 9:25
# @Author: R.Jian
# @Note:
import visualdl
with open("data/loss.txt", "r", encoding="utf8") as fl, open(
"data/score.txt", "r", encoding="utf8") as fs:
out = [(float(l),float(s)) for l,s in zip(fl.readlines(),fs.readlines())]
out = sorted(out,key=lambda x:x[0])
logwriter_l = visualdl.LogWriter("log/loss")
logwriter_s = visualdl.LogWriter("log/score")
for i in range(len(out)):
logwriter_l.add_scalar("show/com",out[i][0],step=i)
logwriter_s.add_scalar("show/com",out[i][1],step=i)
import Class_OS.o1_获得当前工作目录
import linecache # 读取指定行
import os
import visualdl
# 指定路径
path = Class_OS.o1_获得当前工作目录.main()
params_dirname = path + "test02.inference.model"
print("训练后文件夹路径" + params_dirname)
# 参数初始化
place = fluid.CUDAPlace(0)
# place=fluid.CPUPlace()
exe = fluid.Executor(place)
# visualdl
logw = visualdl.LogWriter("g:/log/main_log", sync_cycle=10)
# Scalar-损失指标
with logw.mode('train') as logger:
trainTag = logger.scalar("损失指标")
with logw.mode('test') as logger:
testTag = logger.scalar("损失指标")
# visualDL --logdir g:/log/main_log --port 8080 --host 127.0.0.10
# 加载数据
datatype = 'float32'
with open(path + "data/ocrData.txt", 'rt') as f:
a = f.read()
def dataReader():
path = "../"
params_dirname = path + "test03/test.inference.model"
print("训练后文件夹路径" + params_dirname)
'''
若不放心当前位置，可以查看当前路径
CSY 2020-1-14
import os
print(os.getcwd())
'''
'''
定义VisualDL日志文件保存在vdl_log文件夹中
第一个参数为日志保存路径，
第二个参数为指定多少次写操作后才从内存写入到硬盘日志文件，越大越占用内存，越小则越占用硬盘IO。
'''
logw = visualdl.LogWriter(path + "test03/vdl_log",
sync_cycle=100) #定义保存VisualDL日志文件的路径
# 创建scalar图, mode定义了 line0 显示的名称
with logw.mode('loss') as logger:
line0 = logger.scalar('train')
# 参数初始化
# place = fluid.CUDAPlace(0)
place = fluid.CPUPlace()
exe = fluid.Executor(place)
# 加载数据
# r--只读方式打开;t--文本模式
with open(path + "data/ocrData.txt", 'rt') as f:
a = f.read() #打开标签数据集文件. a是一个list, a[i]为第i个验证码的标签
'''