def test_tda_inference(create_fn=create_vhda):
dataset = create_dummy_dataset()
dataloader = create_dataloader(dataset, batch_size=3)
model = create_fn(dataset)
model.reset_parameters()
batch: BatchData = next(iter(dataloader))
asv = dataset.processor.tensorize_state_vocab("goal_state")
logit, post, prior = model({
"conv_lens": batch.conv_lens,
"sent": batch.sent.value,
"sent_lens": batch.sent.lens1,
"speaker": batch.speaker.value,
"state": batch.state.value,
"state_lens": batch.state.lens1,
"goal": batch.goal.value,
"goal_lens": batch.goal.lens1,
"asv": asv.value,
"asv_lens": asv.lens
})
print(logit, post, prior)
for k, v in logit.items():
assert not utils.has_nan(v), f"nan detected in {k} logit"
for k, gaus in post.items():
assert not utils.has_nan(gaus.mu) and not utils.has_nan(gaus.logvar), \
f"nan detected in {k} posterior distribution"
for k, gaus in prior.items():
assert not utils.has_nan(gaus.mu) and not utils.has_nan(gaus.logvar), \
f"nan detected in {k} prior distribution"
def eval_data():
opt = EvalOptions().parse()
train_dataloader, test_dataloader = create_dataloader(opt)
model = create_model(opt)
if opt.which_epoch is not None:
model = load_network(model, opt)
print("Extracting train set features...")
train_data = extract_features(train_dataloader, model)
print("Extracting test set features...")
test_data = extract_features(test_dataloader, model)
classfiers = load_classifiers(opt)
'''
if opt.num_splits > 0:
train_features, train_labels = train_data
split_ids = np.linspace(0, len(train_labels), opt.num_splits + 1, dtype=np.int)
for i in range(opt.num_splits):
test_mask = np.zeros_like(train_labels, dtype=np.int)
test_mask[split_ids[i]:split_ids[i+1]] = 1
train_mask = 1 - test_mask
train_split_data = (train_features[train_mask], train_labels[train_mask])
test_split_data = (train_features[test_mask], train_labels[test_mask])
print('Running split {:d}...'.format(i+1))
run_classifiers(classfiers, train_split_data, test_split_data)
else:
'''
run_classifiers(classfiers, train_data, test_data)
stats, measures = get_stats(train_data)
for m in measures.keys():
print('{}: '.format(m), measures[m])
def train_data():
opt = TrainOptions().parse()
train_dataloader, test_dataloader = create_dataloader(opt)
model = create_model(opt)
optim = model.create_optimizer(opt)
print('Start training.')
width = len(str(opt.niter))
for it in range(1, opt.niter+1):
it_str = '{:0{width}d}'.format(it, width=width)
tqdm_desc = 'Epoch #{}/{:d}'.format(it_str, opt.niter)
iter_start_time = time.time()
train_pbar = tqdm(train_dataloader, desc=tqdm_desc)
for data_i in train_pbar:
metrics = train_one_step(model, optim, data_i)
train_pbar.set_postfix(metrics)
val_losses = []
val_accs = []
for data_i in test_dataloader:
losses_sum, accs_sum = validate_one_step(model, data_i)
val_losses.append(losses_sum)
val_accs.append(accs_sum)
val_size = len(test_dataloader.dataset)
ave_val_loss = torch.stack(val_losses).sum().data.item() / val_size
ave_val_acc = torch.stack(val_accs).sum().data.item() / val_size
print('test loss: {:.5f}, test accuracy: {:.3%}, time elapsed {:.3f}s'.format(
ave_val_loss, ave_val_acc, time.time() - iter_start_time
))
if it % opt.save_epoch_freq == 0:
save_network(model, it_str, opt)
def test_generator():
dataset = create_dummy_dataset()
dataloader = create_dataloader(dataset, batch_size=3)
model = create_vhda(dataset)
model.reset_parameters()
trainer = TestInferencer(
model=model,
report_every=1,
loss=VHDALoss(vocabs=dataset.processor.vocabs),
writer=tensorboard.SummaryWriter("/tmp/test"),
display_stats={
"loss", "kld", "spkr-acc-turn", "state-acc-turn", "goal-acc-turn"
},
progress_stat="loss",
evaluators=(SpeakerEvaluator(dataset.processor.vocabs.speaker),
DialogStateEvaluator(dataset.processor.vocabs)),
processor=dataset.processor)
for _ in range(20):
trainer(dataloader)
generator = TestGenerator(
model=model,
processor=dataset.processor,
beam_size=2,
evaluators=(BLEUEvaluator(dataset.processor.vocabs),
DistinctEvaluator(dataset.processor.vocabs),
SentLengthEvaluator(dataset.processor.vocabs),
RougeEvaluator(dataset.processor.vocabs),
EmbeddingEvaluator(
vocab=dataset.processor.vocabs.word,
embeds=GloveFormatEmbeddings(
path="tests/data/glove/glove.840B.300d.woz.txt").
preload()), WordEntropyEvaluator(dataset),
StateEntropyEvaluator(dataset)),
display_stats=({
"bleu-smooth7-user1", "bleu-smooth7-user2", "dist-1", "dist-2",
"rouge-l-f1-user1", "sent-len-user1", "emb-greedy"
}),
report_every=1)
samples, stats = generator(dataset.data, 10)
print(samples)
print(stats)
def test_inferencer():
dataset = create_dummy_dataset()
dataloader = create_dataloader(dataset, batch_size=3)
model = create_vhda(dataset)
model.reset_parameters()
looper = TestInferencer(
model=model,
report_every=1,
loss=VHDALoss(vocabs=dataset.processor.vocabs),
writer=tensorboard.SummaryWriter("/tmp/test"),
display_stats={
"loss", "kld", "spkr-acc-turn", "state-acc-turn", "goal-acc-turn"
},
progress_stat="loss",
evaluators=(SpeakerEvaluator(dataset.processor.vocabs.speaker),
DialogStateEvaluator(dataset.processor.vocabs)),
processor=dataset.processor)
stats = None
for _ in range(100):
stats = looper(dataloader)
assert stats is not None
for k, v in stats.items():
assert not utils.has_nan(v), f"{k} item contains NaN"
def dataloaders(self, **kwargs):
dataloaders = {}
for type, data in self.datasets.items():
if "dataset" not in data or "dataloader" not in data:
raise ValueError("dataset info missing for {%type}")
dataset_args = {}
if "transforms" in data:
dataset_args.update(
{"transforms": from_dict(data["transform"])})
else:
dataset_args.update({
"transforms":
kwargs["transforms"][type] if "transforms" in kwargs
and type in kwargs["transforms"] else None
})
if "folds" in kwargs and type in kwargs["folds"]:
dataset_args.update({"fold": kwargs["folds"][type]})
elif "folds" in data:
dataset_args.update({"fold": data["folds"]})
dataset = create_dataset(data["dataset"]["class"], **dataset_args,
**data["dataset"].get("kwargs", {}))
if "sampler" in data:
sampler = create_sampler(data["sampler"]["class"], dataset,
data["sampler"].get("kwargs", {}))
else:
sampler = None
if "collate_fn" in data:
collate = get_collator(data["collate_fn"])
else:
collate = None
dataloader = create_dataloader(
data["dataloader"]["class"], dataset, sampler, collate,
**data["dataloader"].get("kwargs", {}))
dataloaders.update({type: dataloader})
return dataloaders
def test_jda(create_fn=create_vhda, gen_fn=vhda_gen):
dataset = create_dummy_dataset()
dataloader = create_dataloader(
dataset,
batch_size=2,
)
model = create_fn(dataset)
optimizer = op.Adam(p for p in model.parameters() if p.requires_grad)
ce = nn.CrossEntropyLoss(ignore_index=-1, reduction="none")
bce = nn.BCEWithLogitsLoss(reduction="none")
model.reset_parameters()
for eidx in range(300):
model.train()
for i, batch in enumerate(dataloader):
batch: BatchData = batch
optimizer.zero_grad()
model.inference()
w, p = batch.word, batch.speaker
g, g_lens = batch.goal, batch.goal_lens
s, s_lens = batch.turn, batch.turn_lens
sent_lens, conv_lens = batch.sent_lens, batch.conv_lens
batch_size, max_conv_len, max_sent_len = w.size()
w_logit, p_logit, g_logit, s_logit, info = model(batch.to_dict())
w_target = w.masked_fill(~utils.mask(conv_lens).unsqueeze(-1), -1)
w_target = w_target.view(-1, max_sent_len).masked_fill(
~utils.mask(sent_lens.view(-1)), -1
).view(batch_size, max_conv_len, -1)
recon_loss = ce(
w_logit[:, :, :-1].contiguous().view(-1, w_logit.size(-1)),
w_target[:, :, 1:].contiguous().view(-1)
).view(batch_size, max_conv_len, max_sent_len - 1).sum(-1).sum(-1)
goal_loss = bce(
g_logit,
utils.to_dense(g, g_lens, g_logit.size(-1)).float()
)
goal_loss = (goal_loss.masked_fill(~utils.mask(conv_lens)
.unsqueeze(-1).unsqueeze(-1), 0)
.sum(-1).sum(-1).sum(-1))
turn_loss = bce(
s_logit,
utils.to_dense(s, s_lens, s_logit.size(-1)).float()
)
turn_loss = (turn_loss.masked_fill(~utils.mask(conv_lens)
.unsqueeze(-1).unsqueeze(-1), 0)
.sum(-1).sum(-1).sum(-1))
speaker_loss = ce(
p_logit.view(-1, p_logit.size(-1)),
p.masked_fill(~utils.mask(conv_lens), -1).view(-1)
).view(batch_size, max_conv_len).sum(-1)
kld_loss = sum(v for k, v in info.items()
if k in {"sent", "conv", "speaker", "goal", "turn"})
loss = (recon_loss + goal_loss + turn_loss + speaker_loss +
kld_loss * min(0.3, max(0.01, i / 500)))
print(f"[e{eidx + 1}] "
f"loss={loss.mean().item(): 4.4f} "
f"recon={recon_loss.mean().item(): 4.4f} "
f"goal={goal_loss.mean().item(): 4.4f} "
f"turn={turn_loss.mean().item(): 4.4f} "
f"speaker={speaker_loss.mean().item(): 4.4f} "
f"kld={kld_loss.mean().item(): 4.4f}")
loss.mean().backward()
optimizer.step()
model.eval()
model.genconv_post()
batch_gen, info = gen_fn(model)(batch.to_dict())
print("Input: ")
print(f"{dataset.processor.lexicalize(batch[0])}")
print()
print(f"Predicted (prob={info['logprob'][0].exp().item():.4f}): ")
print(f"{dataset.processor.lexicalize(batch_gen[0])}")
model.eval()
model.genconv_post()
for batch in dataloader:
batch_gen, logprobs = gen_fn(model)(batch.to_dict())
for x, y in zip(map(dataset.processor.lexicalize, batch),
map(dataset.processor.lexicalize, batch_gen)):
assert x == y, f"{x}\n!=\n{y}"
def __call__(
self,
data: Optional[Sequence[Dialog]] = None,
num_instances: Optional[int] = None
) -> Tuple[Sequence[Sample], TensorMap]:
if data is None and num_instances is None:
raise ValueError(f"must provide a data source or "
f"number of instances.")
dataloader = None
if data is not None:
dataloader = create_dataloader(dataset=DialogDataset(
data=data, processor=self.processor),
batch_size=self.batch_size,
drop_last=False,
shuffle=False)
if num_instances is None:
num_instances = len(data)
self._num_instances = num_instances
self.on_run_started()
dataloader = (itertools.repeat(None)
if dataloader is None else itertools.cycle(dataloader))
cum_stats = collections.defaultdict(float)
samples = []
for batch in dataloader:
self.model.eval()
if batch is None:
batch_size = min(self.batch_size, num_instances - len(samples))
self.model.genconv_prior()
with torch.no_grad():
pred, info = self.model(
torch.tensor(batch_size).to(self.device),
**self.generate_kwargs())
else:
batch = batch.to(self.device)
batch_size = batch.batch_size
self.global_step += batch_size
batch = self.on_batch_started(batch)
self.model.genconv_post()
with torch.no_grad():
pred, info = self.model(self.prepare_batch(batch),
**self.generate_kwargs())
batch_samples = list(
filter(self.validate_sample, (Sample(*args) for args in zip(
map(self.processor.lexicalize_global, batch),
map(self.processor.lexicalize_global, pred),
info["logprob"]))))
num_res = max(0, len(samples) + len(batch_samples) - num_instances)
if num_res > 0:
batch_samples = random.sample(batch_samples,
num_instances - len(samples))
batch_size = len(batch_samples)
self.global_step += batch_size
stats = self.on_batch_ended(batch_samples)
samples.extend(batch_samples)
for k, v in stats.items():
cum_stats[k] += v * batch_size
if len(samples) >= num_instances:
break
assert len(samples) == num_instances
cum_stats = {k: v / len(samples) for k, v in cum_stats.items()}
return self.on_run_ended(samples, cum_stats)
import sys
from collections import OrderedDict
from options.train_options import TrainOptions
import datasets
from util.iter_counter import IterationCounter
from util.visualizer import Visualizer
from trainers.pix2pix_trainer import Pix2PixTrainer
# parse options
opt = TrainOptions().parse()
# print options to help debugging
print(' '.join(sys.argv))
# load the dataset
dataloader = datasets.create_dataloader(opt)
# create trainer for our model
trainer = Pix2PixTrainer(opt)
# create tool for counting iterations
iter_counter = IterationCounter(opt, len(dataloader))
# create tool for visualization
visualizer = Visualizer(opt)
for epoch in iter_counter.training_epochs():
iter_counter.record_epoch_start(epoch)
for i, data_i in enumerate(dataloader, start=iter_counter.epoch_iter):
iter_counter.record_one_iteration()
def train(
checkpoints_dir=None, exp_name=None, model=None, train_dataset=None, valid_dataset=None, \
use_gpu=True, epoch_total=10, epoch_start=1, load_epoch=None, \
print_steps=500, display_steps=2000, \
save_model_epoch=1, save_latest_steps=50000 \
):
assert checkpoints_dir and exp_name and model and train_dataset
experiment_dir = Path(checkpoints_dir) / exp_name
experiment_dir.mkdir(parents=True, exist_ok=True)
# Prepare for Training
logger = Logger(experiment_dir, only_train=(valid_dataset is None))
device = torch.device("cuda" if use_gpu else "cpu")
model = model(isTrain=True, device=device)
train_dataloader = create_dataloader(train_dataset(isTrain=True), shuffle=True)
valid_dataloader = create_dataloader(valid_dataset(isTrain=False), shuffle=False) if valid_dataset else None
# Copy the config file and print params
timestamp = arrow.utcnow().to("local").format("[YYYY_MMDD] HH'mm'ss")
exp_config = experiment_dir / "config_{}.gin".format(timestamp)
exp_config.write_text(Path(opt.config).read_text())
exp_params = experiment_dir / "params_{}.txt".format(timestamp)
exp_params.write_text(gin.operative_config_str())
if load_epoch:
model.load_networks(experiment_dir, load_epoch)
steps = (epoch_start - 1) * len(train_dataloader)
for epoch in range(epoch_start, epoch_total+1):
# Training
train_losses = {k: [] for k in model.losses}
for data in tqdm(train_dataloader, total=len(train_dataloader), ascii=True):
model.set_input(data)
model.forward()
model.optimize_parameters()
for k, v in model.get_losses().items():
train_losses[k].append(v)
steps += 1
if steps % print_steps == 0:
train_message = ""
for k, v in model.get_losses().items():
logger.add_scalar(k, v, steps, phase="train")
train_message += "{}: {:.4f}, ".format(k, v)
train_message = "[Epoch {:0>3d}, {:0>6d}] train - {}".format(epoch, steps, train_message)
tqdm.write(train_message)
if steps % display_steps == 0:
for tag, image in model.get_visuals().items():
tag_arr = tag.split("_")
logger.add_images("/".join(tag_arr), image, steps)
if steps % save_latest_steps == 0:
model.save_networks(experiment_dir, "latest")
model.update_epoch(epoch)
if epoch % save_model_epoch == 0:
model.save_networks(experiment_dir, epoch)
model.save_networks(experiment_dir, "latest", with_optimizer=True)
# Log training loss
train_message = ""
for k, v in train_losses.items():
train_message += "{}: {:.4f}, ".format(k, np.mean(v))
train_message = "[Epoch {:0>3d}] train - {}".format(epoch, train_message)
# Validation
if valid_dataloader is None:
logger.save_message(train_message)
continue
valid_losses = {k: [] for k in model.losses}
with torch.no_grad():
for data in tqdm(valid_dataloader, total=len(valid_dataloader), ascii=True):
model.set_input(data)
model.forward()
model.evaluate()
for k, v in model.get_losses().items():
valid_losses[k].append(v)
# Log validation loss
valid_message = ""
for k, v in valid_losses.items():
logger.add_scalar(k, np.mean(v), steps, phase="valid")
valid_message += "{}: {:.4f}, ".format(k, np.mean(v))
valid_message = "[Epoch {:0>3d}] valid - {}".format(epoch, valid_message)
logger.save_message(train_message)
logger.save_message(valid_message)
def evaluate(args: EvaluateArugments) -> utils.TensorMap:
model, device = args.model, args.device
logger = logging.getLogger("evaluate")
dataset = datasets.DialogDataset(args.test_data, args.processor)
logger.info("preparing evaluation environment...")
loss = train.create_loss(
model=model,
vocabs=args.processor.vocabs,
enable_kl=True,
kl_mode="kl-mi+"
)
runners = {
"fine": FinegrainedEvaluator(
model=model,
processor=args.processor,
device=args.device,
loss=loss,
evaluators=[
PosteriorEvaluator(),
SpeakerEvaluator(args.processor.vocabs.speaker),
DialogStateEvaluator(args.processor.vocabs)
],
run_end_report=False
),
"dial": DialogGenerationEvaluator(
model=model,
processor=args.processor,
batch_size=args.batch_size,
device=args.device,
beam_size=args.beam_size,
max_conv_len=args.max_conv_len,
max_sent_len=args.max_sent_len,
evaluators=[
BLEUEvaluator(args.processor.vocabs),
DistinctEvaluator(args.processor.vocabs),
EmbeddingEvaluator(
vocab=args.processor.vocabs.word,
embeds=dict(
glove=GloveFormatEmbeddings,
hdf5=HDF5Embeddings,
bin=BinaryEmbeddings
)[args.embed_type](args.embed_path).preload()
),
SentLengthEvaluator(args.processor.vocabs),
RougeEvaluator(args.processor.vocabs),
DialogLengthEvaluator(),
WordEntropyEvaluator(
datasets.DialogDataset(args.train_data, args.processor)
),
LanguageNoveltyEvaluator(
datasets.DialogDataset(args.train_data, args.processor)
),
StateEntropyEvaluator(
datasets.DialogDataset(args.train_data, args.processor)
),
StateCountEvaluator(args.processor.vocabs),
DistinctStateEvaluator(args.processor.vocabs),
StateNoveltyEvaluator(
datasets.DialogDataset(args.train_data, args.processor)
)
],
run_end_report=False
),
# missing: 1-turn and 3-turn generation
}
logger.info("commencing evaluation...")
eval_stats = dict()
model.eval()
for name, runner in runners.items():
if isinstance(runner, Inferencer):
stats = runner(datasets.create_dataloader(
dataset=dataset,
batch_size=args.batch_size,
drop_last=False,
shuffle=False
))
elif isinstance(runner, Generator):
stats = runner(dataset.data)[1]
else:
raise TypeError(f"unsupported runner type: {type(runner)}")
eval_stats.update({k: v.detach().cpu().item()
for k, v in stats.items()})
logger.info(f"evaluation summary: {pprint.pformat(eval_stats)}")
return eval_stats
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', type=str, default='configs/test.yaml')
parser.add_argument('--distributed', type=int, default=0)
parser.add_argument('--exam_id', type=str, default='')
parser.add_argument('--ckpt_path', type=str, default='')
parser.add_argument('--dataset', type=str, default='')
parser.add_argument('--compress', type=str, default='')
parser.add_argument('--constract', type=bool, default=False)
parser.add_argument('--debug', action='store_true', default=False)
args = parser.parse_args()
local_config = OmegaConf.load(args.config)
for k, v in local_config.items():
if args.dataset != '':
if k == 'dataset':
v["name"] = args.dataset
if args.compress != '':
v[args.dataset]["compressions"] = args.compress
setattr(args, k, v)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
if args.exam_id:
ckpt_path = glob(f'wandb/*{args.exam_id}/ckpts/model_best.pth.tar')
if len(ckpt_path) >= 1:
args.ckpt_path = ckpt_path[0]
if args.ckpt_path:
args.output_dir = os.path.dirname(args.ckpt_path)
os.environ['TORCH_HOME'] = args.torch_home
model, args = load_model(args)
model = model.to(device)
model.eval()
test_dataloader = datasets.create_dataloader(args, split='test')
y_trues = []
y_preds = []
acces = []
img_paths = []
for i, datas in enumerate(tqdm(test_dataloader)):
with torch.no_grad():
images = datas[0].to(device)
targets = datas[1].float().numpy()
y_trues.extend(targets)
prob = model_forward(args, images, model)
prediction = (prob >= args.test.threshold).astype(float)
y_preds.extend(prob)
acces.extend(targets == prediction)
if args.test.record_results:
img_paths.extend(datas[2])
acc = np.mean(acces)
fpr, tpr, thresholds = metrics.roc_curve(y_trues, y_preds, pos_label=1)
AUC = metrics.auc(fpr, tpr)
eer = brentq(lambda x: 1. - x - interp1d(fpr, tpr)(x), 0., 1.)
thresh = interp1d(fpr, thresholds)(eer)
print(f'#Total# ACC:{acc:.5f} AUC:{AUC:.5f} EER:{100*eer:.2f}(Thresh:{thresh:.3f})')
preds = np.array(y_preds) >= args.test.threshold
pred_fake_nums = np.sum(preds)
pred_real_nums = len(preds) - pred_fake_nums
print(f"pred dataset:{args.dataset.name},pred id: {args.exam_id},compress:{args.compress}")
print(f'pred real nums:{pred_real_nums} pred fake nums:{pred_fake_nums}')
if args.test.record_results:
if args.exam_id is not None:
task_id = args.exam_id
filename = glob(f'wandb/*{task_id}/preds_{args.dataset.name}.log')
else:
task_id = args.model.params.model_path.split('/')[1]
filename = f'logs/test/{task_id}_preds_{args.dataset.name}.log'
save_test_results(y_trues, y_preds, img_paths, filename=filename)
def main():
parser = argparse.ArgumentParser(description='Test Super Resolution Models')
parser.add_argument('-opt', type=str, required=True, help='Path to options JSON file.')
opt = option.parse(parser.parse_args().opt)
opt = option.dict_to_nonedict(opt)
# make sure the CUDA_VISIBLE_DEVICES is set before import torch.
from utils import util
from solvers import create_solver
from datasets import create_dataloader
from datasets import create_dataset
# initial configure
scale = opt['scale']
degrad = opt['degradation']
network_opt = opt['networks']
model_name = network_opt['which_model'].upper()
if opt['self_ensemble']: model_name += 'plus'
# create test dataloader
bm_names =[]
test_loaders = []
percent10 = True
for _, dataset_opt in sorted(opt['datasets'].items()):
test_set = create_dataset(dataset_opt)
test_loader = create_dataloader(test_set, dataset_opt)
test_loaders.append(test_loader)
print('===> Test Dataset: [%s] Number of images: [%d]' % (test_set.name(), len(test_set)))
bm_names.append(test_set.name())
# create solver (and load model)
solver = create_solver(opt)
# Test phase
print('===> Start Test')
print("==================================================")
print("Method: %s || Scale: %d || Degradation: %s"%(model_name, scale, degrad))
for bm, test_loader in zip(bm_names, test_loaders):
print("Test set : [%s]"%bm)
sr_list = []
path_list = []
total_psnr = []
total_ssim = []
total_time = []
need_HR = False if test_loader.dataset.__class__.__name__.find('LRHR') < 0 else True
for iter, batch in enumerate(test_loader):
solver.feed_data(batch, need_HR=need_HR)
# calculate forward time
t0 = time.time()
solver.test()
t1 = time.time()
total_time.append((t1 - t0))
visuals = solver.get_current_visual(need_HR=need_HR)
sr_list.append(visuals['SR'])
# calculate PSNR/SSIM metrics on Python
# 这里仅支持batch size = 1的情况!!!
if need_HR:
psnr, ssim = util.calc_metrics(visuals['SR'], visuals['HR'], crop_border=scale)
total_psnr.append(psnr)
total_ssim.append(ssim)
path_list.append(os.path.basename(batch['HR_path'][0]).replace('HR', model_name))
print("[%d/%d] %s || PSNR(dB)/SSIM: %.2f/%.4f || Timer: %.4f sec ." % (iter+1, len(test_loader),
os.path.basename(batch['LR_path'][0]),
psnr, ssim,
(t1 - t0)))
else:
file_dir = batch['LR_path'][0].split('/')[-2]
path_list.append(os.path.join(file_dir, os.path.basename(batch['LR_path'][0])))
print("[%d/%d] %s || Timer: %.4f sec ." % (iter + 1, len(test_loader),
os.path.join(file_dir, os.path.basename(batch['LR_path'][0])),
(t1 - t0)))
if need_HR:
print("---- Average PSNR(dB) /SSIM /Speed(s) for [%s] ----" % bm)
print("PSNR: %.2f SSIM: %.4f Speed: %.4f" % (sum(total_psnr)/len(total_psnr),
sum(total_ssim)/len(total_ssim),
sum(total_time)/len(total_time)))
else:
print("---- Average Speed(s) for [%s] is %.4f sec ----" % (bm,
sum(total_time)/len(total_time)))
if need_HR:
save_img_path = os.path.join('../submit/SR/'+degrad, model_name, bm, "x%d"%scale)
else:
save_img_path = os.path.join('../submit/')
print("===> Saving SR images of [%s]... Save Path: [%s]\n" % (bm, save_img_path))
middle_name = 'h_Res' if percent10 else 'h_Sub25_Res'
filter_idx = -1 if percent10 else -7
if not os.path.exists(save_img_path): os.makedirs(save_img_path)
for img, name in zip(sr_list, path_list):
store_path = os.path.join(save_img_path, name)
base_dir = os.path.dirname(store_path)[:filter_idx] + middle_name
if not os.path.exists(base_dir): os.makedirs(base_dir)
store_path = os.path.join(base_dir, os.path.basename(name))
print('write into {}.'.format(store_path))
imageio.imwrite(store_path, img)
percent10 = False
print("==================================================")
print("===> Finished !")
def main():
parser = argparse.ArgumentParser(
description='Train Super Resolution Models')
parser.add_argument('-opt',
type=str,
required=True,
help='Path to options JSON file.')
opt = option.parse(parser.parse_args().opt)
# make sure the CUDA_VISIBLE_DEVICES is set before import torch.
import torch
from utils import util
from solvers import create_solver
from datasets import create_dataloader
from datasets import create_dataset
from tensorboardX import SummaryWriter
# random seed
seed = opt['solver']['manual_seed']
if seed is None: seed = random.randint(1, 10000)
print("===> Random Seed: [%d]" % seed)
random.seed(seed)
torch.manual_seed(seed)
# create train and val dataloader
for phase, dataset_opt in sorted(opt['datasets'].items()):
if phase == 'train':
train_set = create_dataset(dataset_opt)
train_loader = create_dataloader(train_set, dataset_opt)
print('===> Train Dataset: %s Number of images: [%d]' %
(train_set.name(), len(train_set)))
if train_loader is None:
raise ValueError("[Error] The training data does not exist")
elif phase == 'val':
val_set = create_dataset(dataset_opt)
val_loader = create_dataloader(val_set, dataset_opt)
print('===> Val Dataset: %s Number of images: [%d]' %
(val_set.name(), len(val_set)))
else:
raise NotImplementedError(
"[Error] Dataset phase [%s] in *.json is not recognized." %
phase)
solver = create_solver(opt)
scale = opt['scale']
model_name = opt['networks']['which_model'].upper()
print('===> Start Train')
print("==================================================")
solver_log = solver.get_current_log()
NUM_EPOCH = int(opt['solver']['num_epochs'])
start_epoch = solver_log['epoch']
print("Method: %s || Scale: %d || Epoch Range: (%d ~ %d)" %
(model_name, scale, start_epoch, NUM_EPOCH))
writer = SummaryWriter(os.path.join(opt['path']['exp_root'], 'tbx_log/'))
for epoch in range(start_epoch, NUM_EPOCH + 1):
print('\n===> Training Epoch: [%d/%d]... Learning Rate: %.9f' %
(epoch, NUM_EPOCH, solver.get_current_learning_rate()))
# Initialization
solver_log['epoch'] = epoch
# Train model
train_loss_list = []
with tqdm(total=len(train_loader),
desc='Epoch: [%d/%d]' % (epoch, NUM_EPOCH),
miniters=1) as t:
for iter, batch in enumerate(train_loader):
solver.feed_data(batch)
iter_loss = solver.train_step()
batch_size = batch['LR'].size(0)
train_loss_list.append(iter_loss * batch_size)
t.set_postfix_str("Batch Loss: %.4f" % iter_loss)
t.update()
solver_log['records']['train_loss'].append(
sum(train_loss_list) / len(train_set))
solver_log['records']['lr'].append(solver.get_current_learning_rate())
writer.add_scalar('Train/lr', solver.get_current_learning_rate(),
epoch)
writer.add_scalar('Train/Loss',
sum(train_loss_list) / len(train_set), epoch)
print('\nEpoch: [%d/%d] Avg Train Loss: %.6f' %
(epoch, NUM_EPOCH, sum(train_loss_list) / len(train_set)))
print('===> Validating...', )
psnr_list = []
ssim_list = []
val_loss_list = []
with tqdm(total=len(val_loader),
desc='Epoch: [%d/%d]' % (epoch, NUM_EPOCH),
miniters=1) as t:
for iter, batch in enumerate(val_loader):
solver.feed_data(batch)
iter_loss = solver.test()
val_loss_list.append(iter_loss)
# calculate evaluation metrics
visuals = solver.get_current_visual()
psnr, ssim = util.calc_metrics(visuals['SR'],
visuals['HR'],
crop_border=scale)
psnr_list.append(psnr)
ssim_list.append(ssim)
if opt["save_image"]:
solver.save_current_visual(epoch, iter)
t.set_postfix_str("Batch Loss: %.4f" % iter_loss)
t.update()
solver_log['records']['val_loss'].append(
sum(val_loss_list) / len(val_loss_list))
solver_log['records']['psnr'].append(sum(psnr_list) / len(psnr_list))
solver_log['records']['ssim'].append(sum(ssim_list) / len(ssim_list))
# record the best epoch
epoch_is_best = False
if solver_log['best_pred'] < (sum(psnr_list) / len(psnr_list)):
solver_log['best_pred'] = (sum(psnr_list) / len(psnr_list))
epoch_is_best = True
solver_log['best_epoch'] = epoch
print(
"[%s] PSNR: %.2f SSIM: %.4f Loss: %.6f Best PSNR: %.2f in Epoch: [%d]"
%
(val_set.name(), sum(psnr_list) / len(psnr_list), sum(ssim_list) /
len(ssim_list), sum(val_loss_list) / len(val_loss_list),
solver_log['best_pred'], solver_log['best_epoch']))
writer.add_scalar('Val/PSNR', sum(psnr_list) / len(psnr_list), epoch)
writer.add_scalar('Val/Loss',
sum(val_loss_list) / len(val_loss_list), epoch)
writer.add_scalar('Val/SSIM', sum(ssim_list) / len(ssim_list), epoch)
solver.set_current_log(solver_log)
solver.save_checkpoint(epoch, epoch_is_best)
solver.save_current_log()
# update lr
solver.update_learning_rate(epoch)
print('===> Finished !')
writer.close()
def main(args=None):
args = utils.parse_args(create_parser(), args)
if args.logging_config is not None:
logging.config.dictConfig(utils.load_yaml(args.logging_config))
save_dir = pathlib.Path(args.save_dir)
if (not args.overwrite and save_dir.exists()
and utils.has_element(save_dir.glob("*.json"))):
raise FileExistsError(f"save directory ({save_dir}) is not empty")
shell = utils.ShellUtils()
shell.mkdir(save_dir, silent=True)
logger = logging.getLogger("interpolate")
data_dir = pathlib.Path(args.data_dir)
data = {
split: list(
map(Dialog.from_json,
utils.load_json(data_dir.joinpath(f"{split}.json"))))
for split in set(args.splits)
}
processor: DialogProcessor = utils.load_pickle(args.processor_path)
logger.info("preparing model...")
torchmodels.register_packages(models)
model_cls = torchmodels.create_model_cls(models, args.model_path)
model: models.AbstractTDA = model_cls(processor.vocabs)
model.reset_parameters()
model.load_state_dict(torch.load(args.ckpt_path))
device = torch.device("cpu")
if args.gpu is not None:
device = torch.device(f"cuda:{args.gpu}")
model = model.to(device)
samples = (sample_data(data,
args.anchor1), sample_data(data, args.anchor2))
formatter = utils.DialogTableFormatter()
logger.info(f"first sample: \n{formatter.format(samples[0])}")
logger.info(f"second sample: \n{formatter.format(samples[1])}")
logger.info("preparing environment...")
dataloader = datasets.create_dataloader(dataset=datasets.DialogDataset(
data=samples, processor=processor),
batch_size=1,
shuffle=False,
pin_memory=False)
inferencer = InterpolateInferencer(model=model,
processor=processor,
device=device)
logger.info("interpolating...")
with torch.no_grad():
zconv_a, zconv_b = inferencer.encode(dataloader)
zconv = torch.stack([
zconv_a + (zconv_b - zconv_a) / args.steps * i
for i in range(args.steps + 1)
])
gen_samples = inferencer.generate(td.DataLoader(zconv, shuffle=False))
# use original data points for two extremes
samples = [samples[0]] + list(gen_samples[1:-1]) + [samples[1]]
logger.info("interpolation results: ")
for i, sample in enumerate(samples):
logger.info(f"interpolation step {i / args.steps:.2%}: \n"
f"{formatter.format(sample)}")
logger.info("saving results...")
json_dir = save_dir.joinpath("json")
json_dir.mkdir(exist_ok=True)
for i, sample in enumerate(samples, 1):
utils.save_json(sample.to_json(), json_dir.joinpath(f"{i:02d}.json"))
tbl_dir = save_dir.joinpath("table")
tbl_dir.mkdir(exist_ok=True)
for i, sample in enumerate(samples, 1):
utils.save_lines([formatter.format(sample)],
tbl_dir.joinpath(f"{i:02d}.txt"))
ltx_dir = save_dir.joinpath("latex")
ltx_dir.mkdir(exist_ok=True)
ltx_formatter = utils.DialogICMLLatexFormatter()
for i, sample in enumerate(samples, 1):
utils.save_lines([ltx_formatter.format(sample)],
ltx_dir.joinpath(f"{i:02d}.tex"))
logger.info("done!")
def train(args: TrainArguments) -> Record:
model, device = args.model, args.device
save_dir = args.save_dir
shell = utils.ShellUtils()
shell.mkdir(save_dir, silent=True)
utils.save_json(args.to_json(), str(save_dir.joinpath("args.json")))
logger = logging.getLogger("train")
processor = args.processor
vocabs: datasets.VocabSet = processor.vocabs
train_dataset = datasets.DialogDataset(
data=args.train_data,
processor=processor
)
valid_dataset = datasets.DialogDataset(
data=args.valid_data,
processor=processor
)
logger.info("preparing training environment...")
loss = create_loss(
model=model,
vocabs=vocabs,
kld_weight=args.kld_schedule,
enable_kl=not args.disable_kl,
kl_mode=args.kl_mode
)
if args.optimizer == "adam":
op_cls = op.Adam
else:
raise ValueError(f"unsupported optimizer: {args.optimizer}")
fval_cls = FinegrainedValidator
fval_kwg = dict(
model=model,
processor=processor,
device=device,
evaluators=list(filter(None, (
SpeakerEvaluator(vocabs.speaker),
DialogStateEvaluator(vocabs),
PosteriorEvaluator()
))),
report_every=None,
run_end_report=False,
progress_stat="loss",
loss=loss
)
if isinstance(model, models.VHDA):
@dataclass
class VHDAValidator(VHDAInferencer, fval_cls):
pass
fval_cls = VHDAValidator
fval_kwg.update(dict(
sample_scale=1.0
))
fval = fval_cls(**fval_kwg)
gval_cls = GenerativeValidator
gval_kwg = dict(
model=model,
processor=processor,
batch_size=args.valid_batch_size,
device=device,
evaluators=list(filter(None, [
DistinctEvaluator(vocabs),
SentLengthEvaluator(vocabs),
RougeEvaluator(vocabs),
DialogLengthEvaluator(),
WordEntropyEvaluator(train_dataset)
])),
report_every=None,
run_end_report=False,
beam_size=args.beam_size,
max_sent_len=args.max_gen_len
)
gval = gval_cls(**gval_kwg)
trainer_cls = Trainer
trainer_kwargs = dict(
model=model,
processor=processor,
device=device,
writer=torch.utils.tensorboard.SummaryWriter(
log_dir=str(args.save_dir)
),
evaluators=list(filter(None, (
SpeakerEvaluator(vocabs.speaker),
DialogStateEvaluator(vocabs)
))),
progress_stat="loss",
display_stats={"loss", "kld", "goal-acc-turn-user",
"rouge-l-f1", "conv-mi", "nll", "conv-len"},
report_every=args.report_every,
stats_formatter=utils.StatsFormatter(num_cols=3),
dialog_formatter=utils.DialogTableFormatter(
max_col_len=50
),
loss=loss,
optimizer_cls=functools.partial(
op_cls,
lr=args.learning_rate
),
grad_clip=args.gradient_clip,
l2norm=args.l2norm_weight,
save_dir=pathlib.Path(args.save_dir),
num_epochs=args.num_epochs,
fin_valid=fval,
gen_valid=gval,
validate_every=args.validate_every,
early_stop=args.early_stop,
early_stop_criterion=args.early_stop_criterion,
early_stop_patience=args.early_stop_patience,
save_every=args.save_every
)
if isinstance(model, models.VHDA):
@dataclass
class VHDATrainer(VHDAInferencer, trainer_cls):
pass
trainer_cls = VHDATrainer
trainer_kwargs.update(dict(
dropout_scale=args.dropout_schedule
))
trainer = trainer_cls(**trainer_kwargs)
train_dataloader = datasets.create_dataloader(
train_dataset,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
drop_last=False
)
valid_dataloader = datasets.create_dataloader(
valid_dataset,
batch_size=args.valid_batch_size,
shuffle=False,
pin_memory=True,
drop_last=False
)
logger.info("commencing training...")
record = trainer.train(train_dataloader, valid_dataloader)
logger.info(f"final summary: {pprint.pformat(record.to_short_json())}")
logger.info("done!")
return record
