def __init__(self, config):
self.config = config
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
self.generator = Generator(config).to(self.device)
self.discriminator = Discriminator(config).to(self.device)
self.loss_func = nn.BCELoss()
self.generator_optimizer = optim.Adam(
params=self.generator.parameters(),
lr=config["generator_learning_rate"],
betas=config["generator_betas"])
self.discriminator_optimizer = optim.Adam(
params=self.discriminator.parameters(),
lr=config["discriminator_learning_rate"],
betas=config["discriminator_betas"])
self.true_labels = torch.ones(config["batch_size"],
dtype=torch.float32,
device=self.device)
self.fake_labels = torch.zeros(config["batch_size"],
dtype=torch.float32,
device=self.device)
def get_network(LEARNING_RATE: float, device: str):
G = Generator().to(device)
D = Discriminator().to(device)
criterion = nn.BCELoss()
d_optimizer = optim.Adam(D.parameters(), lr=LEARNING_RATE)
g_optimizer = optim.Adam(G.parameters(), lr=LEARNING_RATE)
return G, D, criterion, d_optimizer, g_optimizer
def __init__(self, encoder_kernels, decoder_kernels, dis_kernels, trainning=True):
self.encoder_kernels = encoder_kernels
self.decoder_kernels = decoder_kernels
self.dis_kernels = dis_kernels
self.training = trainning
self.gen_factor = Generator_Unet(name='gen', encoder_kernels=self.encoder_kernels,
decoder_kernels=self.decoder_kernels)
self.dis_factor = Discriminator(name='dis', kernels=self.dis_kernels)
self.batch_size = 4
self.image_size = 256
self.train_data = Dateset(batch_size=self.batch_size, image_size=self.image_size, data_path='./data/colorize/train.txt')
self.test_data = Dateset(batch_size=self.batch_size * 2, image_size=self.image_size, data_path='./data/colorize/test.txt')
def __init__(self):
self.generator = Generator()
self.generator.to(Params.Device)
self.discriminator = Discriminator()
self.discriminator.to(Params.Device)
self.loss_fn = nn.BCELoss()
self.optimizer_g = torch.optim.Adam(self.generator.parameters(), Params.LearningRateG, betas=(Params.Beta, 0.999))
self.optimizer_d = torch.optim.Adam(self.discriminator.parameters(), Params.LearningRateD, betas=(Params.Beta, 0.999))
self.exemplar_latent_vectors = torch.randn( (64, Params.LatentVectorSize), device=Params.Device )
def _initialise_networks(self):
self.generator = Generator(final_size=self.final_size)
self.generator.generate_network()
self.g_optimizer = Adam(self.generator.parameters(), lr=0.001, betas=(0, 0.99))
self.discriminator = Discriminator(final_size=self.final_size)
self.discriminator.generate_network()
self.d_optimizer = Adam(self.discriminator.parameters(), lr=0.001, betas=(0, 0.99))
self.num_channels = min(self.generator.num_channels,
self.generator.max_channels)
self.upsample = [Upsample(scale_factor=2**i)
for i in reversed(range(self.generator.num_blocks))]
def build_train(self,
inputs=None,
origin_domains=None,
target_domains=None):
# origin domains
if origin_domains is None:
self.origin_domains = tf.placeholder(tf.int64,
self.domain_shape,
name='OriginDomain')
else:
self.origin_domains = tf.identity(origin_domains,
name='OriginDomain')
self.origin_domains.set_shape(self.domain_shape)
# build model
self.build_model(inputs, target_domains)
# reconstruction
self.reconstructs = self.generator(self.outputs,
self.origin_domains,
reuse=True)
# discrimination
self.discriminator = Discriminator('Discriminator', self.config)
critic_logit, domain_logit = self.discriminator(self.outputs,
reuse=None)
# build loss
self.build_g_loss(self.inputs, self.outputs, self.reconstructs,
self.target_domains, critic_logit, domain_logit)
self.build_d_loss(self.inputs, self.origin_domains, self.outputs,
critic_logit)
def init_net(depth, dropout, window, cgan):
input_shape = (1 if not cgan else 2, window)
# Create the 3 networks
gen = Generator(depth, dropout, verbose=0)
discr = Discriminator(depth, dropout, input_shape,
verbose=0) if not cgan else ConditionalDiscriminator(
depth, dropout, input_shape, verbose=0)
ae = AutoEncoder(depth, dropout, verbose=0)
# Put them on cuda if available
if torch.cuda.is_available():
gen.cuda()
discr.cuda()
ae.cuda()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("Using : " + str(device))
print("Network initialized\n")
return gen, discr, ae, device
def _init_network(self):
def init_weights(m):
if type(m) == nn.Conv2d:
nn.init.normal_(m.weight.data, 0.0, 0.02)
if type(m) == nn.BatchNorm2d:
nn.init.normal_(m.weight.data, 0.0, 0.02)
nn.init.constant_(m.bias.data, 0.0)
generator = Generator(DCGAN_Config).to(self.device)
discriminator = Discriminator(DCGAN_Config).to(self.device)
if self.device.type =='cuda' and DCGAN_Config['ngpu']>1:
generator = nn.DataParallel(generator, list(range(DCGAN_Config['ngpu'])))
discriminator = nn.DataParallel(discriminator, list(range(DCGAN_Config['ngpu'])))
generator.apply(init_weights)
discriminator.apply(init_weights)
print(generator)
print(discriminator)
return generator, discriminator
def __init__(self, opt):
super(MUNIT, self).__init__()
# generators and discriminators
self.gen_a = Generator(opt.ngf, opt.style_dim, opt.mlp_dim)
self.gen_b = Generator(opt.ngf, opt.style_dim, opt.mlp_dim)
self.dis_a = Discriminator(opt.ndf)
self.dis_b = Discriminator(opt.ndf)
#random style code
self.s_a = torch.randn(opt.display_size,
opt.style_dim,
1,
1,
requires_grad=True).cuda()
self.s_b = torch.randn(opt.display_size,
opt.style_dim,
1,
1,
requires_grad=True).cuda()
#optimizers
dis_params = list(self.dis_a.parameters()) + list(
self.dis_b.parameters())
gen_params = list(self.gen_a.parameters()) + list(
self.gen_b.parameters())
self.dis_opt = torch.optim.Adam(dis_params,
lr=opt.lr,
beta=opt.beta1,
weight_delay=opt.weight_delay)
self.gen_opt = torch.optim.Adam(gen_params,
lr=opt.lr,
beta=opt.beta1,
weight_delay=opt.weight_delay)
# nerwork weight initialization
self.apply(weight_init('kaiming'))
self.dis_a.apply(weight_init('gaussian'))
self.dis_b.apply(weight_init('gaussian'))
def build_model(self, inputs=None):
# inputs
if inputs is None:
self.inputs = tf.placeholder(self.dtype, self.input_shape, name='Input')
else:
self.inputs = tf.identity(inputs, name='Input')
self.inputs.set_shape(self.input_shape)
# forward pass
self.discriminator = Discriminator('Discriminator', self.config)
self.embeddings, self.outputs = self.discriminator(self.inputs, reuse=None)
# embeddings
self.embeddings = tf.identity(self.embeddings, name='Embedding')
# outputs
self.outputs = tf.identity(self.outputs, name='Output')
# all the saver variables
self.svars = self.discriminator.svars
# all the restore variables
self.rvars = self.discriminator.rvars
# return outputs
return self.outputs
def main():
G = Generator(args.dim_disc + args.dim_cont)
D = Discriminator()
if os.path.isfile(args.model):
model = torch.load(args.model)
G.load_state_dict(model[0])
D.load_state_dict(model[1])
if use_cuda:
G.cuda()
D.cuda()
if args.mode == "train":
G, D = train(G, D)
if args.model:
torch.save([G.state_dict(), D.state_dict()],
args.model,
pickle_protocol=4)
elif args.mode == "gen":
gen(G)
def __init__(self, args):
if args.model == 'tag2pix':
from network import Generator
elif args.model == 'senet':
from model.GD_senet import Generator
elif args.model == 'resnext':
from model.GD_resnext import Generator
elif args.model == 'catconv':
from model.GD_cat_conv import Generator
elif args.model == 'catall':
from model.GD_cat_all import Generator
elif args.model == 'adain':
from model.GD_adain import Generator
elif args.model == 'seadain':
from model.GD_seadain import Generator
else:
raise Exception('invalid model name: {}'.format(args.model))
self.args = args
self.epoch = args.epoch
self.batch_size = args.batch_size
self.gpu_mode = not args.cpu
self.input_size = args.input_size
self.color_revert = ColorSpace2RGB(args.color_space)
self.layers = args.layers
[self.cit_weight, self.cvt_weight] = args.cit_cvt_weight
self.load_dump = (args.load is not "")
self.load_path = Path(args.load)
self.l1_lambda = args.l1_lambda
self.guide_beta = args.guide_beta
self.adv_lambda = args.adv_lambda
self.save_freq = args.save_freq
self.two_step_epoch = args.two_step_epoch
self.brightness_epoch = args.brightness_epoch
self.save_all_epoch = args.save_all_epoch
self.iv_dict, self.cv_dict, self.id_to_name = get_tag_dict(
args.tag_dump)
cvt_class_num = len(self.cv_dict.keys())
cit_class_num = len(self.iv_dict.keys())
self.class_num = cvt_class_num + cit_class_num
self.start_epoch = 1
#### load dataset
if not args.test:
self.train_data_loader, self.test_data_loader = get_dataset(args)
self.result_path = Path(args.result_dir) / time.strftime(
'%y%m%d-%H%M%S', time.localtime())
if not self.result_path.exists():
self.result_path.mkdir()
self.test_images = self.get_test_data(self.test_data_loader,
args.test_image_count)
else:
self.test_data_loader = get_dataset(args)
self.result_path = Path(args.result_dir)
##### initialize network
self.net_opt = {
'guide': not args.no_guide,
'relu': args.use_relu,
'bn': not args.no_bn,
'cit': not args.no_cit
}
if self.net_opt['cit']:
self.Pretrain_ResNeXT = se_resnext_half(
dump_path=args.pretrain_dump,
num_classes=cit_class_num,
input_channels=1)
else:
self.Pretrain_ResNeXT = nn.Sequential()
self.G = Generator(input_size=args.input_size,
layers=args.layers,
cv_class_num=cvt_class_num,
iv_class_num=cit_class_num,
net_opt=self.net_opt)
self.D = Discriminator(input_dim=3,
output_dim=1,
input_size=self.input_size,
cv_class_num=cvt_class_num,
iv_class_num=cit_class_num)
for param in self.Pretrain_ResNeXT.parameters():
param.requires_grad = False
if args.test:
for param in self.G.parameters():
param.requires_grad = False
for param in self.D.parameters():
param.requires_grad = False
self.Pretrain_ResNeXT = nn.DataParallel(self.Pretrain_ResNeXT)
self.G = nn.DataParallel(self.G)
self.D = nn.DataParallel(self.D)
self.G_optimizer = optim.Adam(self.G.parameters(),
lr=args.lrG,
betas=(args.beta1, args.beta2))
self.D_optimizer = optim.Adam(self.D.parameters(),
lr=args.lrD,
betas=(args.beta1, args.beta2))
self.BCE_loss = nn.BCELoss()
self.CE_loss = nn.CrossEntropyLoss()
self.L1Loss = nn.L1Loss()
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
print("gpu mode: ", self.gpu_mode)
print("device: ", self.device)
print(torch.cuda.device_count(), "GPUS!")
if self.gpu_mode:
self.Pretrain_ResNeXT.to(self.device)
self.G.to(self.device)
self.D.to(self.device)
self.BCE_loss.to(self.device)
self.CE_loss.to(self.device)
self.L1Loss.to(self.device)
class Solver(object):
def __init__(self, configuration):
self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# retrieve configuration variables
self.data_path = configuration.data_path
self.crop_size = configuration.crop_size
self.final_size = configuration.final_size
self.batch_size = configuration.batch_size
self.alternating_step = configuration.alternating_step
self.ncritic = configuration.ncritic
self.lambda_gp = configuration.lambda_gp
self.debug_step = configuration.debug_step
self.save_step = configuration.save_step
self.max_checkpoints = configuration.max_checkpoints
self.log_step = configuration.log_step
# self.tflogger = Logger(configuration.log_dir)
## directoriess
self.train_dir = configuration.train_dir
self.img_dir = configuration.img_dir
self.models_dir = configuration.models_dir
## variables
self.eps_drift = 0.001
self.resume_training = configuration.resume_training
self._initialise_networks()
def _initialise_networks(self):
self.generator = Generator(final_size=self.final_size)
self.generator.generate_network()
self.g_optimizer = Adam(self.generator.parameters(), lr=0.001, betas=(0, 0.99))
self.discriminator = Discriminator(final_size=self.final_size)
self.discriminator.generate_network()
self.d_optimizer = Adam(self.discriminator.parameters(), lr=0.001, betas=(0, 0.99))
self.num_channels = min(self.generator.num_channels,
self.generator.max_channels)
self.upsample = [Upsample(scale_factor=2**i)
for i in reversed(range(self.generator.num_blocks))]
def print_debugging_images(self, generator, latent_vectors, shape, index,
alpha, iteration):
with torch.no_grad():
columns = []
for i in range(shape[0]):
row = []
for j in range(shape[1]):
img_ij = generator(latent_vectors[i * shape[1] +
j].unsqueeze_(0),
index, alpha)
img_ij = self.upsample[index](img_ij)
row.append(img_ij)
columns.append(torch.cat(row, dim=3))
debugging_image = torch.cat(columns, dim=2)
# denorm
debugging_image = (debugging_image + 1) / 2
debugging_image.clamp_(0, 1)
save_image(debugging_image.data,
os.path.join(self.img_dir, "debug_{}_{}.png".format(index,
iteration)))
def save_trained_networks(self, block_index, phase, step):
models_file = os.path.join(self.models_dir, "models.json")
if os.path.isfile(models_file):
with open(models_file, 'r') as file:
models_config = json.load(file)
else:
models_config = json.loads('{ "checkpoints": [] }')
generator_save_name = "generator_{}_{}_{}.pth".format(
block_index, phase, step
)
torch.save(self.generator.state_dict(),
os.path.join(self.models_dir, generator_save_name))
discriminator_save_name = "discriminator_{}_{}_{}.pth".format(
block_index, phase, step
)
torch.save(self.discriminator.state_dict(),
os.path.join(self.models_dir, discriminator_save_name))
models_config["checkpoints"].append(OrderedDict({
"block_index": block_index,
"phase": phase,
"step": step,
"generator": generator_save_name,
"discriminator": discriminator_save_name
}))
if len(models_config["checkpoints"]) > self.max_checkpoints:
old_save = models_config["checkpoints"][0]
os.remove(os.path.join(self.models_dir, old_save["generator"]))
os.remove(os.path.join(self.models_dir, old_save["discriminator"]))
models_config["checkpoints"] = models_config["checkpoints"][1:]
with open(os.path.join(self.models_dir, "models.json"), 'w') as file:
json.dump(models_config, file, indent=4)
def load_trained_networks(self):
models_file = os.path.join(self.models_dir, "models.json")
if os.path.isfile(models_file):
with open(models_file, 'r') as file:
models_config = json.load(file)
else:
raise FileNotFoundError("File 'models.json' not found in {"
"}".format(self.models_dir))
last_checkpoint = models_config["checkpoints"][-1]
block_index = last_checkpoint["block_index"]
phase = last_checkpoint["phase"]
step = last_checkpoint["step"]
generator_save_name = os.path.join(
self.models_dir, last_checkpoint["generator"])
discriminator_save_name = os.path.join(
self.models_dir, last_checkpoint["discriminator"])
self.generator.load_state_dict(torch.load(generator_save_name))
self.discriminator.load_state_dict(torch.load(discriminator_save_name))
return block_index, phase, step
def train(self):
# get debugging vectors
N = (5, 10)
debug_vectors = torch.randn(N[0] * N[1], self.num_channels, 1,
1).to(self.device)
# get loader
loader = get_loader(self.data_path, self.crop_size, self.batch_size)
losses = {
"d_loss_real": None,
"d_loss_fake": None,
"g_loss": None
}
# resume training if needed
if self.resume_training:
start_index, start_phase, start_step = self.load_trained_networks()
else:
start_index, start_phase, start_step = (0, "fade", 0)
# training loop
start_time = time.time()
absolute_step = -1
for index in range(start_index, self.generator.num_blocks):
loader.dataset.set_transform_by_index(index)
data_iterator = iter(loader)
for phase in ('fade', 'stabilize'):
if index == 0 and phase == 'fade': continue
if self.resume_training and \
index == start_index and \
phase is not start_phase:
continue #
if phase == 'phade': self.alternating_step = 10000 #FIXME del
print("index: {}, size: {}x{}, phase: {}".format(
index, 2 ** (index + 2), 2 ** (index + 2), phase))
if self.resume_training and \
phase == start_phase and \
index == start_index:
step_range = range(start_step, self.alternating_step)
else:
step_range = range(self.alternating_step)
for i in step_range:
absolute_step += 1
try:
batch = next(data_iterator)
except:
data_iterator = iter(loader)
batch = next(data_iterator)
alpha = i / self.alternating_step if phase == "fade" else 1.0
batch = batch.to(self.device)
d_loss_real = - torch.mean(
self.discriminator(batch, index, alpha))
losses["d_loss_real"] = torch.mean(d_loss_real).data[0]
latent = torch.randn(
batch.size(0), self.num_channels, 1, 1).to(self.device)
fake_batch = self.generator(latent, index, alpha).detach()
d_loss_fake = torch.mean(
self.discriminator(fake_batch, index, alpha))
losses["d_loss_fake"] = torch.mean(d_loss_fake).data[0]
# drift factor
drift = d_loss_real.pow(2) + d_loss_fake.pow(2)
d_loss = d_loss_real + d_loss_fake + self.eps_drift * drift
self.d_optimizer.zero_grad()
d_loss.backward() # if retain_graph=True
# then gp works but I'm not sure it's right
self.d_optimizer.step()
# Compute gradient penalty
alpha_gp = torch.rand(batch.size(0), 1, 1, 1).to(self.device)
# mind that x_hat must be both detached from the previous
# gradient graph (from fake_barch) and with
# requires_graph=True so that the gradient can be computed
x_hat = (alpha_gp * batch + (1 - alpha_gp) *
fake_batch).requires_grad_(True)
# x_hat = torch.cuda.FloatTensor(x_hat).requires_grad_(True)
out = self.discriminator(x_hat, index, alpha)
grad = torch.autograd.grad(
outputs=out,
inputs=x_hat,
grad_outputs=torch.ones_like(out).to(self.device),
retain_graph=True,
create_graph=True,
only_inputs=True
)[0]
grad = grad.view(grad.size(0), -1) # is this the same as
# detach?
l2norm = torch.sqrt(torch.sum(grad ** 2, dim=1))
d_loss_gp = torch.mean((l2norm - 1) ** 2)
d_loss_gp *= self.lambda_gp
self.d_optimizer.zero_grad()
d_loss_gp.backward()
self.d_optimizer.step()
# train generator
if (i + 1) % self.ncritic == 0:
latent = torch.randn(
self.batch_size, self.num_channels, 1, 1).to(self.device)
fake_batch = self.generator(latent, index, alpha)
g_loss = - torch.mean(self.discriminator(
fake_batch, index, alpha))
losses["g_loss"] = torch.mean(g_loss).data[0]
self.g_optimizer.zero_grad()
g_loss.backward()
self.g_optimizer.step()
# tensorboard logging
if (i + 1) % self.log_step == 0:
elapsed = time.time() - start_time
elapsed = str(datetime.timedelta(seconds=elapsed))
print("{}:{}:{}/{} time {}, d_loss_real {}, "
"d_loss_fake {}, "
"g_loss {}, alpha {}".format(index, phase, i,
self.alternating_step,
elapsed,
d_loss_real,
d_loss_fake,
g_loss, alpha))
for name, value in losses.items():
self.tflogger.scalar_summary(name, value, absolute_step)
# print debugging images
if (i + 1) % self.debug_step == 0:
self.print_debugging_images(
self.generator, debug_vectors, N, index, alpha, i)
# save trained networks
if (i + 1) % self.save_step == 0:
self.save_trained_networks(index, phase, i)
transforms=composed,
limit=args.limit)
dataloader = DataLoader(dataset,
batch_size=args.batchsize,
num_workers=4 * args.ngpu,
worker_init_fn=worker_init_fn)
print("Training Datas:", len(dataset))
if not os.path.exists(args.weight_folder):
os.mkdir(args.weight_folder)
if not os.path.exists(args.result_folder):
os.mkdir(args.result_folder)
G = Generator(config)
D = Discriminator(config)
G_optim = optim.Adam(G.parameters(), lr=args.lr, betas=(0.5, 0.9))
D_optim = optim.Adam(D.parameters(), lr=args.lr, betas=(0.5, 0.9))
writer = SummaryWriter()
print("Running On:", device)
train_handler = Train_Handler(args.dataset_name,
args.epochs,
G_optim,
D_optim,
dataloader,
device=device,
writer=writer,
save_per_epoch=args.save_per_epoch,
def train(data_path,
crop_size=128,
final_size=64,
batch_size=16,
alternating_step=10000,
ncritic=1,
lambda_gp=0.1,
debug_step=100):
# define networks
generator = Generator(final_size=final_size)
generator.generate_network()
g_optimizer = Adam(generator.parameters())
discriminator = Discriminator(final_size=final_size)
discriminator.generate_network()
d_optimizer = Adam(discriminator.parameters())
num_channels = min(generator.num_channels, generator.max_channels)
# get debugging vectors
N = (5, 10)
debug_vectors = torch.randn(N[0] * N[1], num_channels, 1, 1).to(device)
global upsample
upsample = [
Upsample(scale_factor=2**i)
for i in reversed(range(generator.num_blocks))
]
# get loader
loader = get_loader(data_path, crop_size, batch_size)
# training loop
start_time = time.time()
for index in range(generator.num_blocks):
loader.dataset.set_transform_by_index(index)
data_iterator = iter(loader)
for phase in ('fade', 'stabilize'):
if index == 0 and phase == 'fade': continue
print("index: {}, size: {}x{}, phase: {}".format(
index, 2**(index + 2), 2**(index + 2), phase))
for i in range(alternating_step):
print(i)
try:
batch = next(data_iterator)
except:
data_iterator = iter(loader)
batch = next(data_iterator)
alpha = i / alternating_step if phase == "fade" else 1.0
batch = batch.to(device)
d_loss_real = -torch.mean(discriminator(batch, index, alpha))
latent = torch.randn(batch_size, num_channels, 1, 1).to(device)
fake_batch = generator(latent, index, alpha).detach()
d_loss_fake = torch.mean(
discriminator(fake_batch, index, alpha))
d_loss = d_loss_real + d_loss_fake
d_optimizer.zero_grad()
d_loss.backward() # if retain_graph=True
# then gp works but I'm not sure it's right
d_optimizer.step()
# Compute gradient penalty
alpha_gp = torch.rand(batch.size(0), 1, 1, 1).to(device)
# mind that x_hat must be both detached from the previous
# gradient graph (from fake_barch) and with
# requires_graph=True so that the gradient can be computed
x_hat = (alpha_gp * batch +
(1 - alpha_gp) * fake_batch).requires_grad_(True)
# x_hat = torch.cuda.FloatTensor(x_hat).requires_grad_(True)
out = discriminator(x_hat, index, alpha)
grad = torch.autograd.grad(
outputs=out,
inputs=x_hat,
grad_outputs=torch.ones_like(out).to(device),
retain_graph=True,
create_graph=True,
only_inputs=True)[0]
grad = grad.view(grad.size(0), -1) #is this the same as
# detach?
l2norm = torch.sqrt(torch.sum(grad**2, dim=1))
d_loss_gp = torch.mean((l2norm - 1)**2)
d_loss_gp *= lambda_gp
d_optimizer.zero_grad()
d_loss_gp.backward()
d_optimizer.step()
if (i + 1) % ncritic == 0:
latent = torch.randn(batch_size, num_channels, 1,
1).to(device)
fake_batch = generator(latent, index, alpha)
g_loss = -torch.mean(
discriminator(fake_batch, index, alpha))
g_optimizer.zero_grad()
g_loss.backward()
g_optimizer.step()
# print debugging images
if (i + 1) % debug_step == 0:
print_debugging_images(generator, debug_vectors, N, index,
alpha, i)
def train(savedir, train_list_A, train_list_B, test_list_A, test_list_B, root, epochs, batch_size):
# 画像のチャンネル数
channel = 1
# LossにおけるCycle Lossの割合を決めるパラメータ（Cycle Lossにかかる係数）
cycle_rate = 10.0
# LossにおけるIdentity Lossの割合を決めるパラメータ（Identity Lossにかかる係数）
iden_rate = 0
# ジェネレータのAdam設定(default: lr=0.001, betas=(0.9, 0.999), weight_decay=0)
G_opt_para = {'lr': 0.0002, 'betas': (0.5, 0.9), 'weight_decay': 0}
# ディスクリミネータのAdam設定
D_A_opt_para = {'lr': 0.0002, 'betas': (0.5, 0.9), 'weight_decay': 0}
D_B_opt_para = {'lr': 0.0002, 'betas': (0.5, 0.9), 'weight_decay': 0}
device = 'cuda'
myloss = MyLoss()
# 保存先のファイルを作成
if os.path.exists(savedir):
num = 1
while 1:
if os.path.exists('{}({})'.format(savedir, num)):
num += 1
else:
savedir = '{}({})'.format(savedir, num)
break
os.makedirs(savedir, exist_ok=True)
os.makedirs('{}/generating_image'.format(savedir), exist_ok=True)
os.makedirs('{}/model'.format(savedir), exist_ok=True)
os.makedirs('{}/loss'.format(savedir), exist_ok=True)
G_A2B_model, G_B2A_model, D_A_model, D_B_model = Generator(channel), Generator(channel), Discriminator(channel), Discriminator(channel)
G_A2B_model, G_B2A_model, D_A_model, D_B_model = nn.DataParallel(G_A2B_model), nn.DataParallel(G_B2A_model), nn.DataParallel(D_A_model), nn.DataParallel(D_B_model)
G_A2B_model, G_B2A_model, D_A_model, D_B_model = G_A2B_model.to(device), G_B2A_model.to(device), D_A_model.to(device), D_B_model.to(device)
# 最適化アルゴリズムの設定
G_para = torch.optim.Adam(chain(G_A2B_model.parameters(), G_B2A_model.parameters()), lr=G_opt_para['lr'], betas=G_opt_para['betas'], weight_decay=G_opt_para['weight_decay'])
D_A_para = torch.optim.Adam(D_A_model.parameters(), lr=D_A_opt_para['lr'], betas=D_A_opt_para['betas'], weight_decay=D_A_opt_para['weight_decay'])
D_B_para = torch.optim.Adam(D_B_model.parameters(), lr=D_B_opt_para['lr'], betas=D_B_opt_para['betas'], weight_decay=D_B_opt_para['weight_decay'])
# ロスの推移を保存するためのリストを確保
result = {}
result['G_log_loss'] = []
result['D_A_log_loss'] = []
result['D_B_log_loss'] = []
df_A = pd.read_csv(train_list_A, usecols=['Path'])
df_B = pd.read_csv(train_list_B, usecols=['Path'])
df_test_A = pd.read_csv(test_list_A, usecols=['Path'])
df_test_A = df_test_A.sample(frac=1)
df_test_B = pd.read_csv(test_list_B, usecols=['Path'])
df_test_B = df_test_B.sample(frac=1)
train_dataset = LoadDataset(df_A, df_B, root, transform=Trans())
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, drop_last=True)
test_dataset = LoadDataset(df_test_A[0:batch_size], df_test_B[0:batch_size], root, transform=Trans())
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, drop_last=True)
output_env('{}/env.txt'.format(savedir), batch_size, G_opt_para, D_A_opt_para, D_B_opt_para, G_A2B_model, G_B2A_model, D_A_model, D_B_model)
for epoch in range(epochs):
print('########## epoch : {}/{} ##########'.format(epoch+1, epochs))
G_log_loss, D_A_log_loss, D_B_log_loss = [], [], []
for img_A, img_B in tqdm(train_loader):
# GPU用の変数に変換
img_A = img_A.to(device)
img_B = img_B.to(device)
# 真正画像をジェネレータに入力
fake_img_A = G_B2A_model(img_B)
fake_img_B = G_A2B_model(img_A)
# 生成画像をジェネレータに入力
rec_img_A = G_B2A_model(fake_img_B)
rec_img_B = G_A2B_model(fake_img_A)
# ディスクリミネータに真正画像と生成画像を入力
real_pred_A = D_A_model(img_A)
real_pred_B = D_B_model(img_B)
fake_pred_A = D_A_model(fake_img_A)
fake_pred_B = D_B_model(fake_img_B)
# ジェネレータに出力画像と同一ドメインの画像を入力（恒等写像）
if iden_rate == 0:
iden_img_A = None
iden_img_B = None
else:
iden_img_A = G_B2A_model(img_A)
iden_img_B = G_A2B_model(img_B)
# ジェネレータのロス計算
G_loss = myloss.G_loss(fake_pred_A, fake_pred_B, torch.tensor(1.0).expand_as(fake_pred_A).to(device), img_A, img_B, rec_img_A, rec_img_B, iden_img_A, iden_img_B, alpha=cycle_rate, beta=iden_rate)
G_log_loss.append(G_loss.item())
# ディスクリミネータのロス計算
D_A_loss = myloss.D_A_loss(real_pred_A, torch.tensor(1.0).expand_as(real_pred_A).to(device), fake_pred_A, torch.tensor(0.0).expand_as(fake_pred_A).to(device))
D_A_log_loss.append(D_A_loss.item())
D_B_loss = myloss.D_B_loss(real_pred_B, torch.tensor(1.0).expand_as(real_pred_B).to(device), fake_pred_B, torch.tensor(0.0).expand_as(fake_pred_B).to(device))
D_B_log_loss.append(D_B_loss.item())
# ジェネレータの重み更新
G_para.zero_grad()
G_loss.backward(retain_graph=True)
G_para.step()
# ディスクリミネータの重み更新
D_A_para.zero_grad()
D_A_loss.backward(retain_graph=True)
D_A_para.step()
D_B_para.zero_grad()
D_B_loss.backward()
D_B_para.step()
result['G_log_loss'].append(statistics.mean(G_log_loss))
result['D_A_log_loss'].append(statistics.mean(D_A_log_loss))
result['D_B_log_loss'].append(statistics.mean(D_B_log_loss))
print('G_loss = {} , D_A_loss = {} , D_B_loss = {}'.format(result['G_log_loss'][-1], result['D_A_log_loss'][-1], result['D_B_log_loss'][-1]))
# ロスのログを保存
with open('{}/loss/log.txt'.format(savedir), mode='a') as f:
f.write('##### Epoch {:03} #####\n'.format(epoch+1))
f.write('G: {}, D_A: {}, D_B: {}\n'.format(result['G_log_loss'][-1], result['D_A_log_loss'][-1], result['D_B_log_loss'][-1]))
# 定めた保存周期ごとにモデル，出力画像を保存する
if (epoch+1)%10 == 0:
# モデルの保存
torch.save(G_A2B_model.module.state_dict(), '{}/model/G_A2B_model_{}.pth'.format(savedir, epoch+1))
torch.save(G_B2A_model.module.state_dict(), '{}/model/G_B2A_model_{}.pth'.format(savedir, epoch+1))
torch.save(D_A_model.module.state_dict(), '{}/model/D_A_model_{}.pth'.format(savedir, epoch+1))
torch.save(D_B_model.module.state_dict(), '{}/model/D_B_model_{}.pth'.format(savedir, epoch+1))
G_A2B_model.eval()
G_B2A_model.eval()
# メモリ節約のためパラメータの保存は止める（テスト時にパラメータの保存は不要）
with torch.no_grad():
for test_img_A, test_img_B in test_loader:
fake_img_test_A = G_B2A_model(test_img_B)
fake_img_test_B = G_A2B_model(test_img_A)
torchvision.utils.save_image(fake_img_test_A[:batch_size], "{}/generating_image/A_epoch_{:03}.png".format(savedir, epoch+1))
torchvision.utils.save_image(fake_img_test_B[:batch_size], "{}/generating_image/B_epoch_{:03}.png".format(savedir, epoch+1))
G_A2B_model.train()
G_B2A_model.train()
# 定めた保存周期ごとにロスを保存する
if (epoch+1)%50 == 0:
x = np.linspace(1, epoch+1, epoch+1, dtype='int')
plot(result['G_log_loss'], result['D_A_log_loss'], result['D_B_log_loss'], x, savedir)
# 最後のエポックが保存周期でない場合に，保存する
if (epoch+1)%10 != 0 and epoch+1 == epochs:
torch.save(G_A2B_model.module.state_dict(), '{}/model/G_A2B_model_{}.pth'.format(savedir, epoch+1))
torch.save(G_B2A_model.module.state_dict(), '{}/model/G_B2A_model_{}.pth'.format(savedir, epoch+1))
torch.save(D_A_model.module.state_dict(), '{}/model/D_A_model_{}.pth'.format(savedir, epoch+1))
torch.save(D_B_model.module.state_dict(), '{}/model/D_B_model_{}.pth'.format(savedir, epoch+1))
G_A2B_model.eval()
G_B2A_model.eval()
# メモリ節約のためパラメータの保存は止める（テスト時にパラメータの保存は不要）
with torch.no_grad():
for test_img_A, test_img_B in test_loader:
fake_img_test_A = G_B2A_model(test_img_B)
fake_img_test_B = G_A2B_model(test_img_A)
torchvision.utils.save_image(fake_img_test_A[:batch_size], "{}/generating_image/A_epoch_{:03}.png".format(savedir, epoch+1))
torchvision.utils.save_image(fake_img_test_B[:batch_size], "{}/generating_image/B_epoch_{:03}.png".format(savedir, epoch+1))
G_A2B_model.train()
G_B2A_model.train()
x = np.linspace(1, epoch+1, epoch+1, dtype='int')
plot(result['G_log_loss'], result['D_A_log_loss'], result['D_B_log_loss'], x, savedir)
def train(opt, dataloader_m=None):
# Loss function
adversarial_loss = torch.nn.BCELoss()
# Initialize generator and discriminator
generator = TinyNet()
discriminator = Discriminator()
cuda = True if torch.cuda.is_available() else False
if cuda:
generator.cuda()
discriminator.cuda()
adversarial_loss.cuda()
Tensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor
# Configure data loader
os.makedirs('./data/mnist', exist_ok=True)
dataloader = torch.utils.data.DataLoader(datasets.MNIST(
'./data/mnist',
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])),
batch_size=opt.batch_size,
shuffle=True)
# Optimizers
optimizer_G = torch.optim.Adam(generator.parameters(),
lr=opt.lr,
betas=(0.9, 0.99))
optimizer_D = torch.optim.Adam(discriminator.parameters(),
lr=opt.lr,
betas=(0.9, 0.99))
# ----------
# Training
# ----------
for epoch in range(opt.n_epochs):
for i, (imgs, imgs_ns, labels) in enumerate(dataloader):
# Configure input
real_imgs = Variable(labels.type(Tensor))
# Adversarial ground truths
valid = Variable(Tensor(imgs.size(0), 1).fill_(1.0),
requires_grad=False)
fake = Variable(Tensor(imgs.size(0), 1).fill_(0.0),
requires_grad=False)
# -----------------
# Train Generator
# -----------------
optimizer_G.zero_grad()
# Sample noise as generator input
#z = Variable(Tensor(np.random.normal(0, 1, (imgs.shape[0], opt.latent_dim)))) # [64, 256]
# Generate a batch of images
gen_imgs = generator(imgs.cuda()) + imgs_ns.cuda() #[64, 64, 64]
# Loss measures generator's ability to fool the discriminator
g_loss = adversarial_loss(discriminator(gen_imgs), valid)
g_loss.backward()
optimizer_G.step()
# ---------------------
# Train Discriminator
# ---------------------
optimizer_D.zero_grad()
# Measure discriminator's ability to classify real from generated samples
real_loss = adversarial_loss(discriminator(real_imgs), valid)
fake_loss = adversarial_loss(discriminator(gen_imgs.detach()),
fake)
d_loss = (real_loss + fake_loss) / 2
d_loss.backward()
optimizer_D.step()
print("[Epoch %d/%d] [Batch %d/%d] [D loss: %f] [G loss: %f]" %
(epoch, opt.n_epochs, i, len(dataloader), d_loss.item(),
g_loss.item()))
batches_done = epoch * len(dataloader) + i
if batches_done % opt.sample_interval == 0:
save_image(gen_imgs.data[:25],
'images/%d.png' % batches_done,
nrow=5,
normalize=True)
weights_path = checkpoint_path.format(epoch=batches_done,
loss=g_loss)
print(
'saving generator weights file to {}'.format(weights_path))
torch.save(generator.state_dict(), weights_path)
imageSize = int(opt.imageSize)
lr = opt.lr
gamma = opt.gamma
def weights_init(m):
classname = m.__class__.__name__
if classname.find('Conv') != -1:
m.weight.data.normal_(0.0, 0.02)
elif classname.find('BatchNorm') != -1:
m.weight.data.normal_(1.0, 0.02)
m.bias.data.fill_(0)
NetG = Decoder(nc, ngf, nz).to(device)
NetD = Discriminator(imageSize, nc, ndf, nz).to(device)
NetE = Encoder(imageSize, nc, ngf, nz).to(device)
Sampler = Sampler().to(device)
NetE.apply(weights_init)
NetG.apply(weights_init)
NetD.apply(weights_init)
# load weights
if opt.netE != '':
NetE.load_state_dict(torch.load(opt.netE))
if opt.netG != '':
NetG.load_state_dict(torch.load(opt.netG))
if opt.netD != '':
NetD.load_state_dict(torch.load(opt.netD))
d_out = D(fake)
return -(d_out).mean()
# device
if torch.cuda.is_available() and args.cuda:
device = torch.device('cuda')
torch.backends.cudnn.benchmark = True
print('cudnn benchmark enabled')
else:
device = torch.device('cpu')
print('Device:', device)
# load dataset
# construct networks
G = Generator(ch_style=3, ch_content=1).to(device)
Ds = Discriminator(3+3).to(device)
Dc = Discriminator(3+1).to(device)
if args.dummy_input: # debug purpose
BATCH_SIZE = 16
s1 = torch.randn(BATCH_SIZE, 3, args.resolution, args.resolution).to(device)
contour = torch.randn(BATCH_SIZE, 1, args.resolution, args.resolution).to(device)
fake = G(s1, contour)
print('fake.shape', fake.shape)
ds_out = Ds(torch.cat([fake, s1], dim=1))
dc_out = Dc(torch.cat([fake, contour], dim=1))
print('Ds_out.shape', ds_out.shape)
print('Dc_out.shape', dc_out.shape)
sys.exit(0)
optimG = optim.Adam(G.parameters(), lr=args.lrG, betas=(0, 0.999))
class tag2pix(object):
def __init__(self, args):
if args.model == 'tag2pix':
from network import Generator
elif args.model == 'senet':
from model.GD_senet import Generator
elif args.model == 'resnext':
from model.GD_resnext import Generator
elif args.model == 'catconv':
from model.GD_cat_conv import Generator
elif args.model == 'catall':
from model.GD_cat_all import Generator
elif args.model == 'adain':
from model.GD_adain import Generator
elif args.model == 'seadain':
from model.GD_seadain import Generator
else:
raise Exception('invalid model name: {}'.format(args.model))
self.args = args
self.epoch = args.epoch
self.batch_size = args.batch_size
self.gpu_mode = not args.cpu
self.input_size = args.input_size
self.color_revert = ColorSpace2RGB(args.color_space)
self.layers = args.layers
[self.cit_weight, self.cvt_weight] = args.cit_cvt_weight
self.load_dump = (args.load is not "")
self.load_path = Path(args.load)
self.l1_lambda = args.l1_lambda
self.guide_beta = args.guide_beta
self.adv_lambda = args.adv_lambda
self.save_freq = args.save_freq
self.two_step_epoch = args.two_step_epoch
self.brightness_epoch = args.brightness_epoch
self.save_all_epoch = args.save_all_epoch
self.iv_dict, self.cv_dict, self.id_to_name = get_tag_dict(
args.tag_dump)
cvt_class_num = len(self.cv_dict.keys())
cit_class_num = len(self.iv_dict.keys())
self.class_num = cvt_class_num + cit_class_num
self.start_epoch = 1
#### load dataset
if not args.test:
self.train_data_loader, self.test_data_loader = get_dataset(args)
self.result_path = Path(args.result_dir) / time.strftime(
'%y%m%d-%H%M%S', time.localtime())
if not self.result_path.exists():
self.result_path.mkdir()
self.test_images = self.get_test_data(self.test_data_loader,
args.test_image_count)
else:
self.test_data_loader = get_dataset(args)
self.result_path = Path(args.result_dir)
##### initialize network
self.net_opt = {
'guide': not args.no_guide,
'relu': args.use_relu,
'bn': not args.no_bn,
'cit': not args.no_cit
}
if self.net_opt['cit']:
self.Pretrain_ResNeXT = se_resnext_half(
dump_path=args.pretrain_dump,
num_classes=cit_class_num,
input_channels=1)
else:
self.Pretrain_ResNeXT = nn.Sequential()
self.G = Generator(input_size=args.input_size,
layers=args.layers,
cv_class_num=cvt_class_num,
iv_class_num=cit_class_num,
net_opt=self.net_opt)
self.D = Discriminator(input_dim=3,
output_dim=1,
input_size=self.input_size,
cv_class_num=cvt_class_num,
iv_class_num=cit_class_num)
for param in self.Pretrain_ResNeXT.parameters():
param.requires_grad = False
if args.test:
for param in self.G.parameters():
param.requires_grad = False
for param in self.D.parameters():
param.requires_grad = False
self.Pretrain_ResNeXT = nn.DataParallel(self.Pretrain_ResNeXT)
self.G = nn.DataParallel(self.G)
self.D = nn.DataParallel(self.D)
self.G_optimizer = optim.Adam(self.G.parameters(),
lr=args.lrG,
betas=(args.beta1, args.beta2))
self.D_optimizer = optim.Adam(self.D.parameters(),
lr=args.lrD,
betas=(args.beta1, args.beta2))
self.BCE_loss = nn.BCELoss()
self.CE_loss = nn.CrossEntropyLoss()
self.L1Loss = nn.L1Loss()
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
print("gpu mode: ", self.gpu_mode)
print("device: ", self.device)
print(torch.cuda.device_count(), "GPUS!")
if self.gpu_mode:
self.Pretrain_ResNeXT.to(self.device)
self.G.to(self.device)
self.D.to(self.device)
self.BCE_loss.to(self.device)
self.CE_loss.to(self.device)
self.L1Loss.to(self.device)
def train(self):
self.train_hist = {}
self.train_hist['D_loss'] = []
self.train_hist['G_loss'] = []
self.train_hist['per_epoch_time'] = []
self.train_hist['total_time'] = []
self.y_real_, self.y_fake_ = torch.ones(self.batch_size,
1), torch.zeros(
self.batch_size, 1)
if self.gpu_mode:
self.y_real_, self.y_fake_ = self.y_real_.to(
self.device), self.y_fake_.to(self.device)
if self.load_dump:
self.load(self.load_path)
print("continue training!!!!")
else:
self.end_epoch = self.epoch
self.print_params()
self.D.train()
print('training start!!')
start_time = time.time()
for epoch in range(self.start_epoch, self.end_epoch + 1):
print("EPOCH: {}".format(epoch))
self.G.train()
epoch_start_time = time.time()
if epoch == self.brightness_epoch:
print('changing brightness ...')
self.train_data_loader.dataset.enhance_brightness(
self.input_size)
max_iter = self.train_data_loader.dataset.__len__(
) // self.batch_size
for iter, (original_, sketch_, iv_tag_, cv_tag_) in enumerate(
tqdm(self.train_data_loader, ncols=80)):
if iter >= max_iter:
break
if self.gpu_mode:
sketch_, original_, iv_tag_, cv_tag_ = sketch_.to(
self.device), original_.to(self.device), iv_tag_.to(
self.device), cv_tag_.to(self.device)
# update D network
self.D_optimizer.zero_grad()
with torch.no_grad():
feature_tensor = self.Pretrain_ResNeXT(sketch_)
if self.gpu_mode:
feature_tensor = feature_tensor.to(self.device)
D_real, CIT_real, CVT_real = self.D(original_)
D_real_loss = self.BCE_loss(D_real, self.y_real_)
G_f, _ = self.G(sketch_, feature_tensor, cv_tag_)
if self.gpu_mode:
G_f = G_f.to(self.device)
D_f_fake, CIT_f_fake, CVT_f_fake = self.D(G_f)
D_f_fake_loss = self.BCE_loss(D_f_fake, self.y_fake_)
if self.two_step_epoch == 0 or epoch >= self.two_step_epoch:
CIT_real_loss = self.BCE_loss(
CIT_real, iv_tag_) if self.net_opt['cit'] else 0
CVT_real_loss = self.BCE_loss(CVT_real, cv_tag_)
C_real_loss = self.cvt_weight * CVT_real_loss + self.cit_weight * CIT_real_loss
CIT_f_fake_loss = self.BCE_loss(
CIT_f_fake, iv_tag_) if self.net_opt['cit'] else 0
CVT_f_fake_loss = self.BCE_loss(CVT_f_fake, cv_tag_)
C_f_fake_loss = self.cvt_weight * CVT_f_fake_loss + self.cit_weight * CIT_f_fake_loss
else:
C_real_loss = 0
C_f_fake_loss = 0
D_loss = self.adv_lambda * (D_real_loss + D_f_fake_loss) + (
C_real_loss + C_f_fake_loss)
self.train_hist['D_loss'].append(D_loss.item())
D_loss.backward()
self.D_optimizer.step()
# update G network
self.G_optimizer.zero_grad()
G_f, G_g = self.G(sketch_, feature_tensor, cv_tag_)
if self.gpu_mode:
G_f, G_g = G_f.to(self.device), G_g.to(self.device)
D_f_fake, CIT_f_fake, CVT_f_fake = self.D(G_f)
D_f_fake_loss = self.BCE_loss(D_f_fake, self.y_real_)
if self.two_step_epoch == 0 or epoch >= self.two_step_epoch:
CIT_f_fake_loss = self.BCE_loss(
CIT_f_fake, iv_tag_) if self.net_opt['cit'] else 0
CVT_f_fake_loss = self.BCE_loss(CVT_f_fake, cv_tag_)
C_f_fake_loss = self.cvt_weight * CVT_f_fake_loss + self.cit_weight * CIT_f_fake_loss
else:
C_f_fake_loss = 0
L1_D_f_fake_loss = self.L1Loss(G_f, original_)
L1_D_g_fake_loss = self.L1Loss(
G_g, original_) if self.net_opt['guide'] else 0
G_loss = (D_f_fake_loss + C_f_fake_loss) + \
(L1_D_f_fake_loss + L1_D_g_fake_loss * self.guide_beta) * self.l1_lambda
self.train_hist['G_loss'].append(G_loss.item())
G_loss.backward()
self.G_optimizer.step()
if ((iter + 1) % 100) == 0:
print(
"Epoch: [{:2d}] [{:4d}/{:4d}] D_loss: {:.8f}, G_loss: {:.8f}"
.format(epoch, (iter + 1), max_iter, D_loss.item(),
G_loss.item()))
self.train_hist['per_epoch_time'].append(time.time() -
epoch_start_time)
with torch.no_grad():
self.visualize_results(epoch)
utils.loss_plot(self.train_hist, self.result_path, epoch)
if epoch >= self.save_all_epoch > 0:
self.save(epoch)
elif self.save_freq > 0 and epoch % self.save_freq == 0:
self.save(epoch)
print("Training finish!... save training results")
if self.save_freq == 0 or epoch % self.save_freq != 0:
if self.save_all_epoch <= 0 or epoch < self.save_all_epoch:
self.save(epoch)
self.train_hist['total_time'].append(time.time() - start_time)
print(
"Avg one epoch time: {:.2f}, total {} epochs time: {:.2f}".format(
np.mean(self.train_hist['per_epoch_time']), self.epoch,
self.train_hist['total_time'][0]))
def test(self):
self.load_test(self.args.load)
self.D.eval()
self.G.eval()
load_path = self.load_path
result_path = self.result_path / load_path.stem
if not result_path.exists():
result_path.mkdir()
with torch.no_grad():
for sketch_, index_, _, cv_tag_ in tqdm(self.test_data_loader,
ncols=80):
if self.gpu_mode:
sketch_, cv_tag_ = sketch_.to(self.device), cv_tag_.to(
self.device)
with torch.no_grad():
feature_tensor = self.Pretrain_ResNeXT(sketch_)
if self.gpu_mode:
feature_tensor = feature_tensor.to(self.device)
# D_real, CIT_real, CVT_real = self.D(original_)
G_f, _ = self.G(sketch_, feature_tensor, cv_tag_)
G_f = self.color_revert(G_f.cpu())
for ind, result in zip(index_.cpu().numpy(), G_f):
save_path = result_path / f'{ind}.png'
if save_path.exists():
for i in range(100):
save_path = result_path / f'{ind}_{i}.png'
if not save_path.exists():
break
img = Image.fromarray(result)
img.save(save_path)
def visualize_results(self, epoch, fix=True):
if not self.result_path.exists():
self.result_path.mkdir()
self.G.eval()
# test_data_loader
original_, sketch_, iv_tag_, cv_tag_ = self.test_images
image_frame_dim = int(np.ceil(np.sqrt(len(original_))))
# iv_tag_ to feature tensor 16 * 16 * 256 by pre-reained Sketch.
with torch.no_grad():
feature_tensor = self.Pretrain_ResNeXT(sketch_)
if self.gpu_mode:
original_, sketch_, iv_tag_, cv_tag_, feature_tensor = original_.to(
self.device), sketch_.to(self.device), iv_tag_.to(
self.device), cv_tag_.to(
self.device), feature_tensor.to(self.device)
G_f, G_g = self.G(sketch_, feature_tensor, cv_tag_)
if self.gpu_mode:
G_f = G_f.cpu()
G_g = G_g.cpu()
G_f = self.color_revert(G_f)
G_g = self.color_revert(G_g)
utils.save_images(
G_f[:image_frame_dim * image_frame_dim, :, :, :],
[image_frame_dim, image_frame_dim],
self.result_path / 'tag2pix_epoch{:03d}_G_f.png'.format(epoch))
utils.save_images(
G_g[:image_frame_dim * image_frame_dim, :, :, :],
[image_frame_dim, image_frame_dim],
self.result_path / 'tag2pix_epoch{:03d}_G_g.png'.format(epoch))
def save(self, save_epoch):
if not self.result_path.exists():
self.result_path.mkdir()
with (self.result_path / 'arguments.txt').open('w') as f:
f.write(pprint.pformat(self.args.__dict__))
save_dir = self.result_path
torch.save(
{
'G': self.G.state_dict(),
'D': self.D.state_dict(),
'G_optimizer': self.G_optimizer.state_dict(),
'D_optimizer': self.D_optimizer.state_dict(),
'finish_epoch': save_epoch,
'result_path': str(save_dir)
}, str(save_dir / 'tag2pix_{}_epoch.pkl'.format(save_epoch)))
with (save_dir /
'tag2pix_{}_history.pkl'.format(save_epoch)).open('wb') as f:
pickle.dump(self.train_hist, f)
print("============= save success =============")
print("epoch from {} to {}".format(self.start_epoch, save_epoch))
print("save result path is {}".format(str(self.result_path)))
def load_test(self, checkpoint_path):
checkpoint = torch.load(str(checkpoint_path))
self.G.load_state_dict(checkpoint['G'])
def load(self, checkpoint_path):
checkpoint = torch.load(str(checkpoint_path))
self.G.load_state_dict(checkpoint['G'])
self.D.load_state_dict(checkpoint['D'])
self.G_optimizer.load_state_dict(checkpoint['G_optimizer'])
self.D_optimizer.load_state_dict(checkpoint['D_optimizer'])
self.start_epoch = checkpoint['finish_epoch'] + 1
self.finish_epoch = self.args.epoch + self.start_epoch - 1
print("============= load success =============")
print("epoch start from {} to {}".format(self.start_epoch,
self.finish_epoch))
print("previous result path is {}".format(checkpoint['result_path']))
def get_test_data(self, test_data_loader, count):
test_count = 0
original_, sketch_, iv_tag_, cv_tag_ = [], [], [], []
for orig, sket, ivt, cvt in test_data_loader:
original_.append(orig)
sketch_.append(sket)
iv_tag_.append(ivt)
cv_tag_.append(cvt)
test_count += len(orig)
if test_count >= count:
break
original_ = torch.cat(original_, 0)
sketch_ = torch.cat(sketch_, 0)
iv_tag_ = torch.cat(iv_tag_, 0)
cv_tag_ = torch.cat(cv_tag_, 0)
self.save_tag_tensor_name(iv_tag_, cv_tag_,
self.result_path / "test_image_tags.txt")
image_frame_dim = int(np.ceil(np.sqrt(len(original_))))
if self.gpu_mode:
original_ = original_.cpu()
sketch_np = sketch_.data.numpy().transpose(0, 2, 3, 1)
original_np = self.color_revert(original_)
utils.save_images(
original_np[:image_frame_dim * image_frame_dim, :, :, :],
[image_frame_dim, image_frame_dim],
self.result_path / 'tag2pix_original.png')
utils.save_images(
sketch_np[:image_frame_dim * image_frame_dim, :, :, :],
[image_frame_dim, image_frame_dim],
self.result_path / 'tag2pix_sketch.png')
return original_, sketch_, iv_tag_, cv_tag_
def save_tag_tensor_name(self, iv_tensor, cv_tensor, save_file_path):
'''iv_tensor, cv_tensor: batched one-hot tag tensors'''
iv_dict_inverse = {
tag_index: tag_id
for (tag_id, tag_index) in self.iv_dict.items()
}
cv_dict_inverse = {
tag_index: tag_id
for (tag_id, tag_index) in self.cv_dict.items()
}
with open(save_file_path, 'w') as f:
f.write("CIT tags\n")
for tensor_i, batch_unit in enumerate(iv_tensor):
tag_list = []
f.write(f'{tensor_i} : ')
for i, is_tag in enumerate(batch_unit):
if is_tag:
tag_name = self.id_to_name[iv_dict_inverse[i]]
tag_list.append(tag_name)
f.write(f"{tag_name}, ")
f.write("\n")
f.write("\nCVT tags\n")
for tensor_i, batch_unit in enumerate(cv_tensor):
tag_list = []
f.write(f'{tensor_i} : ')
for i, is_tag in enumerate(batch_unit):
if is_tag:
tag_name = self.id_to_name[cv_dict_inverse[i]]
tag_list.append(self.id_to_name[cv_dict_inverse[i]])
f.write(f"{tag_name}, ")
f.write("\n")
def print_params(self):
params_cnt = [0, 0, 0]
for param in self.G.parameters():
params_cnt[0] += param.numel()
for param in self.D.parameters():
params_cnt[1] += param.numel()
for param in self.Pretrain_ResNeXT.parameters():
params_cnt[2] += param.numel()
print(
f'Parameter #: G - {params_cnt[0]} / D - {params_cnt[1]} / Pretrain - {params_cnt[2]}'
)
def main(params):
if params['load_dataset']:
dataset = load_pkl(params['load_dataset'])
elif params['dataset_class']:
dataset = globals()[params['dataset_class']](**params[params['dataset_class']])
if params['save_dataset']:
save_pkl(params['save_dataset'], dataset)
else:
raise Exception('One of either load_dataset (path to pkl) or dataset_class needs to be specified.')
result_dir = create_result_subdir(params['result_dir'], params['exp_name'])
losses = ['G_loss', 'D_loss', 'D_real', 'D_fake']
stats_to_log = [
'tick_stat',
'kimg_stat',
]
if params['progressive_growing']:
stats_to_log.extend([
'depth',
'alpha',
'lod',
'minibatch_size'
])
stats_to_log.extend([
'time',
'sec.tick',
'sec.kimg'
] + losses)
logger = TeeLogger(os.path.join(result_dir, 'log.txt'), stats_to_log, [(1, 'epoch')])
logger.log(params_to_str(params))
if params['resume_network']:
G, D = load_models(params['resume_network'], params['result_dir'], logger)
else:
G = Generator(dataset.shape, **params['Generator'])
D = Discriminator(dataset.shape, **params['Discriminator'])
if params['progressive_growing']:
assert G.max_depth == D.max_depth
G.cuda()
D.cuda()
latent_size = params['Generator']['latent_size']
logger.log(str(G))
logger.log('Total nuber of parameters in Generator: {}'.format(
sum(map(lambda x: reduce(lambda a, b: a*b, x.size()), G.parameters()))
))
logger.log(str(D))
logger.log('Total nuber of parameters in Discriminator: {}'.format(
sum(map(lambda x: reduce(lambda a, b: a*b, x.size()), D.parameters()))
))
def get_dataloader(minibatch_size):
return DataLoader(dataset, minibatch_size, sampler=InfiniteRandomSampler(dataset),
num_workers=params['num_data_workers'], pin_memory=False, drop_last=True)
def rl(bs):
return lambda: random_latents(bs, latent_size)
# Setting up learning rate and optimizers
opt_g = Adam(G.parameters(), params['G_lr_max'], **params['Adam'])
opt_d = Adam(D.parameters(), params['D_lr_max'], **params['Adam'])
def rampup(cur_nimg):
if cur_nimg < params['lr_rampup_kimg'] * 1000:
p = max(0.0, 1 - cur_nimg / (params['lr_rampup_kimg'] * 1000))
return np.exp(-p * p * 5.0)
else:
return 1.0
lr_scheduler_d = LambdaLR(opt_d, rampup)
lr_scheduler_g = LambdaLR(opt_g, rampup)
mb_def = params['minibatch_size']
D_loss_fun = partial(wgan_gp_D_loss, return_all=True, iwass_lambda=params['iwass_lambda'],
iwass_epsilon=params['iwass_epsilon'], iwass_target=params['iwass_target'])
G_loss_fun = wgan_gp_G_loss
trainer = Trainer(D, G, D_loss_fun, G_loss_fun,
opt_d, opt_g, dataset, iter(get_dataloader(mb_def)), rl(mb_def), **params['Trainer'])
# plugins
if params['progressive_growing']:
max_depth = min(G.max_depth, D.max_depth)
trainer.register_plugin(DepthManager(get_dataloader, rl, max_depth, **params['DepthManager']))
for i, loss_name in enumerate(losses):
trainer.register_plugin(EfficientLossMonitor(i, loss_name))
checkpoints_dir = params['checkpoints_dir'] if params['checkpoints_dir'] else result_dir
trainer.register_plugin(SaverPlugin(checkpoints_dir, **params['SaverPlugin']))
def subsitute_samples_path(d):
return {k:(os.path.join(result_dir, v) if k == 'samples_path' else v) for k,v in d.items()}
postprocessors = [ globals()[x](**subsitute_samples_path(params[x])) for x in params['postprocessors'] ]
trainer.register_plugin(OutputGenerator(lambda x: random_latents(x, latent_size),
postprocessors, **params['OutputGenerator']))
trainer.register_plugin(AbsoluteTimeMonitor(params['resume_time']))
trainer.register_plugin(LRScheduler(lr_scheduler_d, lr_scheduler_g))
trainer.register_plugin(logger)
init_comet(params, trainer)
trainer.run(params['total_kimg'])
dataset.close()
fig.savefig(save_path, dpi=300)
plt.close(fig)
parser = argparse.ArgumentParser()
parser.add_argument('--data_folder', type=str, default='data/')
parser.add_argument('--train_file', type=str, default='3.csv')
config = parser.parse_args()
train_file = config.data_folder + config.train_file
#dataset = read_csv_faster('./data/1.csv')
dataset = read_csv_faster(train_file)
FinalData = dataset['mz_exp'].transpose()
AllLabel = dataset['labels']
num_inputs = FinalData.shape[1]
discriminator = Discriminator(num_inputs=num_inputs)
def matric(cluster, labels):
TP, TN, FP, FN = 0, 0, 0, 0
n = len(labels)
for i in range(n):
if cluster[i]:
if labels[i]:
TP += 1
else:
FP += 1
elif labels[i]:
FN += 1
else:
TN += 1
return TP, TN, FP, FN
#input
dataloader = torch.utils.data.DataLoader(
TrainDataset(img_list),
batch_size=config.train['batch_size'],
shuffle=True,
num_workers=0,
pin_memory=True)
# G = torch.nn.DataParallel( Generator(zdim = config.G['zdim'], use_batchnorm = config.G['use_batchnorm'] , use_residual_block = config.G['use_residual_block'] , num_classes = config.G['num_classes'])).cuda()
# D = torch.nn.DataParallel( Discriminator(use_batchnorm = config.D['use_batchnorm'])).cuda()
G = Generator(zdim=config.G['zdim'],
use_batchnorm=config.G['use_batchnorm'],
use_residual_block=config.G['use_residual_block'],
num_classes=config.G['num_classes']).cuda()
D = Discriminator(use_batchnorm=config.D['use_batchnorm']).cuda()
optimizer_G = torch.optim.Adam(filter(lambda p: p.requires_grad,
G.parameters()),
lr=1e-4)
optimizer_D = torch.optim.Adam(filter(lambda p: p.requires_grad,
D.parameters()),
lr=1e-5)
last_epoch = -1
if config.train['resume_model'] is not None:
e1 = resume_model(G, config.train['resume_model'])
e2 = resume_model(D, config.train['resume_model'])
assert e1 == e2
last_epoch = e1
if config.train['resume_optimizer'] is not None:
# Create directories for images, tensorboard results and saved models.
if not args.dry_run:
if not os.path.exists(EXPERIMENT_DIR):
os.makedirs(EXPERIMENT_DIR) # Set up root experiment directory.
os.makedirs(args.save_image_dir)
os.makedirs(args.tensorboard_dir)
os.makedirs(args.save_model_dir)
WRITER = SummaryWriter(args.tensorboard_dir) # Set up TensorBoard.
else:
print('Dry run! Just for testing, data is not saved')
DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Set up the GAN.
discriminator_model = Discriminator().to(DEVICE)
generator_model = Generator().to(DEVICE)
# Load pre-trained models if they are provided.
if args.load_discriminator_model_path:
discriminator_model.load_state_dict(
torch.load(args.load_discriminator_model_path))
if args.load_generator_model_path:
generator_model.load_state_dict(torch.load(args.load_generator_model_path))
# Set up Adam optimizers for both models.
discriminator_optimizer = optim.Adam(discriminator_model.parameters(),
lr=args.learning_rate,
betas=(0, 0.9))
generator_optimizer = optim.Adam(generator_model.parameters(),
# get the data loader
# rawdataset = ImageDataset(root, True)
# dataloader = DataLoader(rawdataset, batch_size=32, shuffle=True, num_workers=4)
# %%
"""
G12 is generator that learns to change image from 1 to 2
G21 is generator that learns to change image from 2 to 1
D1 is discriminator that differentiates fake generated by G21 from images of class 1
D2 is discriminator that differentiates fake generated by G12 from images of class 2
"""
G12 = Generator(3, 3)
G21 = Generator(3, 3)
D1 = Discriminator(3)
D2 = Discriminator(3)
# shift models to cuda if possible
if torch.cuda.is_available():
G12.cuda()
G21.cuda()
D1.cuda()
D2.cuda()
# %%
# optimizer and loss
LGAN = MSELoss()
LCYC = L1Loss()
LIdentity = L1Loss()
#input
dataloader = torch.utils.data.DataLoader(
TrainDataset(img_list),
batch_size=config.train['batch_size'],
shuffle=True,
num_workers=8,
pin_memory=True)
G = torch.nn.DataParallel(
Generator(zdim=config.G['zdim'],
use_batchnorm=config.G['use_batchnorm'],
use_residual_block=config.G['use_residual_block'],
num_classes=config.G['num_classes'])).cuda()
D = torch.nn.DataParallel(
Discriminator(use_batchnorm=config.D['use_batchnorm'])).cuda()
optimizer_G = torch.optim.Adam(filter(lambda p: p.requires_grad,
G.parameters()),
lr=1e-4)
optimizer_D = torch.optim.Adam(filter(lambda p: p.requires_grad,
D.parameters()),
lr=1e-4)
last_epoch = -1
if config.train['resume_model'] is not None:
e1 = resume_model(G, config.train['resume_model'])
e2 = resume_model(D, config.train['resume_model'])
assert e1 == e2
last_epoch = e1
if config.train['resume_optimizer'] is not None:
e3 = resume_optimizer(optimizer_G, G, config.train['resume_optimizer'])
for i in range(4):
for j in range(4):
ax[i, j].imshow((out[i * 4 + j] + 1.) / 2.)
fig.savefig('./result/epoch_%d_result.png' % (epoch + 1))
plt.close(fig)
# print (' 에포크 {} 에서 걸린 시간은 {} 초 입니다'.format(epoch +1, time.time()-start))
print('Time for epoch {} is {} sec'.format(epoch + 1,
time.time() - start))
print('Loss for Generator is {}'.format(gen_loss / itr_num))
print('Loss for Discriminator is {}'.format(dis_loss / itr_num))
d = Discriminator()
g = Generator()
# db_helper = FoodDBHelper('/home/ybrain/sangmin/food_dataset/preprocessed_224_224', BATCH_SIZE)
db_helper = DBHelper(
'/home/ybrain/sangmin/ybrain_db/topo_artifact_db/EEGArtifact/topomap',
BATCH_SIZE)
generator_optimizer = tf.keras.optimizers.Adam(1e-3)
discriminator_optimizer = tf.keras.optimizers.Adam(1e-3)
checkpoint_dir = './training_checkpoints'
checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")
checkpoint = tf.train.Checkpoint(
generator_optimizer=generator_optimizer,
in_resized[i] * 255, o * 255,
out_sample_images[i] * 255
],
axis=1)) for i, o in enumerate(preds)
]
},
commit=False)
from network import Generator, Discriminator, get_gan_network
shape = (config.input_width, config.input_height, 3)
image_shape = (config.output_width, config.output_height, 3)
generator = Generator(shape).generator()
discriminator = Discriminator(image_shape).discriminator()
#adam = optimizers.Adam(lr=1E-4, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
generator.compile(loss="mse", optimizer="adam")
discriminator.compile(loss="binary_crossentropy", optimizer="adam")
gan = get_gan_network(discriminator,
shape,
generator,
"adam",
gan_metric=perceptual_distance)
for e in range(1, config.num_epochs + 1):
print('-' * 15, 'Epoch %d' % e, '-' * 15)
for _ in range(config.steps_per_epoch):
num_workers=opt.num_workers,
drop_last=True)
#make reference dir
result_model_path = "./result/model/%s" % (opt.model_name)
log_path = "./log/%s" % (opt.model_name)
mkdir(result_model_path)
mkdir(log_path)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print('device: ', device)
#creat network
netG_A2B = Generator(opt.input_nc, opt.output_nc, opt.ngf).to(device)
netG_B2A = Generator(opt.input_nc, opt.output_nc, opt.ngf).to(device)
netD_A = Discriminator(opt.input_nc, opt.ndf).to(device)
netD_B = Discriminator(opt.input_nc, opt.ndf).to(device)
#instialize weights
if opt.start_epoch == 0:
netG_A2B.apply(weights_init)
netG_B2A.apply(weights_init)
netD_A.apply(weights_init)
netD_B.apply(weights_init)
print("training start from begining")
else: #read trained network param
netG_A2B.load_state_dict(
torch.load('%s/%s_netG_A2B_ep%s.pth' %
(result_model_path, opt.model_name, opt.start_epoch)))
netG_B2A.load_state_dict(
torch.load('%s/%s_netG_B2A_ep%s.pth' %
(0.5, 0.5, 0.5))
]),
mode='train')
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=opt.batchsize,
shuffle=True,
num_workers=opt.num_workers)
# print(dataloader.__len__())
# img=dataset.__getitem__(10)
# print(img['A'].size())
netG_A2B = Generator(opt.input_nc, opt.output_nc, opt.ngf)
netG_B2A = Generator(opt.input_nc, opt.output_nc, opt.ngf)
netD_A = Discriminator(opt.input_nc, opt.ndf)
netD_B = Discriminator(opt.input_nc, opt.ndf)
summary(netG_A2B, input_size=(3, 256, 256), device='cpu')
summary(netD_A, input_size=(3, 256, 256), device='cpu')
#instialize weights
netG_A2B.apply(weights_init)
netG_B2A.apply(weights_init)
netD_A.apply(weights_init)
netD_B.apply(weights_init)
#print(netG_A2B)
#print(netD_A)
sumple_num = pd.read_csv(config.data_folder + 'sumple-num-' +
config.train_file)
sumple_num = list(_flatten(np.array(sumple_num).tolist()))
sample_ids = len(sumple_num)
num_epochs = config.num_epochs
batch_size = config.batch_size # batch size for each cluster
base_lr = config.base_lr
lr_step = config.lr_step # step decay of learning rates
l2_decay = config.l2_decay
dataset = read_csv_faster(train_file)
FinalData = dataset['mz_exp'].transpose()
AllLabel = dataset['labels']
num_inputs = FinalData.shape[1]
discriminator = Discriminator(num_inputs=num_inputs)
def matric(cluster, labels):
TP, TN, FP, FN = 0, 0, 0, 0
n = len(labels)
for i in range(n):
if cluster[i]:
if labels[i]:
TP += 1
else:
FP += 1
elif labels[i]:
FN += 1
else:
TN += 1
