data_sampler = None
dl = DataLoader(ds, BATCH_SIZE, shuffle=not data_sampler, sampler=data_sampler)
vae_params = dict(image_size=IMAGE_SIZE,
num_layers=NUM_LAYERS,
num_tokens=NUM_TOKENS,
codebook_dim=EMB_DIM,
hidden_dim=HID_DIM,
num_resnet_blocks=NUM_RESNET_BLOCKS)
vae = DiscreteVAE(**vae_params,
smooth_l1_loss=SMOOTH_L1_LOSS,
kl_div_loss_weight=KL_LOSS_WEIGHT)
if not using_deepspeed:
vae = vae.cuda()
assert len(ds) > 0, 'folder does not contain any images'
if distr_backend.is_root_worker():
print(f'{len(ds)} images found for training')
def save_model(path):
if not distr_backend.is_root_worker():
return
save_obj = {'hparams': vae_params, 'weights': vae.state_dict()}
torch.save(save_obj, path)
from torch.utils.data import DataLoader
from dalle_pytorch import DiscreteVAE
imgSize = 256
load_epoch = 280
vae = DiscreteVAE(image_size=imgSize,
num_layers=3,
channels=3,
num_tokens=2048,
codebook_dim=1024,
hidden_dim=128)
vae_dict = torch.load("./models/dvae-" + str(load_epoch) + ".pth")
vae.load_state_dict(vae_dict)
vae.cuda()
batchSize = 12
n_epochs = 500
log_interval = 20
#images = torch.randn(4, 3, 256, 256)
t = transforms.Compose([
transforms.Resize(imgSize),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5),
(0.5, 0.5, 0.5)) #(0.267, 0.233, 0.234))
])
train_set = datasets.ImageFolder('./imagedata',
# optimizer
get_codes = vae.get_codebook_indices
get_hard_recons = vae.decode
opt = Adam(vae.parameters(), lr = LEARNING_RATE)
sched = ExponentialLR(optimizer = opt, gamma = LR_DECAY_RATE)
logger.debug(f"args.local_rank : {args.local_rank}")
if args.local_rank is not None:
torch.cuda.set_device(args.local_rank)
else:
torch.cuda.set_device(0)
if args.multigpus_distributed:
vae.cuda(args.local_rank)
if args.model_parallel:
vae = smp.DistributedModel(vae)
args.scaler = smp.amp.GradScaler()
opt = smp.DistributedOptimizer(opt)
if args.partial_checkpoint:
args.checkpoint = smp.load(args.partial_checkpoint, partial=True)
vae.load_state_dict(args.checkpoint["model_state_dict"])
opt.load_state_dict(args.checkpoint["optimizer_state_dict"])
elif args.full_checkpoint:
args.checkpoint = smp.load(args.full_checkpoint, partial=False)
vae.load_state_dict(args.checkpoint["model_state_dict"])
opt.load_state_dict(args.checkpoint["optimizer_state_dict"])
else: