def make_dataset(cfg):
if cfg.folder:
_dataset = torchvision.datasets.ImageFolder(
root=cfg.img_dir,
transform=get_transform(new_size=(cfg.resolution, cfg.resolution)))
return _dataset
else:
Dataset = FlatDirectoryImageDataset
_dataset = Dataset(data_dir=cfg.img_dir,
transform=get_transform(new_size=(cfg.resolution,
cfg.resolution)))
return _dataset
def get_data_loader(cfg, batch_size, num_workers):
"""
generate the data_loader from the given dataset
:param dataset: dataset for training (Should be a PyTorch dataset)
Make sure every item is an Image
:param batch_size: batch size of the data
:param num_workers: num of parallel readers
:return: dl => data_loader for the dataset
"""
'''
from torch.utils.data import DataLoader
dl = DataLoader(
dataset,
batch_size=batch_size,
shuffle=True,
num_workers=num_workers,
drop_last=True,
pin_memory=True
)
'''
if cfg.folder:
# raise NotImplementedError
Dataset = FoldersDistributedDataset
else:
Dataset = FlatDirectoryImageDataset
dl = Dataset(data_dir=cfg.img_dir, transform=get_transform(new_size=(cfg.resolution, cfg.resolution))).set_attrs(
batch_size=batch_size,
shuffle=True,
num_workers=num_workers,
drop_last=True,
)
return dl
def make_dataset(cfg):
if cfg.folder:
# raise NotImplementedError
Dataset = FoldersDistributedDataset
else:
Dataset = FlatDirectoryImageDataset
_dataset = Dataset(data_dir=cfg.img_dir, transform=get_transform(new_size=(cfg.resolution, cfg.resolution)))
return _dataset
def make_caricatures(
idx,
video_dir,
out_dir,
amp=5,
mode='full',
size='Small',
batch_size=1,
basemodel_name='resnet18',
):
os.makedirs(outdir, exist_ok=True)
transform = transforms.get_transform(split='val',
normalize=False,
rescale=False)
dataset = VideoFolder(datadir, step=1, transform=transform)
dataset_orig = VideoFolder(datadir, step=1)
# model = core.get_model(f'SeriesPretrainedFrozen{size}ManipulatorAttnDetector')
# dataloader = torch.utils.data.DataLoader(
# dataset,
# batch_size=batch_size,
# shuffle=False,
# num_workers=4,
# pin_memory=False,
# drop_last=False,
# )
# dataloader_orig = torch.utils.data.DataLoader(
# dataset_orig,
# batch_size=batch_size,
# shuffle=False,
# num_workers=4,
# pin_memory=False,
# drop_last=False,
# )
# for i, ((names, frames, target), (_, frames_orig, _)) in enumerate(
# zip(dataset, dataset_orig)
# ):
names, frames, target = dataset[idx]
names, frames_orig, target = dataset_orig[idx]
names = [names]
frames = frames.unsqueeze(0)
frames_orig = frames_orig.unsqueeze(0)
print(f'Processing [{i} / {len(dataset)}]: {", ".join(names)}')
process(i, frames, frames_orig, names, basemodel_name, size, amp, mode)
del frames
del frames_orig
del names
del target
torch.cuda.empty_cache()
def make_dataset(cfg, conditional=False):
if conditional:
Dataset = ImageFolder
else:
if cfg.folder:
Dataset = FoldersDistributedDataset
else:
Dataset = FlatDirectoryImageDataset
transforms = get_transform(new_size=(cfg.resolution, cfg.resolution))
_dataset = Dataset(cfg.img_dir, transform=transforms)
return _dataset
def _make_caricatures(
video_dir,
out_dir,
amp=5,
mode='full',
size='Small',
batch_size=1,
basemodel_name='resnet18',
):
os.makedirs(outdir, exist_ok=True)
transform = transforms.get_transform(split='val',
normalize=False,
rescale=False)
dataset = VideoFolder(datadir, step=1, transform=transform)
dataset_orig = VideoFolder(datadir, step=1)
# model = core.get_model(f'SeriesPretrainedFrozen{size}ManipulatorAttnDetector')
model = core.get_model(f'SeriesPretrainedFrozen{size}ManipulatorDetector')
ckpt_file = CHECKPOINTS[basemodel_name]
ckpt = torch.load(ckpt_file)
state_dict = mutils.remove_prefix(ckpt['state_dict'])
model.detector_model.load_state_dict(state_dict, strict=False)
model = model.to(device)
model.eval()
dataloader = torch.utils.data.DataLoader(
dataset,
batch_size=batch_size,
shuffle=False,
num_workers=4,
pin_memory=False,
drop_last=False,
)
dataloader_orig = torch.utils.data.DataLoader(
dataset_orig,
batch_size=batch_size,
shuffle=False,
num_workers=4,
pin_memory=False,
drop_last=False,
)
for i, ((names, frames, target),
(_, frames_orig, _)) in enumerate(zip(dataloader,
dataloader_orig)):
print(f'Processing [{i} / {len(dataloader)}]: {", ".join(names)}')
if i == 0:
continue
model = model.cpu()
model = model.to(device)
frames = frames.to(device)
attn_map = None
model.zero_grad()
if mode.lower() == 'gradcam':
normalize_attn = True
gcam_model = vutils.grad_cam.GradCAM(model.detector_model.model)
out, attn_map = gcam_forward(gcam_model, frames)
attn_map.detach_()
attn_map = attn_map.cpu()
del gcam_model
with torch.no_grad():
if mode == 'attn':
normalize_attn = True
out, attn_map = model.detector_model(frames)
del frames
frames_orig = frames_orig.to(device)
cari = model.manipulate(
frames_orig,
amp=torch.tensor(amp),
attn_map=attn_map,
normalize_attn=normalize_attn,
)
cari = cari.cpu()
del frames_orig
torch.cuda.empty_cache()
for n, (name, c) in enumerate(zip(names, cari)):
c = c.permute(1, 2, 3, 0)
outname = f'{name.replace(".mp4", "")}_cari_{size}_{mode}_amp{amp}' + '.mp4'
outfile = os.path.join(outdir, outname)
pretorched.data.utils.async_array_to_video(c, outfile)
if attn_map is not None:
am = normalize(attn_map[n]).cpu()
attn_outname = (
f'{name.replace(".mp4", "")}_attn_{size}_{mode}_amp{amp}' +
'.mp4')
attn_outfile = os.path.join(outdir, attn_outname)
if mode not in ['gradcam']:
pretorched.data.utils.async_array_to_video(
255 * am.unsqueeze(-1).repeat(1, 1, 1, 3),
attn_outfile,
)
heatmap_outname = (
f'{name.replace(".mp4", "")}_heatmap_{size}_{mode}_amp{amp}'
+ '.mp4')
heatmap = [
vutils.grad_cam.apply_heatmap(a, cc)
for a, cc in zip(am.numpy(), c.numpy())
]
heatmap_outfile = os.path.join(outdir, heatmap_outname)
pretorched.data.utils.async_array_to_video(
heatmap,
heatmap_outfile,
)