def init(self, ref):
ref_tensor = TF.ToTensor()(ref)
self.canvas = ParameterizedImg(3,
ref_tensor.shape[1],
ref_tensor.shape[2],
init_img=ref_tensor,
space='spectral',
colors='uncorr').to(self.device)
self.opt = DeepDreamOptim(self.canvas.parameters(), lr=self.lr)
def fit(self, n_iters, neuron):
"""
Run the recipe
Args:
n_iters (int): number of iterations to run
neuron (int): the feature map to maximize
Returns:
the optimized image
"""
canvas = ParameterizedImg(3, self.input_size + 10,
self.input_size + 10)
def forward(_):
cim = canvas()
rnd = random.randint(0, cim.shape[2] // 10)
im = cim[:, :, rnd:, rnd:]
im = torch.nn.functional.interpolate(im,
size=(self.input_size,
self.input_size),
mode='bilinear')
_, acts = self.model(self.norm(im), detach=False)
fmap = acts[self.layer]
loss = -fmap[0][neuron].sum()
loss.backward()
return {'loss': loss, 'img': cim}
loop = Recipe(forward, range(n_iters))
loop.register('canvas', canvas)
loop.register('model', self)
loop.callbacks.add_callbacks([
tcb.Counter(),
tcb.Log('loss', 'loss'),
tcb.Log('img', 'img'),
tcb.Optimizer(DeepDreamOptim(canvas.parameters(), lr=self.lr)),
tcb.VisdomLogger(visdom_env=self.visdom_env, log_every=10),
tcb.StdoutLogger(log_every=10)
])
loop.to(self.device)
loop.run(1)
return canvas.render().cpu()
def fit(self, ref, iters, lr=3e-4, device='cpu', visdom_env='deepdream'):
"""
Args:
lr (float, optional): the learning rate
visdom_env (str or None): the name of the visdom env to use, or None
to disable Visdom
"""
ref_tensor = TF.ToTensor()(ref).unsqueeze(0)
canvas = ParameterizedImg(1,
3,
ref_tensor.shape[2],
ref_tensor.shape[3],
init_img=ref_tensor,
space='spectral',
colors='uncorr')
def forward(_):
img = canvas()
rnd = random.randint(0, 10)
loss = self.loss(self.norm(img[:, :, rnd:, rnd:]))
loss.backward()
return {'loss': loss, 'img': img}
loop = Recipe(forward, range(iters))
loop.register('model', self)
loop.register('canvas', canvas)
loop.callbacks.add_callbacks([
tcb.Counter(),
tcb.Log('loss', 'loss'),
tcb.Log('img', 'img'),
tcb.Optimizer(DeepDreamOptim(canvas.parameters(), lr=lr)),
tcb.VisdomLogger(visdom_env=visdom_env, log_every=10),
tcb.StdoutLogger(log_every=10)
])
loop.to(device)
loop.run(1)
return canvas.render().cpu()
class DeepDreamRecipe(ImageOptimizationBaseRecipe):
"""
Deep Dream recipe
First instantiate the recipe then call `recipe(n_iter, img)`
Args:
model (nn.Module): the trained model to use
dream_layer (str): the layer to use on which activations will be
maximized
lr (float, optional): the learning rate
device (device): where to run the computation
visdom_env (str or None): the name of the visdom env to use, or None
to disable Visdom
"""
def __init__(self,
model,
dream_layer,
lr=3e-4,
device='cpu',
visdom_env='deepdream'):
super(DeepDreamRecipe, self).__init__(visdom_env=visdom_env)
self.device = device
self.loss = DeepDreamLoss(model, dream_layer).to(device)
self.norm = tnn.ImageNetInputNorm().to(device)
self.lr = lr
def init(self, ref):
ref_tensor = TF.ToTensor()(ref)
self.canvas = ParameterizedImg(3,
ref_tensor.shape[1],
ref_tensor.shape[2],
init_img=ref_tensor,
space='spectral',
colors='uncorr').to(self.device)
self.opt = DeepDreamOptim(self.canvas.parameters(), lr=self.lr)
def forward(self):
self.opt.zero_grad()
cim = self.canvas()
loss = self.loss(
self.norm(cim[:, :, self.iters % 10:, self.iters % 10:]))
loss.backward()
self.opt.step()
return {'loss': loss}
def result(self):
return self.canvas.render()
def __call__(self, n_iters, ref):
"""
Run the recipe
Args:
n_iters (int): number of iterations to perform
ref (PIL.Image): the base image
Returns:
the optimized image
"""
return super(DeepDreamRecipe, self).__call__(n_iters, ref)
def init(self, channel):
self.channel = channel
self.canvas = ParameterizedImg(3, self.input_size + 10,
self.input_size + 10).to(self.device)
self.opt = DeepDreamOptim(self.canvas.parameters(), lr=self.lr)
class FeatureVisRecipe(ImageOptimizationBaseRecipe):
"""
Feature viz
First instantiate the recipe then call `recipe(n_iter, img)`
Args:
model (nn.Module): the trained model to use
layer (str): the layer to use on which activations will be maximized
input_size (int): the size of the square image the model accepts as
input
lr (float, optional): the learning rate
device (device): where to run the computation
visdom_env (str or None): the name of the visdom env to use, or None
to disable Visdom
"""
def __init__(self,
model,
layer,
input_size,
lr=1e-3,
device='cpu',
visdom_env='feature_vis'):
super(FeatureVisRecipe, self).__init__(visdom_env=visdom_env)
self.device = device
self.model = tnn.WithSavedActivations(model, names=[layer]).to(device)
self.model.eval()
self.layer = layer
self.input_size = input_size
self.norm = tnn.ImageNetInputNorm().to(device)
self.lr = lr
def init(self, channel):
self.channel = channel
self.canvas = ParameterizedImg(3, self.input_size + 10,
self.input_size + 10).to(self.device)
self.opt = DeepDreamOptim(self.canvas.parameters(), lr=self.lr)
def forward(self):
self.opt.zero_grad()
cim = self.canvas()
im = cim[:, :, self.iters % 10:, self.iters % 10:]
im = torch.nn.functional.interpolate(im,
size=(self.input_size,
self.input_size),
mode='bilinear')
_, acts = self.model(self.norm(im), detach=False)
fmap = acts[self.layer]
loss = -fmap[0][self.channel].sum()
loss.backward()
self.opt.step()
return {'loss': loss, 'x': self.canvas.render()}
def result(self):
return self.canvas.render()
def __call__(self, n_iters, neuron):
"""
Run the recipe
Args:
n_iters (int): number of iterations to run
neuron (int): the the feature map to maximize
Returns:
the optimized image
"""
return super(FeatureVisRecipe, self).__call__(n_iters, neuron)