def dream(self):
# Process image and return variable
self.processed_image = preprocess_image(self.created_image, True)
# Define optimizer for the image
# Earlier layers need higher learning rates to visualize whereas layer layers need less
optimizer = SGD([self.processed_image], lr=12, weight_decay=1e-4)
for i in range(1, 251):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = self.processed_image
for index, layer in enumerate(self.model):
# Forward
x = layer(x)
# Only need to forward until we the selected layer is reached
if index == self.selected_layer:
break
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = -torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:',
"{0:.2f}".format(loss.data.numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(self.processed_image)
# Save image every 20 iteration
if i % 10 == 0:
print(self.created_image.shape)
im_path = '../generated/ddream_l' + str(self.selected_layer) + \
'_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg'
save_image(self.created_image, im_path)
def generate(self):
initial_learning_rate = 6
for i in range(1, 150):
# Process image and return variable
self.processed_image = preprocess_image(self.created_image)
# Define optimizer for the image
optimizer = SGD([self.processed_image], lr=initial_learning_rate)
# Forward
output = self.model(self.processed_image)
# Target specific class
class_loss = -output[0, self.target_class]
print('Iteration:', str(i), 'Loss',
"{0:.2f}".format(class_loss.data.numpy()[0]))
# Zero grads
self.model.zero_grad()
# Backward
class_loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(self.processed_image)
# Save image
cv2.imwrite('../generated/c_specific_iteration_' + str(i) + '.jpg',
self.created_image)
return self.processed_image
def visualise_layer_with_hooks(self):
# Hook the selected layer
self.hook_layer() # 여기서 선택된 필터
# Generate a random image
random_image = np.uint8(np.random.uniform(150, 180, (224, 224, 3)))
# Process image and return variable
processed_image = preprocess_image(
random_image, False) # 이미지 제로민 노말라이즈, torch tensor 형태로 만듬
# Define optimizer for the image
optimizer = Adam([processed_image], lr=0.1, weight_decay=1e-6)
for i in range(1, 31):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = processed_image
for index, layer in enumerate(self.model):
# Forward pass layer by layer
# x is not used after this point because it is only needed to trigger
# the forward hook function
x = layer(x)
# print(">>>",x.size()) => 여기서 forward 해서 계속 feature map이 나온다.
# Only need to forward until the selected layer is reached
if index == self.selected_layer:
# (forward hook function triggered) => for 문나갈 떄 hook trigger 된다.
break
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
# print(self.conv_output.size())
loss = -torch.mean(
self.conv_output) # 특정 filter 의 output의 mean을 backprob으로 미니마이즈
exit()
print('Iteration:', str(i), 'Loss:',
"{0:.2f}".format(loss.data.numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(
processed_image) # 다시 이미지 형태로 복원
# Save image
if i % 5 == 0:
im_path = '../generated/layer_vis_l' + str(self.selected_layer) + \
'_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg'
save_image(self.created_image, im_path)
def visualise_layer_without_hooks(self):
# Process image and return variable
# Generate a random image
random_image = np.uint8(np.random.uniform(150, 180, (224, 224, 3)))
# Process image and return variable
processed_image = preprocess_image(random_image, False)
# Define optimizer for the image
optimizer = Adam([processed_image], lr=0.1, weight_decay=1e-6)
for i in range(1, 31):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = processed_image
for index, layer in enumerate(self.model):
# Forward pass layer by layer
x = layer(x)
if index == self.selected_layer:
# Only need to forward until the selected layer is reached
# Now, x is the output of the selected layer
break
# Here, we get the specific filter from the output of the convolution operation
# x is a tensor of shape 1x512x28x28.(For layer 17)
# So there are 512 unique filter outputs
# Following line selects a filter from 512 filters so self.conv_output will become
# a tensor of shape 28x28
self.conv_output = x[0, self.selected_filter]
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = -torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:',
"{0:.2f}".format(loss.data.numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(processed_image)
# Save image
if i % 5 == 0:
im_path = '../generated/layer_vis_l' + str(self.selected_layer) + \
'_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg'
save_image(self.created_image, im_path)
def visualise_layer_without_hooks(self):
# Process image and return variable
self.processed_image = preprocess_image(self.created_image)
# Define optimizer for the image
# Earlier layers need higher learning rates to visualize whereas later layers need less
optimizer = SGD([self.processed_image], lr=5, weight_decay=1e-6)
for i in range(1, 51):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = self.processed_image
for index, layer in enumerate(self.model):
# Forward pass layer by layer
x = layer(x)
if index == self.selected_layer:
# Only need to forward until the selected layer is reached
# Now, x is the output of the selected layer
break
# Here, we get the specific filter from the output of the convolution operation
# x is a tensor of shape 1x512x28x28.(For layer 17)
# So there are 512 unique filter outputs
# Following line selects a filter from 512 filters so self.conv_output will become
# a tensor of shape 28x28
self.conv_output = x[0, self.selected_filter]
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:',
"{0:.2f}".format(loss.data.numpy()[0]))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(self.processed_image)
# Save image
if i % 5 == 0:
cv2.imwrite(
'generated/layer_vis_l' + str(self.selected_layer) + '_f' +
str(self.selected_filter) + '_iter' + str(i) + '.jpg',
self.created_image)
def visualise_layer_with_hooks(self):
# Hook the selected layer
self.hook_layer()
# Process image and return variable
self.processed_image = preprocess_image(self.created_image)
# Define optimizer for the image
# Earlier layers need higher learning rates to visualize whereas later layers need less
optimizer = SGD([self.processed_image], lr=5, weight_decay=1e-6)
for i in range(1, 51):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = self.processed_image
for index, layer in enumerate(self.model):
# Forward pass layer by layer
# x is not used after this point because it is only needed to trigger
# the forward hook function
x = layer(x)
# Only need to forward until the selected layer is reached
if index == self.selected_layer:
# (forward hook function triggered)
break
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:',
"{0:.2f}".format(loss.data.numpy()[0]))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(self.processed_image)
# Save image
if i % 5 == 0:
cv2.imwrite(
'generated/layer_vis_l' + str(self.selected_layer) + '_f' +
str(self.selected_filter) + '_iter' + str(i) + '.jpg',
self.created_image)
if os.path.isdir(vis_args.DATASET.path):
files = os.listdir(vis_args.DATASET.path)
paths = [os.path.join(vis_args.DATASET.path, f) for f in files]
elif os.path.isfile(vis_args.DATASET.path):
files = list(vis_args.DATASET.path.split("/")[-1])
paths = [vis_args.DATASET.path]
# Poining Game Constructor
if vis_args.RESULTS.POINTING_GAME.state:
from src.pointing_game import PointingGame
tolerance = vis_args.RESULTS.POINTING_GAME.tolerance
pg = PointingGame(vis_args.MODEL.n_classes, tolerance=tolerance)
for f, path in tqdm.tqdm(zip(files, paths)):
img = open_image(path)
prep_img = preprocess_image(img, h=h, w=w)
guided_grads = GBP.generate_gradients(prep_img,
vis_args.DATASET.target_class)
pos_sal, neg_sal = get_positive_negative_saliency(guided_grads)
bw_pos_sal = convert_to_grayscale(pos_sal)
bw_neg_sal = convert_to_grayscale(neg_sal)
if vis_args.RESULTS.POINTING_GAME.state:
boxes = get_boxes(vis_args.RESULTS.DRAW_GT_BBOX.gt_src, f, img)
hit = pg.evaluate(boxes, bw_pos_sal.squeeze())
_ = pg.accumulate(hit[0], 1)
prep_img = prep_img[0].mean(dim=0, keepdim=True)
r_pos = alpha * bw_pos_sal + (1 - alpha) * prep_img.detach().numpy()
r_neg = alpha * bw_neg_sal + (1 - alpha) * prep_img.detach().numpy()