def iterative_grad_attack(inception_model,
source_tensor,
target_tensor,
n_steps=200,
lr=0.01):
with torch.no_grad():
target_rep = inception_model(target_tensor).detach()
perturbed_tensor = source_tensor.clone()
for step in range(1, n_steps + 1):
grad, similarity = cal_source_grad(inception_model, perturbed_tensor,
target_rep)
perturbed_tensor = perturbed_tensor + lr * grad
perturbed_tensor = torch.clamp(perturbed_tensor, -1.0, 1.0).detach_()
if similarity > 0.99:
break
adv_rep = inception_model(perturbed_tensor)
rep_dist = (target_rep * adv_rep).sum(dim=1).mean().cpu().item()
adv_img = tensor_to_image(
fixed_image_standardization_inverse(perturbed_tensor.cpu()).squeeze(0))
tgt_img = tensor_to_image(
fixed_image_standardization_inverse(target_tensor.cpu()).squeeze(0))
pixel_dist = compute_dist(np.asarray(adv_img), np.asarray(tgt_img))
return adv_img, pixel_dist, rep_dist
def validation_loop(dataloader, model, loss_fn, epoch, rgb_map: dict,
batch_size: int) -> float:
"""
Validate the neural network with a given loss_fn and optimizer
:param dataloader: a dataloader class that returns random batched image and mask tensor pairs
:param model: the torch nn model to be trained
:param loss_fn: the loss function used as to calculate the 'error' between X and y
:param epoch: number of times we have gone through the loop
:param rgb_map: rgb_map used in original image
:param batch_size: number of batches used to train on
:return: avg_loss
"""
size = len(dataloader)
validation_loss, correct, jaccard_loss = [], [], []
with torch.no_grad():
for i_batch, sample_batched in enumerate(dataloader):
X = sample_batched['image']
y = sample_batched['mask']
pred = model(X)
y = y.to(device)
validation_loss.append(loss_fn(pred, y).item())
correct.append(
(pred.argmax(1) == y).type(torch.float).sum().item() /
(y.shape[1] * y.shape[2]))
# jaccard_loss.append(smp.utils.functional.iou(pred, y).item())
# writer.add_histogram("accuracy-distribution/train", (pred.argmax(1) == y).type(torch.float).sum().item()
# / (y.shape[1] * y.shape[2]), i_batch)
if i_batch % 20 == 1:
pred_fig = tensor_to_image(pred, rgb_map, True)
y_fig = tensor_to_image(y, rgb_map, False)
X_fig = tensor_image_to_image(X)
writer.add_figure('prediction/' + str(i_batch),
pred_fig,
global_step=epoch)
writer.add_figure('truth/' + str(i_batch),
y_fig,
global_step=epoch)
writer.add_figure('input/' + str(i_batch),
X_fig,
global_step=epoch)
validation_mean = get_sample_mean(validation_loss)
correct_mean = get_sample_mean(correct)
# jarrard_mean = get_sample_mean(jaccard_loss)
print(
f"Validation Error: \n Accuracy: {(100 * correct_mean / batch_size):>0.1f}% Avg loss: "
f"{validation_mean / batch_size:>8f} \n")
writer.add_scalar("val/avg accuracy", correct_mean / batch_size, epoch)
writer.add_scalar("val/avg loss", validation_mean / batch_size, epoch)
# writer.add_scalar("val/avg IoU", jarrard_mean, epoch)
# TODO: Jacquard-loss is doing something weird, IoU doesn't look correct
return validation_mean
def disrupt_stimuli(save_folder, inputs_and_targets, encoder, roi_mask,
target_roi, loss_method):
roi_mask = torch.from_numpy(roi_mask.astype(np.uint8))
towards_target = True if loss_method == 'towards' else False
if not target_roi:
loss_func = roi_loss_func(None, towards_target)
else:
loss_func = roi_loss_func(roi_mask, towards_target)
for input_and_target in tqdm(inputs_and_targets):
orig_image = input_and_target['stimulus_path']
orig_image = utils.image_to_tensor(orig_image)
target = input_and_target['target_voxels']
target = torch.from_numpy(target)
if not target_roi:
target = target[roi_mask]
else:
with torch.no_grad():
orig_voxels = encoder(orig_image.unsqueeze(0)).squeeze(0)
target[1 - roi_mask] = orig_voxels[1 - roi_mask]
disrupted_image = deepdream(orig_image, target, encoder, loss_func)
metrics = loss_metrics(orig_image, disrupted_image, target, encoder,
roi_mask if target_roi else None)
disrupted_image = utils.tensor_to_image(disrupted_image)
save_disrupted_image(save_folder, input_and_target, disrupted_image,
metrics)
async def on_reaction_add(reaction: discord.Reaction,
user: discord.User) -> None:
"""Handle user reactions to select attributes."""
# The reaction should not come from the bot and
# should be on the bots message
if user.id != bot.id and reaction.message.id == bot.sent_panel.id:
await bot.sent_panel.remove_reaction(reaction, user)
emoji = str(reaction)
if emoji in bot.label_emojis:
label_idx = bot.label_emojis.index(emoji)
# Change the label to its opposite
bot.labels[0][label_idx] = float(not bot.labels[0][label_idx])
# Edit panel
await bot.sent_panel.edit(embed=get_info_panel())
if emoji == bot.go_emoji:
# Generate image
generated_data = bot.generator(bot.image_data, bot.labels)
generated_image = tensor_to_image(generated_data, bot.config)
# To upload the image, convert to bytes
# https://stackoverflow.com/questions/59868527
with io.BytesIO() as image_binary:
generated_image.save(image_binary, 'png')
image_binary.seek(0)
image_file = discord.File(fp=image_binary,
filename='gen_img.png')
ctx = reaction.message.channel
bot.sent_image = await ctx.send(file=image_file)
def run_test(args):
it_network = ImageTransformNet(
input_shape=hparams['test_size'],
residual_layers=hparams['residual_layers'],
residual_filters=hparams['residual_filters'])
ckpt_dir = os.path.join(args.name, 'pretrained')
ckpt = tf.train.Checkpoint(network=it_network, step=tf.Variable(0))
ckpt.restore(tf.train.latest_checkpoint(ckpt_dir)).expect_partial()
print('\n###################################################')
print('Perceptual Losses for Real-Time Style Transfer Test')
print('###################################################\n')
print('Restored {} step: {}\n'.format(args.name, str(ckpt.step.numpy())))
dir_size = 'step_{}_{}x{}'.format(str(ckpt.step.numpy()),
str(hparams['test_size'][0]),
str(hparams['test_size'][1]))
dir_model = 'output_img_{}'.format(args.name)
out_dir = os.path.join(args.output_path, dir_model, dir_size)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
content_img_list = os.listdir(args.test_content_img)
for c_file in content_img_list:
content = convert(os.path.join(args.test_content_img, c_file),
hparams['test_size'][:2])[tf.newaxis, :]
output = it_network(content, training=False)
tensor = tensor_to_image(output)
c_name = '{}_{}'.format(args.name, os.path.splitext(c_file)[0])
save_path = os.path.join(out_dir, c_name)
tensor.save(save_path + '.jpeg')
print('Image: {}.jpeg saved'.format(save_path))
def iterative_grad_attack(inception_model,
source_img,
target_img,
n_steps=50,
lr=0.01):
with torch.no_grad():
target_rep = inception_model(target_img).detach()
perturbed_img = source_img.clone()
for step in range(1, n_steps + 1):
grad, similarity = cal_source_grad(inception_model, perturbed_img,
target_rep)
# print('grad range ', grad.max(), grad.min())
perturbed_img = perturbed_img + lr * grad
perturbed_img = torch.clamp(perturbed_img, -1.0, 1.0).detach_()
if similarity > 0.99:
break
# if step % 50 == 1:
# print('step {}, similarity: {:.4f}'.format(step, similarity.item()))
adv_rep = inception_model(perturbed_img)
rep_dist = (target_rep * adv_rep).sum(dim=1).mean().cpu().item()
adv_imgs = [
tensor_to_image(img.squeeze(0))
for img in fixed_image_standardization_inverse(
perturbed_img.cpu()).split(split_size=1, dim=0)
]
target_imgs = [
tensor_to_image(img.squeeze(0))
for img in fixed_image_standardization_inverse(target_img.cpu()).split(
split_size=1, dim=0)
]
dist_list = [
compute_dist(np.asarray(adv_img), np.asarray(target_img))
for (adv_img, target_img) in zip(adv_imgs, target_imgs)
]
dist = np.mean(dist_list)
return adv_imgs, dist, rep_dist
def train(optimizer, model, target, content_features, style_weights, style_grams, content_weight, style_weight, steps=2000, show_every=200):
for ii in tqdm(range(1, steps+1)):
# get the features from your target image
target_features = get_features(target, model)
# the content loss
content_loss = torch.mean((target_features['conv4_2'] - content_features['conv4_2'])**2)
# the style loss
# initialize the style loss to 0
style_loss = 0
# then add to it for each layer's gram matrix loss
for layer in style_weights:
# get the "target" style representation for the layer
target_feature = target_features[layer]
target_gram = gram_matrix(target_feature)
_, d, h, w = target_feature.shape
# get the "style" style representation
style_gram = style_grams[layer]
# the style loss for one layer, weighted appropriately
layer_style_loss = style_weights[layer] * torch.mean((target_gram - style_gram)**2)
# add to the style loss
style_loss += layer_style_loss / (d * h * w)
# calculate the *total* loss
total_loss = content_weight * content_loss + style_weight * style_loss
# update your target image
optimizer.zero_grad()
total_loss.backward()
optimizer.step()
# display intermediate images and print the loss
if ii % show_every == 0:
print('Total loss: ', total_loss.item())
plt.imshow(tensor_to_image(target))
plt.show()
return target
b2a_sample_array = np.zeros((__EVAL_NUM * im_h, (__SAMPLE_NUM + 1) * im_w, 3),
dtype=np.uint8)
pbar = tqdm(total=__EVAL_NUM)
with torch.no_grad():
for i, (image_a, image_b, _) in enumerate(dataloader):
image_a = image_a.cuda()
image_b = image_b.cuda()
pbar.update(1)
if i == __EVAL_NUM: break
a2b_sample_array[i*im_h:(i+1)*im_h, 0:im_w, :] = \
tensor_to_image(image_a)
b2a_sample_array[i*im_h:(i+1)*im_h, 0:im_w, :] = \
tensor_to_image(image_b)
for j in range(__SAMPLE_NUM):
output_images = trainer.sample(image_a, image_b, training=False)
im_array_a2b = tensor_to_image(output_images[3])
im_array_b2a = tensor_to_image(output_images[-1])
a2b_sample_array[i*im_h:(i+1)*im_h, (j+1)*im_w:(j+2)*im_w, :] = \
im_array_a2b
b2a_sample_array[i*im_h:(i+1)*im_h, (j+1)*im_w:(j+2)*im_w, :] = \
im_array_b2a
results = extractor(tf.constant(content_image))
print('Styles:')
print_stats_of_layers(results['style'].items())
print("Contents:")
print_stats_of_layers(results['content'].items())
style_targets = extractor(style_image)['style']
content_targets = extractor(content_image)['content']
image = tf.Variable(content_image)
opt = tf.optimizers.Adam(learning_rate=0.02, beta_1=0.99, epsilon=1e-1)
targs_style = style_targets, style_weight, num_style_layers
targs_content = content_targets, content_weight, num_content_layers
start = time.time()
step = 0
for n in range(epochs):
for m in range(steps_per_epoch):
step += 1
train_step(image, extractor, opt, targs_style, targs_content)
print(".", end='')
if save_progress:
tensor_to_image(image).save(output_file[:-4] + str(n) + '.png')
print("Train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end - start))
tensor_to_image(image).save(output_file)
for step in progress:
optimizer.zero_grad()
net_output = net(net_input)
net_output_ds = ds_net(net_output)
loss = MSE(net_output_ds, target)
loss.backward()
optimizer.step()
train_loss = loss.data[0]
if step % 100 == 0:
if use_cuda:
net_output = net_output.cpu()
filename = None if args.output is None else '%s/%04d.png' % (
args.output, step)
img = tensor_to_image(net_output.data, filename)
psnr = compare_psnr(truth, np.array(img))
max_psnr = max(max_psnr, psnr)
progress.set_description(
'Loss: %.6f | PSNR: %.2f dB | Max PSNR: %.2f dB' %
(train_loss, psnr, max_psnr))
if use_cuda:
noise_new = torch.cuda.FloatTensor(noise.shape).normal_(std=sigma)
else:
noise_new = torch.FloatTensor(noise.shape).normal_(std=sigma)
net_input.data += noise_new
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument('--content', type=str,
dest='content_path',
help='content image path',
required=True)
parser.add_argument('--style', type=str,
dest='style_path',
help='style image path',
required=True)
parser.add_argument('--output', type=str,
dest='dest',
help='output path',
required=True)
parser.add_argument('--content-weight', type=float,
dest='content_weight',
help='content weight',
default=CONTENT_WEIGHT)
parser.add_argument('--style-weight', type=float,
dest='style_weight',
help='style weight',
default=STYLE_WEIGHT)
parser.add_argument('--tv-weight', type=float,
dest='tv_weight',
help='total variation weight',
default=TV_WEIGHT)
parser.add_argument('--learning-rate', type=float,
dest='learning_rate',
help='learning rate',
default=LEARNING_RATE)
parser.add_argument('--epochs', type=int,
dest='epochs',
help='number of epochs',
default=NUM_EPOCHS)
parser.add_argument('--len-epoch', type=int,
dest='len_epoch',
help='steps per epoch',
default=STEPS_PER_EPOCH)
args = parser.parse_args()
content_image = utils.load_image(args.content_path)
style_image = utils.load_image(args.style_path)
content_layers = ['block5_conv2']
style_layers = ['block1_conv1',
'block2_conv1',
'block3_conv1',
'block4_conv1',
'block5_conv1']
model = StyleTransfer(content_layers, style_layers)
optimizer = tf.optimizers.Adam(learning_rate=args.learning_rate,
beta_1=0.99, epsilon=1e-1)
trainer = Trainer(model, style_content_loss, optimizer, content_image,
style_image, args.content_weight, args.style_weight,
args.tv_weight, args.epochs, args.len_epoch)
image = tf.Variable(content_image)
trainer.train(image)
utils.tensor_to_image(image).save(args.dest)
ret, frame = cap.read()
if ret == False:
break
cv2.imwrite('images_transformed/' + str(i) + '.jpg', frame)
i += 1
cap.release()
cv2.destroyAllWindows()
imgs = sorted(os.listdir('images_dissected/'))
for img in imgs:
content_image = utils.load_img('images_dissected/' + img)
stylized_image = hub_module(tf.constant(content_image),
tf.constant(style_image))[0]
copy = utils.tensor_to_image(stylized_image)
copy.save('images_transformed/transformed_' + img)
imgs_transformed = sorted(os.listdir('images_transformed/'))
img_array = []
for img in imgs_transformed:
image = cv2.imread('images_transformed/' + img)
height, width, layers = image.shape
size = (width, height)
img_array.append(image)
out = cv2.VideoWriter('project.avi', cv2.VideoWriter_fourcc(*'DIVX'), 30, size)
# Write the file
for i in range(len(img_array)):
parser.add_argument('--save_folder', required=True, type=str, help='folder to save disrupted images')
parser.add_argument('--encoder_file', required=True, type=str, help='name of the encoder file')
parser.add_argument('--targets_folder', default=None, type=str,
help='folder containing voxel targets (if not provided, activation will be maximized)')
args = parser.parse_args()
shutil.rmtree(args.save_folder, ignore_errors=True)
os.mkdir(args.save_folder)
encoder = torch.load(os.path.join('saved_models', args.encoder_file))
if args.targets_folder is not None:
print('Generating targeted stimuli')
targets = os.listdir(args.targets_folder)
targets = [t for t in targets if t != '.DS_Store']
targets = [t for t in targets if '.target.pth' in t]
for target_name in tqdm(targets):
target = torch.load(os.path.join(args.targets_folder, target_name))
generated, metrics = generate_stimulus(target, encoder, towards_target=True)
utils.tensor_to_image(generated).save(os.path.join(args.save_folder, target_name.split('.')[0] + '.png'))
with open(os.path.join(args.save_folder, target_name.split('.')[0] + '_metrics.json'), 'w') as f:
f.write(json.dumps(metrics, indent=2))
else:
print('Generating untargeted stimuli')
for i in tqdm(range(20)):
target = torch.zeros(encoder.regressor.linear.out_features)
generated, metrics = generate_stimulus(target, encoder, towards_target=False)
utils.tensor_to_image(generated).save(os.path.join(args.save_folder, '{:05d}.png'.format(i)))
with open(os.path.join(args.save_folder, '{:05d}_metrics.json'.format(i)), 'w') as f:
f.write(json.dumps(metrics, indent=2))
stimuli = os.listdir(args.stimuli_folder)
stimuli = [s for s in stimuli if s != '.DS_Store']
roi_mask = utils.get_roi_mask(args.roi, args.encoder_file)
encoder = torch.load(os.path.join('saved_models', args.encoder_file))
print('Generating disruption examples')
for stimulus_name in tqdm(stimuli):
stimulus = utils.image_to_tensor(
os.path.join(args.stimuli_folder, stimulus_name), resolution)
target = torch.load(
os.path.join(args.targets_folder, stimulus_name + '.target.pth'))
disrupted, metrics = disrupt_stimulus(stimulus, target, encoder,
roi_mask, args.towards_target,
args.random)
shutil.copyfile(
os.path.join(args.stimuli_folder, stimulus_name),
os.path.join(args.save_folder,
stimulus_name).replace('.', '_original.'))
utils.tensor_to_image(disrupted).save(
os.path.join(args.save_folder,
stimulus_name.replace('.', '_disrupted.')))
with open(
os.path.join(args.save_folder,
stimulus_name.split('.')[0] + '_metrics.json'),
'w') as f:
f.write(json.dumps(metrics, indent=2))
def main(config):
logger = config.get_logger('test')
# setup data_loader instances
data_loader = getattr(module_data, config['data_loader']['type'])(
config['data_loader']['args']['data_dir'],
batch_size=8,
shuffle=False,
validation_split=0.0,
training=False,
num_workers=2)
# build model architecture
model = config.init_obj('arch', module_arch)
logger.info(model)
# get function handles of loss and metrics
loss_fn = getattr(module_loss, config['loss'])
metric_fns = [getattr(module_metric, met) for met in config['metrics']]
logger.info('Loading checkpoint: {} ...'.format(config.resume))
checkpoint = torch.load(config.resume)
state_dict = checkpoint['state_dict']
if config['n_gpu'] > 1:
model = torch.nn.DataParallel(model)
model.load_state_dict(state_dict)
# prepare model for testing
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = model.to(device)
model.eval()
total_loss = 0.0
total_metrics = torch.zeros(len(metric_fns))
MAX_IMAGES = 20
current = 0
with torch.no_grad():
for i, (data, target) in enumerate(tqdm(data_loader)):
data, target = data.to(device), target.to(device)
output = model(data)
for i in range(data.shape[0]):
if current <= MAX_IMAGES:
current += 1
pred = torch.sigmoid(output.squeeze(dim=1)[i, :, :]) > 0.5
pred = tensor_to_image(pred)
true = tensor_to_image(target[i, :, :])
pred.save(f'images/{current}-pred.png')
true.save(f'images/{current}-true.png')
# computing loss, metrics on test set
loss = loss_fn(output, target)
batch_size = data.shape[0]
total_loss += loss.item() * batch_size
for i, metric in enumerate(metric_fns):
total_metrics[i] += metric(output, target) * batch_size
n_samples = len(data_loader.sampler)
log = {'loss': total_loss / n_samples}
log.update({
met.__name__: total_metrics[i].item() / n_samples
for i, met in enumerate(metric_fns)
})
logger.info(log)
type=int,
help='iterations to update image')
opt = parser.parse_args()
if opt.use_cuda:
# checking if cuda is available
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# load in content and style image
content = image_loader(opt.content_path, shape=(512, 512)).to(device)
style = image_loader(opt.style_path, shape=(512, 512)).to(device)
# display images
if opt.show_img:
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 10))
ax1.imshow(tensor_to_image(content))
ax2.imshow(tensor_to_image(style))
# weights for each style layer
# weighting earlier layers more will result in *larger* style artifacts
# notice we are excluding `conv4_2` our content representation
style_weights = {
'conv1_1': 1.,
'conv2_1': 0.75,
'conv3_1': 0.2,
'conv4_1': 0.2,
'conv5_1': 0.2
}
content_weight = opt.content_weight # alpha
style_weight = opt.style_weight # beta
import matplotlib.pyplot as plt
from pycocotools.coco import COCO
train_path = '/home/user/Data/coco2014/train2014'
train_ann_file = '/home/user/Data/coco2014/annotations/instances_train2014.json'
val_path = '/home/user/Data/coco2014/val2014'
val_ann_file = '/home/user/Data/coco2014/annotations/instances_val2014.json'
coco = COCO(train_ann_file)
num_labels = 80
transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
train_dataset = CocoDataset(train_path, train_ann_file, transform, num_labels)
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=2, shuffle=True)
images, labels = next(iter(train_loader))
print(f'images.shape: {images.shape}')
print(labels.shape)
img = tensor_to_image(images[0])
class_names = get_classes_from_labels(labels[0])
print(class_names)
plt.imshow(img)
plt.show()
