def run(filename_in, filename_out):
# parameters of CNN-VLM model
model_name = 'vggnet'
layer_names = [
'data',
'conv1_2',
'conv2_2',
'conv3_4',
'conv4_4',
'conv5_4',
]
directory = './theanomodel'
filename = (
'%s/%s_vlm_%s.pkl' %
(directory, model_name, '_'.join(layer_names))
)
# parameters of saliency map
layer_target = 'conv5_4'
pool_size = 2**4
pow_ = 0.5
sigma = 0.03
# load model
model = pickle.load(open(filename))
func = theano.function(
[model['data']],
model['loss_lm_%s_map' % layer_target]
)
img_mean = utils.load_mean_image()
# load image
img1 = utils.load_image(filename_in, img_mean)
# compute unnaturalness map
img2 = func(img1[None, :, :, :]).squeeze()
# pow
img2 = img2 ** pow_
# resize to original size
img2 = rescale_image(img2, pool_size)
img2 = pad_image(img2, img1.shape[1:])
# blur
img2 = scipy.ndimage.filters.gaussian_filter(
img2,
(sigma*img2.shape[0], sigma*img2.shape[1]),
)
# normalize
img2 = (img2 - img2.min()) / (img2.max() - img2.min())
img2 = (img2 * 255).astype(np.uint8)
# save
scipy.misc.imsave(filename_out, img2)
def run(
model_name='alexnet',
layer_names=[
'data',
'conv1',
'conv2',
'conv3',
'conv4',
'conv5',
'fc6', 'fc7', 'fc8'
],
image_size=(227, 227),
):
directory = './theanomodel'
filename_save = '%s/%s_mean_std_%s.pkl'
filename_train = './image/valid/*.JPEG'
filename_model = '%s/%s.model' % (directory, model_name)
# load model
img_mean = utils.load_mean_image()
model = pickle.load(open(filename_model))
# get filename
filename_train = glob.glob(filename_train)
# get function to compute mean and mean**2
func = get_function_mean_var(model, layer_names)
# for all images
means = {layer_name: [] for layer_name in layer_names}
vars_ = {layer_name: [] for layer_name in layer_names}
for i, fn in enumerate(filename_train):
print 'computing mean: %d / %d' % (i, len(filename_train))
img = utils.load_image(fn, img_mean)
img = utils.clip_image(img, image_size)
mvs = func(img[None, :, :, :])
for j, layer_name in enumerate(layer_names):
means[layer_name].append(mvs[j*2])
vars_[layer_name].append(mvs[j*2+1])
# save
for layer_name in layer_names:
mean = np.vstack(means[layer_name]).mean(0)
std = (np.vstack(vars_[layer_name]).mean(0) - mean**2)**0.5
data = {
'mean': mean,
'std': std,
}
filename = filename_save % (directory, model_name, layer_name)
pickle.dump(
data,
open(filename, 'wb'),
protocol=pickle.HIGHEST_PROTOCOL,
)
def run(filename_in, filename_out):
# parameters of CNN-VLM model
model_name = 'vggnet'
layer_names = [
'data',
'conv1_2',
'conv2_2',
'conv3_4',
'conv4_4',
'conv5_4',
]
directory = './theanomodel'
filename = ('%s/%s_vlm_%s.pkl' %
(directory, model_name, '_'.join(layer_names)))
# parameters of saliency map
layer_target = 'conv5_4'
pool_size = 2**4
pow_ = 0.5
sigma = 0.03
# load model
model = pickle.load(open(filename))
func = theano.function([model['data']],
model['loss_lm_%s_map' % layer_target])
img_mean = utils.load_mean_image()
# load image
img1 = utils.load_image(filename_in, img_mean)
# compute unnaturalness map
img2 = func(img1[None, :, :, :]).squeeze()
# pow
img2 = img2**pow_
# resize to original size
img2 = rescale_image(img2, pool_size)
img2 = pad_image(img2, img1.shape[1:])
# blur
img2 = scipy.ndimage.filters.gaussian_filter(
img2,
(sigma * img2.shape[0], sigma * img2.shape[1]),
)
# normalize
img2 = (img2 - img2.min()) / (img2.max() - img2.min())
img2 = (img2 * 255).astype(np.uint8)
# save
scipy.misc.imsave(filename_out, img2)
def run(
model_name='alexnet',
layer_names=['data', 'conv1', 'conv2', 'conv3', 'conv4', 'conv5'],
image_size=(227, 227),
num_samples=100000,
):
directory = './theanomodel'
filename_save = '%s/%s_pca_%s.pkl'
filename_mean_std = '%s/%s_mean_std_%s.pkl'
filename_train = './image/valid/*.JPEG'
filename_model = '%s/%s.model' % (directory, model_name)
# load model
img_mean = utils.load_mean_image()
model = pickle.load(open(filename_model))
# get filename
filename_train = glob.glob(filename_train)
num_samples_per_image = num_samples / len(filename_train) + 1
# get function to sample
func = get_function_sample(model, layer_names, num_samples_per_image)
# sample
samples = {layer_name: [] for layer_name in layer_names}
for i, filename in enumerate(filename_train):
print 'training PCA: %d / %d' % (i, len(filename_train))
img = utils.load_image(filename, img_mean)
img = utils.clip_image(img, image_size)
s = func(img[None, :, :, :])
for j, layer_name in enumerate(layer_names):
samples[layer_name].append(s[j])
# PCA
for layer_name in layer_names:
filename = filename_mean_std % (directory, model_name, layer_name)
mean_std = pickle.load(open(filename))
samples[layer_name] = np.vstack(samples[layer_name])
samples[layer_name] = (
(samples[layer_name] - mean_std['mean'][None, :]) /
mean_std['std'][None, :]
)
pca = sklearn.decomposition.PCA(whiten=False)
pca.fit(samples[layer_name])
filename = filename_save % (directory, model_name, layer_name)
pickle.dump(
pca,
open(filename, 'wb'),
protocol=pickle.HIGHEST_PROTOCOL,
)
def run(
model_name='alexnet',
layer_names=['data', 'conv1', 'conv2', 'conv3', 'conv4', 'conv5'],
image_size=(227, 227),
num_samples=100000,
):
directory = './theanomodel'
filename_save = '%s/%s_pca_%s.pkl'
filename_mean_std = '%s/%s_mean_std_%s.pkl'
filename_train = './image/valid/*.JPEG'
filename_model = '%s/%s.model' % (directory, model_name)
# load model
img_mean = utils.load_mean_image()
model = pickle.load(open(filename_model))
# get filename
filename_train = glob.glob(filename_train)
num_samples_per_image = num_samples / len(filename_train) + 1
# get function to sample
func = get_function_sample(model, layer_names, num_samples_per_image)
# sample
samples = {layer_name: [] for layer_name in layer_names}
for i, filename in enumerate(filename_train):
print 'training PCA: %d / %d' % (i, len(filename_train))
img = utils.load_image(filename, img_mean)
img = utils.clip_image(img, image_size)
s = func(img[None, :, :, :])
for j, layer_name in enumerate(layer_names):
samples[layer_name].append(s[j])
# PCA
for layer_name in layer_names:
filename = filename_mean_std % (directory, model_name, layer_name)
mean_std = pickle.load(open(filename))
samples[layer_name] = np.vstack(samples[layer_name])
samples[layer_name] = (
(samples[layer_name] - mean_std['mean'][None, :]) /
mean_std['std'][None, :])
pca = sklearn.decomposition.PCA(whiten=False)
pca.fit(samples[layer_name])
filename = filename_save % (directory, model_name, layer_name)
pickle.dump(
pca,
open(filename, 'wb'),
protocol=pickle.HIGHEST_PROTOCOL,
)
def run(
model_name='alexnet',
layer_names=['data', 'conv1', 'conv2', 'conv3', 'conv4', 'conv5'],
layer_sizes=[3, 96, 256, 384, 384, 256],
batch_size=16,
lr=1e+1,
max_iter=20000,
image_size=(227, 227),
save_interval=1000,
):
directory = './theanomodel'
filename_train = './image/valid/*.JPEG'
filename_save = '%s/%s_vlm_%s.pkl' % (directory, model_name,
'_'.join(layer_names))
lr_reduce_factor = 0.1
lr_reduce_interval = 5000
# load model
img_mean = utils.load_mean_image()
model, tparams = build_model(model_name, layer_names, layer_sizes)
# build solver
solver = solvers.SGD(
model['loss_lm'],
[model['data']],
tparams.values(),
lr=lr,
)
# get filename
filename_train = glob.glob(filename_train)
# train
loss = []
for iter_ in range(max_iter + 1):
# load images
imgs = []
for filename in np.random.choice(filename_train, batch_size):
img = utils.load_image(filename, img_mean)
img = utils.clip_image(img, image_size)
imgs.append(img)
imgs = np.array(imgs)
# update
l = solver.update(imgs)
loss.append(l)
print 'training VLM: %d / %d %f' % (iter_, max_iter, l)
# reduce learning rate
if iter_ != 0 and iter_ % lr_reduce_interval == 0:
solver.reset()
solver.lr.set_value(solver.lr.get_value() *
np.array(lr_reduce_factor).astype(np.float32))
# save
if iter_ % save_interval == 0:
print 'average loss:', np.mean(loss[-save_interval:])
sys.setrecursionlimit(100000)
pickle.dump(
model,
open(filename_save, 'wb'),
protocol=pickle.HIGHEST_PROTOCOL,
)
def run(
model_name='alexnet',
layer_names=['data', 'conv1', 'conv2', 'conv3', 'conv4', 'conv5'],
layer_sizes=[3, 96, 256, 384, 384, 256],
batch_size=16,
lr=1e+1,
max_iter=20000,
image_size=(227, 227),
save_interval=1000,
):
directory = './theanomodel'
filename_train = './image/valid/*.JPEG'
filename_save = '%s/%s_vlm_%s.pkl' % (directory, model_name, '_'.join(layer_names))
lr_reduce_factor = 0.1
lr_reduce_interval = 5000
# load model
img_mean = utils.load_mean_image()
model, tparams = build_model(model_name, layer_names, layer_sizes)
# build solver
solver = solvers.SGD(
model['loss_lm'],
[model['data']],
tparams.values(),
lr=lr,
)
# get filename
filename_train = glob.glob(filename_train)
# train
loss = []
for iter_ in range(max_iter+1):
# load images
imgs = []
for filename in np.random.choice(filename_train, batch_size):
img = utils.load_image(filename, img_mean)
img = utils.clip_image(img, image_size)
imgs.append(img)
imgs = np.array(imgs)
# update
l = solver.update(imgs)
loss.append(l)
print 'training VLM: %d / %d %f' % (iter_, max_iter, l)
# reduce learning rate
if iter_ != 0 and iter_ % lr_reduce_interval == 0:
solver.reset()
solver.lr.set_value(
solver.lr.get_value() *
np.array(lr_reduce_factor).astype(np.float32)
)
# save
if iter_ % save_interval == 0:
print 'average loss:', np.mean(loss[-save_interval:])
sys.setrecursionlimit(100000)
pickle.dump(
model,
open(filename_save, 'wb'),
protocol=pickle.HIGHEST_PROTOCOL,
)
def run(filename_in, filename_out):
# parameters
lambda_layer = [1e-0, 1e-1, 1e-2, 1e-3, 1e-4]
lambda_loss = [1, 10]
learning_rate = 2**19
momentum = 0.9
max_iter = 1000
model_name = 'alexnet'
image_size = (227, 227)
layer_names = [
'data',
'conv1',
'conv2',
'conv3',
'conv4',
'conv5',
]
directory = './theanomodel'
filename_reconstructor = ('%s/%s_vlm_%s_reconstructor.pkl' %
(directory, model_name, '_'.join(layer_names)))
filename_cnn = ('%s/%s.model' % (directory, model_name))
filename_mean_std = ('%s/%s_mean_std_data.pkl' % (directory, model_name))
# compute target feature
cnn = pickle.load(open(filename_cnn))
img_mean = utils.load_mean_image()
func = theano.function([cnn['data']], cnn['fc8'])
img = utils.load_image(filename_in, img_mean)
target = func(img[None, :, :, :]).flatten()
# load reconstructor
print 'loading reconstructor (which takes several tens of minutes)'
reconstructor = pickle.load(open(filename_reconstructor))
# set hyper-parameters
reconstructor['target'].set_value(target.astype(np.float32))
reconstructor['lambda_layer'].set_value(
np.array(lambda_layer).astype(np.float32))
reconstructor['lambda_loss'].set_value(
np.array(lambda_loss).astype(np.float32))
reconstructor['solver'].lr.set_value(learning_rate)
reconstructor['solver'].momentum.set_value(momentum)
# init solver
ms = pickle.load(open(filename_mean_std))
x_init = (np.random.randn(1, 3, image_size[0], image_size[1]) *
ms['std'][None, :, None, None] +
ms['mean'][None, :, None, None]).astype(np.float32)
reconstructor['x'].set_value(x_init)
reset_solver(reconstructor['solver'])
# optimize
for i in range(max_iter):
loss = reconstructor['solver'].update()
print 'iter %d / %d, loss %f' % (i, max_iter, loss)
# save
img = reconstructor['x'].get_value().squeeze()
img = (img + img_mean.mean(1).mean(1)[:, None, None])
img = img.swapaxes(0, 2).swapaxes(0, 1)
img = np.clip(img, 0, 255).astype(np.uint8)
scipy.misc.imsave(filename_out, img[:, :, ::-1])
