plt.figure(3)
image_diff = np.absolute(activations_orig - activations_adv)
converted_diff = convert_to_pixel_space(image_diff, max_idx)
visualize_img(converted_diff)
return min_idx, max_idx
if __name__ == "__main__":
input_var = T.tensor4('inputs')
target_var = T.ivector('targets')
_, _, classes, mean_image, values = load_weights("/Users/mihaileric/Documents/CS231N/CS231N-FinalProject/vgg19.pkl")
model = build_model(input_var)
lasagne.layers.set_all_param_values(model['prob'], values)
layer_name = "conv1_2"
# Original image
# Change path to images as appropriate
#orig_filename = "/Users/mihaileric/Documents/CS231N/CS231N-FinalProject/datasets/nipun/orig_high_sea slug, nudibranch_0.94_jigsaw puzzle_0.97.png"
#adv_filename = "/Users/mihaileric/Documents/CS231N/CS231N-FinalProject/datasets/nipun/high_sea slug, nudibranch_0.94_jigsaw puzzle_0.97.png"
#_, max_idx = visualize_max_filter_activations(model, layer_name, orig_filename, adv_filename)
# Expects inputs of (num_sample, channels, H, W) dim
# Reconstruct features for original image
#reconstruct_features(model, layer_name, input_var, orig_filename, filter_num=125)
import urllib
import lasagne
from lasagne.layers import InputLayer, DenseLayer, DropoutLayer
from lasagne.layers.conv import Conv2DLayer as ConvLayer
from lasagne.layers import MaxPool2DLayer as PoolLayer
from lasagne.layers import LocalResponseNormalization2DLayer as NormLayer
from lasagne.utils import floatX
from models import vgg19
if __name__ == "__main__":
classes, mean_image, values = vgg19.load_data()
net = vgg19.build_model()
output_layer = net["prob"]
lasagne.layers.set_all_param_values(output_layer, values)
image_urls = vgg19.get_images(3)
count = 0
for url in image_urls:
print url
try:
rawim, im = prep_image(url, mean_image)
prob = np.array(lasagne.layers.get_output(output_layer, im, deterministic=True).eval())
top5 = np.argsort(prob[0])[-1:-6:-1]
fig = plt.figure()
plt.imshow(rawim.astype('uint8'))
def train(model_='VGG_16'):
batch_size = 16
nb_samples = 50
nb_classes = 5
nb_test_samples = 1000
nb_epoch = 50
data_augmentation = False
if os.name in ['nt']:
train_pkl = r'C:\Users\crobe\Google Drive\DataMiningGroup\Datasets\restaurant_photos_with_labels_train.pkl'
test_pkl = r'C:\Users\crobe\Google Drive\DataMiningGroup\Datasets\restaurant_photos_with_labels_test.pkl'
img_path = r'D:\Yelp\restaurant_photos\\'
CLASSES = pickle.load(open(r'C:\Users\crobe\Google Drive\DataMiningGroup\Code\data\categories.pkl', 'rb'))
model_path = r'C:\Users\crobe\Google Drive\DataMiningGroup\Datasets\\'
else:
train_pkl = r'/home/rcamachobarranco/datasets/restaurant_photos_with_labels_train.pkl'
test_pkl = r'/home/rcamachobarranco/datasets/restaurant_photos_with_labels_test.pkl'
img_path = r'/home/rcamachobarranco/datasets/restaurant_photos/'
CLASSES = pickle.load(open(r'/home/rcamachobarranco/code/data/categories.pkl', 'rb'))
model_path = r'/home/rcamachobarranco/datasets/'
# input image dimensions
if model_ in MODELS[0:3]:
img_rows, img_cols = 224, 224
if model_ in MODELS[3]:
img_rows, img_cols = 299, 299
# generate model
if model_ in MODELS[0]:
net = vgg16.build_model()
weights = pickle.load(open(model_path + 'vgg16.pkl', 'rb'))
net_prob = net['prob']
elif model_ in MODELS[1]:
net = vgg19.build_model()
weights = pickle.load(open(model_path + 'vgg19.pkl', 'rb'))
net_prob = net['prob']
elif model_ in MODELS[2]:
net = googlenet.build_model()
weights = pickle.load(open(model_path + 'googlenet.pkl', 'rb'))
net_prob = net['prob']
elif model_ in MODELS[3]:
net = inception_v3.build_network()
weights = pickle.load(open(model_path + 'inception_v3.pkl', 'rb'))
net_prob = net['softmax']
lasagne.layers.set_all_param_values(net_prob, weights['param values'])
logging.info('Finished setting up weights')
if model_ in MODELS[0]:
output_layer = DenseLayer(net['fc7'], num_units=len(CLASSES), nonlinearity=softmax)
elif model_ in MODELS[1]:
output_layer = DenseLayer(net['fc7_dropout'], num_units=len(CLASSES), nonlinearity=softmax)
elif model_ in MODELS[2]:
net['classifier'] = DenseLayer(net['pool5/7x7_s1'],
num_units=len(CLASSES),
nonlinearity=linear)
output_layer = NonlinearityLayer(net['classifier'],
nonlinearity=softmax)
elif model_ in MODELS[3]:
output_layer = DenseLayer(net['pool3'], num_units=len(CLASSES), nonlinearity=softmax)
# Define loss function and metrics, and get an updates dictionary
X_sym = T.tensor4()
y_sym = T.ivector()
prediction = lasagne.layers.get_output(output_layer, X_sym)
loss = lasagne.objectives.categorical_crossentropy(prediction, y_sym)
loss = loss.mean()
acc = T.mean(T.eq(T.argmax(prediction, axis=1), y_sym),
dtype=theano.config.floatX)
params = lasagne.layers.get_all_params(output_layer, trainable=True)
updates = lasagne.updates.nesterov_momentum(
loss, params, learning_rate=0.001, momentum=0.9)
logging.info('Created model, compiling functions')
# Compile functions for training, validation and prediction
train_fn = theano.function([X_sym, y_sym], loss, updates=updates)
val_fn = theano.function([X_sym, y_sym], [loss, acc])
pred_fn = theano.function([X_sym], prediction)
logging.info('Compiled functions, creating batchgen object')
with BatchGen(batch, seed=1, num_workers=1, input_pkl=train_pkl, img_path=img_path,
dtype=np.float32, pixels=img_rows, model=model_, batch_size=batch_size) as train_bg:
with BatchGen(batch, seed=1, num_workers=1, input_pkl=test_pkl, img_path=img_path,
dtype=np.float32, pixels=img_rows, model=model_, batch_size=batch_size) as test_bg:
logging.info('Created batchgen object')
for epoch in range(nb_epoch):
for sample in range(nb_samples):
sample = next(train_bg)
X_train = sample['x']
y_train = sample['y']
train_loss = train_fn(X_train, y_train)
loss_tot = 0.
acc_tot = 0.
for chunk in range(nb_test_samples):
test_sample = next(test_bg)
X_test = test_sample['x']
y_test = test_sample['y']
test_loss, acc = val_fn(X_test, y_test)
loss_tot += test_loss
acc_tot += acc
loss_tot /= nb_test_samples
acc_tot /= nb_test_samples
if acc_tot > 0.86:
np.savez('/home/rcamachobarranco/datasets/googlenet_{0:.4g}.npz'.format(acc_tot * 100), *lasagne.layers.get_all_param_values(output_layer))
#param_values = lasagne.layers.get_all_params(output_layer)
#pickle.dump(param_values, open(r'/home/rcamacho/dm/datasets/googlenet_{0:.4g}.pkl'.format(acc_tot * 100), 'wb'),
#protocol=pickle.HIGHEST_PROTOCOL)
logging.info('Epoch {0} Train_loss {1} Test_loss {2} Test_accuracy {3}'.format(epoch, train_loss, loss_tot, acc_tot * 100))
sys.path.append("/Users/mihaileric/Documents/Research/Lasagne")
import urllib
import lasagne
from lasagne.layers import InputLayer, DenseLayer, DropoutLayer
from lasagne.layers.conv import Conv2DLayer as ConvLayer
from lasagne.layers import MaxPool2DLayer as PoolLayer
from lasagne.layers import LocalResponseNormalization2DLayer as NormLayer
from lasagne.utils import floatX
from models import vgg19
if __name__ == "__main__":
classes, mean_image, values = vgg19.load_data()
net = vgg19.build_model()
output_layer = net["prob"]
lasagne.layers.set_all_param_values(output_layer, values)
image_urls = vgg19.get_images(3)
count = 0
for url in image_urls:
print url
try:
rawim, im = prep_image(url, mean_image)
prob = np.array(
lasagne.layers.get_output(output_layer, im,
deterministic=True).eval())
top5 = np.argsort(prob[0])[-1:-6:-1]
img = img.transpose(1, 2, 0)
return np.clip(img, 0, 255).astype('uint8')
# Loading and preprocessing the images, and setting up the theano symbolic variables
content_img, content_mean = getImage('images/big_photo2.jpg')
style_img, style_mean = getImage('images/big_art2.jpg')
# The content, style, and generated images. Unlike the paper, I am not randomly initializing the generated image.
CON = theano.shared(np.asarray(content_img, dtype=theano.config.floatX))
STY = theano.shared(np.asarray(style_img, dtype=theano.config.floatX))
X = T.vector()
GEN = T.reshape(X, content_img.shape)
# GEN = theano.shared(np.asarray(content_img, dtype=theano.config.floatX))
# Here we build the VGG model, and get the filter responses of each image from the designated layers (not all will be used)
model = build_model()
layer_names = ['conv1_1', 'conv2_1', 'conv3_1', 'conv4_1', 'conv5_1']
CON_F = {name: lasagne.layers.get_output(model[name], inputs=CON) for name in layer_names}
STY_F = {name: lasagne.layers.get_output(model[name], inputs=STY) for name in layer_names}
GEN_F = {name: lasagne.layers.get_output(model[name], inputs=GEN) for name in layer_names}
# Evaluate those filter responses and save as theano shared variables (so that we don't have to evaluate them over and over)
CON_F = {name: theano.shared(F.eval()) for name, F in CON_F.items()}
STY_F = {name: theano.shared(F.eval()) for name, F in STY_F.items()}
def content_loss(A, B):
# Returns the sum of squared errors of A and B
return T.sum(T.sqr(A - B)) / 2.
def gram_matrix(M):
def train(model_='VGG_16'):
batch_size = 16
nb_samples = 50
nb_classes = 5
nb_test_samples = 1000
nb_epoch = 50
data_augmentation = False
# Initialize paths relative to our folder
train_pkl = r'./data/restaurant_photos_with_labels_train.pkl'
test_pkl = r'./data/restaurant_photos_with_labels_test.pkl'
img_path = r'./data/restaurant_photos/'
CLASSES = pickle.load(open(r'./data/categories.pkl', 'rb'))
model_path = r'./data/models/'
# input image dimensions
if model_ in MODELS[0:3]:
img_rows, img_cols = 224, 224
if model_ in MODELS[3]:
img_rows, img_cols = 299, 299
# generate model
if model_ in MODELS[0]:
net = vgg16.build_model()
weights = pickle.load(open(model_path + 'vgg16.pkl', 'rb'))
net_prob = net['prob']
elif model_ in MODELS[1]:
net = vgg19.build_model()
weights = pickle.load(open(model_path + 'vgg19.pkl', 'rb'))
net_prob = net['prob']
elif model_ in MODELS[2]:
net = googlenet.build_model()
weights = pickle.load(open(model_path + 'googlenet.pkl', 'rb'))
net_prob = net['prob']
elif model_ in MODELS[3]:
net = inception_v3.build_network()
weights = pickle.load(open(model_path + 'inception_v3.pkl', 'rb'))
net_prob = net['softmax']
lasagne.layers.set_all_param_values(net_prob, weights['param values'])
logging.info('Finished setting up weights')
# Depending on the model is where we will connect our new classifier
if model_ in MODELS[0]:
output_layer = DenseLayer(net['fc7'],
num_units=len(CLASSES),
nonlinearity=softmax)
elif model_ in MODELS[1]:
output_layer = DenseLayer(net['fc7_dropout'],
num_units=len(CLASSES),
nonlinearity=softmax)
elif model_ in MODELS[2]:
net['classifier'] = DenseLayer(net['pool5/7x7_s1'],
num_units=len(CLASSES),
nonlinearity=linear)
output_layer = NonlinearityLayer(net['classifier'],
nonlinearity=softmax)
elif model_ in MODELS[3]:
output_layer = DenseLayer(net['pool3'],
num_units=len(CLASSES),
nonlinearity=softmax)
# Define loss function and metrics, and get an updates dictionary
X_sym = T.tensor4()
y_sym = T.ivector()
prediction = lasagne.layers.get_output(output_layer, X_sym)
loss = lasagne.objectives.categorical_crossentropy(prediction, y_sym)
loss = loss.mean()
acc = T.mean(T.eq(T.argmax(prediction, axis=1), y_sym),
dtype=theano.config.floatX)
# Define the model parameters as trainable
params = lasagne.layers.get_all_params(output_layer, trainable=True)
# Define the parameter updates
updates = lasagne.updates.nesterov_momentum(loss,
params,
learning_rate=0.001,
momentum=0.9)
logging.info('Created model, compiling functions')
# Compile functions for training, validation and prediction
train_fn = theano.function([X_sym, y_sym], loss, updates=updates)
val_fn = theano.function([X_sym, y_sym], [loss, acc])
pred_fn = theano.function([X_sym], prediction)
logging.info('Compiled functions, creating batchgen object')
# Using batch generation, train the dataset
with BatchGen(batch,
seed=1,
num_workers=1,
input_pkl=train_pkl,
img_path=img_path,
dtype=np.float32,
pixels=img_rows,
model=model_,
batch_size=batch_size) as train_bg:
with BatchGen(batch,
seed=1,
num_workers=1,
input_pkl=test_pkl,
img_path=img_path,
dtype=np.float32,
pixels=img_rows,
model=model_,
batch_size=batch_size) as test_bg:
logging.info('Created batchgen object')
for epoch in range(nb_epoch):
for sample in range(nb_samples):
sample = next(train_bg)
X_train = sample['x']
y_train = sample['y']
# Train current chunk
train_loss = train_fn(X_train, y_train)
loss_tot = 0.
acc_tot = 0.
# Test for many more samples
for chunk in range(nb_test_samples):
test_sample = next(test_bg)
X_test = test_sample['x']
y_test = test_sample['y']
test_loss, acc = val_fn(X_test, y_test)
loss_tot += test_loss
acc_tot += acc
# Compute the test loss and accuracy
loss_tot /= nb_test_samples
acc_tot /= nb_test_samples
# Store the model if accuracy is above 86%
if acc_tot > 0.86:
np.savez(
r'./data/googlenet_model.npz',
*lasagne.layers.get_all_param_values(output_layer))
logging.info(
'Epoch {0} Train_loss {1} Test_loss {2} Test_accuracy {3}'
.format(epoch, train_loss, loss_tot, acc_tot * 100))
def train(model_='VGG_16'):
batch_size = 16
nb_samples = 50
nb_classes = 5
nb_test_samples = 1000
nb_epoch = 50
data_augmentation = False
# Initialize paths relative to our folder
train_pkl = r'./data/restaurant_photos_with_labels_train.pkl'
test_pkl = r'./data/restaurant_photos_with_labels_test.pkl'
img_path = r'./data/restaurant_photos/'
CLASSES = pickle.load(open(r'./data/categories.pkl', 'rb'))
model_path = r'./data/models/'
# input image dimensions
if model_ in MODELS[0:3]:
img_rows, img_cols = 224, 224
if model_ in MODELS[3]:
img_rows, img_cols = 299, 299
# generate model
if model_ in MODELS[0]:
net = vgg16.build_model()
weights = pickle.load(open(model_path + 'vgg16.pkl', 'rb'))
net_prob = net['prob']
elif model_ in MODELS[1]:
net = vgg19.build_model()
weights = pickle.load(open(model_path + 'vgg19.pkl', 'rb'))
net_prob = net['prob']
elif model_ in MODELS[2]:
net = googlenet.build_model()
weights = pickle.load(open(model_path + 'googlenet.pkl', 'rb'))
net_prob = net['prob']
elif model_ in MODELS[3]:
net = inception_v3.build_network()
weights = pickle.load(open(model_path + 'inception_v3.pkl', 'rb'))
net_prob = net['softmax']
lasagne.layers.set_all_param_values(net_prob, weights['param values'])
logging.info('Finished setting up weights')
# Depending on the model is where we will connect our new classifier
if model_ in MODELS[0]:
output_layer = DenseLayer(net['fc7'], num_units=len(CLASSES), nonlinearity=softmax)
elif model_ in MODELS[1]:
output_layer = DenseLayer(net['fc7_dropout'], num_units=len(CLASSES), nonlinearity=softmax)
elif model_ in MODELS[2]:
net['classifier'] = DenseLayer(net['pool5/7x7_s1'],
num_units=len(CLASSES),
nonlinearity=linear)
output_layer = NonlinearityLayer(net['classifier'],
nonlinearity=softmax)
elif model_ in MODELS[3]:
output_layer = DenseLayer(net['pool3'], num_units=len(CLASSES), nonlinearity=softmax)
# Define loss function and metrics, and get an updates dictionary
X_sym = T.tensor4()
y_sym = T.ivector()
prediction = lasagne.layers.get_output(output_layer, X_sym)
loss = lasagne.objectives.categorical_crossentropy(prediction, y_sym)
loss = loss.mean()
acc = T.mean(T.eq(T.argmax(prediction, axis=1), y_sym),
dtype=theano.config.floatX)
# Define the model parameters as trainable
params = lasagne.layers.get_all_params(output_layer, trainable=True)
# Define the parameter updates
updates = lasagne.updates.nesterov_momentum(
loss, params, learning_rate=0.001, momentum=0.9)
logging.info('Created model, compiling functions')
# Compile functions for training, validation and prediction
train_fn = theano.function([X_sym, y_sym], loss, updates=updates)
val_fn = theano.function([X_sym, y_sym], [loss, acc])
pred_fn = theano.function([X_sym], prediction)
logging.info('Compiled functions, creating batchgen object')
# Using batch generation, train the dataset
with BatchGen(batch, seed=1, num_workers=1, input_pkl=train_pkl, img_path=img_path,
dtype=np.float32, pixels=img_rows, model=model_, batch_size=batch_size) as train_bg:
with BatchGen(batch, seed=1, num_workers=1, input_pkl=test_pkl, img_path=img_path,
dtype=np.float32, pixels=img_rows, model=model_, batch_size=batch_size) as test_bg:
logging.info('Created batchgen object')
for epoch in range(nb_epoch):
for sample in range(nb_samples):
sample = next(train_bg)
X_train = sample['x']
y_train = sample['y']
# Train current chunk
train_loss = train_fn(X_train, y_train)
loss_tot = 0.
acc_tot = 0.
# Test for many more samples
for chunk in range(nb_test_samples):
test_sample = next(test_bg)
X_test = test_sample['x']
y_test = test_sample['y']
test_loss, acc = val_fn(X_test, y_test)
loss_tot += test_loss
acc_tot += acc
# Compute the test loss and accuracy
loss_tot /= nb_test_samples
acc_tot /= nb_test_samples
# Store the model if accuracy is above 86%
if acc_tot > 0.86:
np.savez(r'./data/googlenet_model.npz', *lasagne.layers.get_all_param_values(output_layer))
logging.info('Epoch {0} Train_loss {1} Test_loss {2} Test_accuracy {3}'.format(epoch, train_loss, loss_tot, acc_tot * 100))
plt.figure(3)
image_diff = np.absolute(activations_orig - activations_adv)
converted_diff = convert_to_pixel_space(image_diff, max_idx)
visualize_img(converted_diff)
return min_idx, max_idx
if __name__ == "__main__":
input_var = T.tensor4('inputs')
target_var = T.ivector('targets')
_, _, classes, mean_image, values = load_weights(
"/Users/mihaileric/Documents/CS231N/CS231N-FinalProject/vgg19.pkl")
model = build_model(input_var)
lasagne.layers.set_all_param_values(model['prob'], values)
layer_name = "conv1_2"
# Original image
# Change path to images as appropriate
#orig_filename = "/Users/mihaileric/Documents/CS231N/CS231N-FinalProject/datasets/nipun/orig_high_sea slug, nudibranch_0.94_jigsaw puzzle_0.97.png"
#adv_filename = "/Users/mihaileric/Documents/CS231N/CS231N-FinalProject/datasets/nipun/high_sea slug, nudibranch_0.94_jigsaw puzzle_0.97.png"
#_, max_idx = visualize_max_filter_activations(model, layer_name, orig_filename, adv_filename)
# Expects inputs of (num_sample, channels, H, W) dim
# Reconstruct features for original image
#reconstruct_features(model, layer_name, input_var, orig_filename, filter_num=125)
