def executnfn(self):
#style_filename = 'style7.jpg'
style_filename = style_image_path
style_image = self.load_image(style_filename)
content_layer_ids = [4]
style_layer_ids = [1, 2, 3, 4]
content_video_path
global outlocation
#cap = cv2.VideoCapture("rabbit.mp4")
cap = cv2.VideoCapture(content_video_path)
count = 0
#dstnvideoname='output3.avi'
out = cv2.VideoWriter(outlocation, -1, 20.0, (self.w, self.h))
model = vgg16.VGG16()
session = tf.InteractiveSession(graph=model.graph)
img = np.array([1, 1, 1])
self.submit_button.hide()
pb = self.progressBar
loss_style = self.create_style_loss(session=session,
model=model,
style_image=style_image,
layer_ids=style_layer_ids)
loss_denoise = self.create_denoise_loss(model)
while (count <= 211):
ret, frame = cap.read()
count = count + 1
if (count > 100) and count % 10 == 0:
x = int((count - 100) * 100 / 111)
#x=count-100
#print("ch"+str(x))
self.counter.emit(x)
print(x)
QApplication.processEvents()
img = self.style_transfer(content_image=frame,
loss_denoise=loss_denoise,
loss_style=loss_style,
prvsframe=img,
session=session,
model=model,
style_image=style_image,
content_layer_ids=content_layer_ids,
style_layer_ids=style_layer_ids,
weight_content=content_weight_value,
weight_style=style_weight_value,
weight_denoise=99,
weight_temporal=1111,
num_iterations=20,
step_size=8)
out.write(img)
print(count)
pb.hide()
session.close()
cap.release()
out.release()
cv2.destroyAllWindows()
def get_vgg16_model(self):
vgg16_model = vgg16.VGG16(weights='imagenet',
input_shape=(self.input_shape[0],
self.input_shape[1], 3),
pooling='max',
include_top=False)
return vgg16_model
def train():
train_start_time = time.time()
style_img = cv2.imread('comic.jpg')
content_img = cv2.imread('putin.jpg')
#rand_img=np.random.randint(low=0, high=256, size=[512, 512, 3])
rand_img = np.random.rand(512, 512, 3) + 128
rand_img = rand_img.astype(float)
net = vgg16.VGG16()
#print(net.layer_names)
content_layer = [4]
style_layer = range(13)
a = net.get_layer_tensors(layer_ids=content_layer)
#print(a)
feed_dict_content = net.create_feed_dict(image=content_img)
feed_dict_style = net.create_feed_dict(image=style_img)
sess = tf.Session(graph=net.graph)
temp = sess.run(a, feed_dict=feed_dict_content)
print('Temp len:', temp[0].shape)
with net.graph.as_default():
b = tf.constant(temp[0])
print(b.get_shape())
con_l = content_loss(a[0], b)
c = net.get_layer_tensors(layer_ids=style_layer)
temp_1 = sess.run(c, feed_dict=feed_dict_style)
#d=tf.constant(temp_1)
style_l = style_loss(c, temp_1)
denoise_l = create_denoise_loss(net)
#total_loss=style_l+0.0001*con_l+0.00001*denoise_l
total_loss = style_l + 0.0003 * con_l + 0.0001 * denoise_l
num_iter = 1000
grad = tf.gradients(total_loss, net.input)
grad_mag = tf.reduce_mean(tf.reshape(grad, [-1]))
lr = 10000000
for i in range(num_iter):
start_time = time.time()
feed_dict_rand = net.create_feed_dict(image=rand_img)
l, grads, g_m = sess.run([total_loss, grad, grad_mag],
feed_dict=feed_dict_rand)
#print('grads shape:', len(grads))
rand_img -= lr * grads[0].reshape([512, 512, 3])
print('Step:', i, 'Loss:', l, 'Gradients magnitude:', g_m, 'LR:',
lr, 'Time per iter:',
time.time() - start_time, 'Total running time:',
(time.time() - train_start_time) / 60.0)
if i % 5 == 0:
cv2.imwrite('sequence/' + str(i) + 'temp.jpg',
rand_img.clip(0, 255))
print('Writing temp to file....')
if lr > 1000000:
#print('LR update:', lr)
lr = 0.99 * lr
cv2.imwrite('final.jpg', rand_img.clip(0, 255))
print('Total time taken:', time.time() - train_start_time)
def style_transfer(content_image, style_image, content_layer_ids, style_layer_ids, weight_content, weight_style, weight_denoise, num_iterations, step_size):
model = vgg16.VGG16()
session = tf.InteractiveSession(graph=model.graph)
# generate content loss
content_layers = model.get_layer_tensors(content_layer_ids)
content_values = session.run(content_layers, feed_dict=model.create_feed_dict(image=content_image))
with model.graph.as_default():
content_layer_losses = []
for value, layer in zip(content_values, content_layers):
content_layer_losses.append(tf.reduce_mean(tf.square(layer - tf.constant(value))))
loss_content = tf.reduce_mean(content_layer_losses)
# generate style loss
with model.graph.as_default():
style_gram_layers = []
for tensor in model.get_layer_tensors(style_layer_ids):
matrix = tf.reshape(tensor, shape=[-1, int(tensor.get_shape()[3])])
style_gram_layers += [tf.matmul(tf.transpose(matrix), matrix)]
values = session.run(style_gram_layers, feed_dict=model.create_feed_dict(image=style_image))
style_layer_losses = []
for value, gram_layer in zip(values, style_gram_layers):
style_layer_losses.append(tf.reduce_mean(tf.square(gram_layer - tf.constant(value))))
loss_style = tf.reduce_mean(style_layer_losses)
# generate de-noise loss
loss_denoise = tf.reduce_sum(tf.abs(model.input[:,1:,:,:] - model.input[:,:-1,:,:])) + tf.reduce_sum(tf.abs(model.input[:,:,1:,:] - model.input[:,:,:-1,:]))
# calculate difference
adj_content = tf.Variable(1e-10, name='adj_content')
adj_style = tf.Variable(1e-10, name='adj_style')
adj_denoise = tf.Variable(1e-10, name='adj_denoise')
session.run([adj_content.initializer, adj_style.initializer, adj_denoise.initializer])
loss_combined = weight_content * adj_content * loss_content + weight_style * adj_style * loss_style + weight_denoise * adj_denoise * loss_denoise
run_list = [tf.gradients(loss_combined, model.input),
adj_content.assign(1.0 / (loss_content + 1e-10)), # content adjustments
adj_style.assign(1.0 / (loss_style + 1e-10)), # style adjustments
adj_denoise.assign(1.0 / (loss_denoise + 1e-10))] # de-noise adjustments
# apply via gradient
result = np.random.rand(*content_image.shape) + 128
for i in range(num_iterations):
grad, adj_content_val, adj_style_val, adj_denoise_val = session.run(run_list, feed_dict=model.create_feed_dict(image=result))
grad = np.squeeze(grad)
result -= grad * (step_size / (np.std(grad) + 1e-8))
result = np.clip(result, 0.0, 255.0)
session.close()
return result
def init():
if FLAGS.dataset == 'mnist':
import mnist
dataset = mnist.MNIST()
elif FLAGS.dataset == 'cifar10':
import cifar10
dataset = cifar10.CIFAR10()
nH, nW, nD = dataset.dims()
nC = dataset.nC
if FLAGS.model == 'vgg16':
import vgg16
model = vgg16.VGG16(nH, nW, nD, FLAGS.F, nC)
return dataset, model
def extraction_tmp():
#input
mean_bgr = np.array((104.00698793, 116.66876762, 122.67891434))
img = skimage.io.imread('image.png')
img = img.astype(np.float32)
img -= mean_bgr
img = img.transpose((2, 0, 1))
img = img[np.newaxis, :, :, :]
model = vgg16.VGG16()
#chainer.serializers.load_npz(vgg16.pretrained_model, model)
#forward
with chainer.no_backprop_mode():
input_data = chainer.Variable(img)
with chainer.using_config('train', False):
feature = model(input_data)
return feature
def main():
import sys
import vgg16
import imagenet_utils
# from keras.applications import vgg16, imagenet_utils
parser = argparser()
args = parser.parse_args()
image_path = args.image
layer_name = args.layer_name
feature_to_visualize = args.feature
visualize_mode = args.mode
model = vgg16.VGG16(weights='imagenet', include_top=False)
layer_dict = dict([(layer.name, layer) for layer in model.layers])
if not layer_name in layer_dict:
print('Wrong layer name')
sys.exit()
# Load data and preprocess
img = Image.open(image_path)
img_array = np.array(img)
if K.image_dim_ordering() == 'th':
img_array = np.transpose(img_array, (2, 0, 1))
img_array = img_array[np.newaxis, :]
img_array = img_array.astype(np.float)
img_array = imagenet_utils.preprocess_input(img_array)
deconv_network = deconvnet.model.DeconvNetModel(model)
deconv = deconv_network.deconvolve_feature_map(
img_array, layer_name, feature_to_visualize, visualize_mode
)
deconv = deconv - deconv.min()
deconv *= 1.0 / (deconv.max() + 1e-8)
if K.image_dim_ordering() == 'th':
deconv = np.transpose(deconv, (1, 2, 0))
deconv = deconv[:, :, ::-1]
uint8_deconv = (deconv * 255).astype(np.uint8)
img = Image.fromarray(uint8_deconv, 'RGB')
img.save('results/{}_{}_{}.png'.format(layer_name, feature_to_visualize,
visualize_mode))
print(time.ctime())
retrain(epoch, mask_prune)
test()
else:
pass
'''
mask_zero = torch.load('mask_with_batch.dat')
exit()
set_mask(mask_zero, 0, 0)
mask = torch.load('ckpt_clean.dat')
#print(mask[0][0,0])
#exit()
net_clean = torch.load('checkpoint/ckpt_20180813.t0')
net = vgg16.VGG16()
if use_cuda:
#print(torch.cuda.device_count())
net.cuda()
net = torch.nn.DataParallel(net,
device_ids=range(torch.cuda.device_count()))
net_clean.cuda()
net_clean = torch.nn.DataParallel(net_clean,
device_ids=range(
torch.cuda.device_count()))
cudnn.benchmark = True
#net_mask_mul(mask_zero)
#add_network(net,mask)
#net.eval()
net.module.load_state_dict(net_clean.module.state_dict(), strict=False)
def style_transfer(content_image, style_image,
content_layer_ids, style_layer_ids,
weight_content=1.5, weight_style=10.0,
weight_denoise=0.3,
num_iterations=120, step_size=10.0):
# Creating an instance of the pretrained VGG16-model.
model = vgg16.VGG16()
# Creating a TensorFlow session.
session = tf.InteractiveSession(graph=model.graph)
# This is for printing the name of the content-layers.
print("Content layers:")
print(model.get_layer_names(content_layer_ids))
print()
# This is for printing the name of the style-layers
print("Style layers:")
print(model.get_layer_names(style_layer_ids))
print()
# Creating the loss function for the content layers.
loss_content = create_content_loss(session=session,
model=model,
content_image=content_image,
layer_ids=content_layer_ids)
# Creating the loss function for the style layers.
loss_style = create_style_loss(session=session,
model=model,
style_image=style_image,
layer_ids=style_layer_ids)
# Create the denoising loss function of the mixed-image.
loss_denoise = create_denoise_loss(model)
#tensor flow variables to adjuxt the loss functions
adj_content = tf.Variable(1e-10, name='adj_content')
adj_style = tf.Variable(1e-10, name='adj_style')
adj_denoise = tf.Variable(1e-10, name='adj_denoise')
# Initializing of the values for loss functions.
session.run([adj_content.initializer,
adj_style.initializer,
adj_denoise.initializer])
update_adj_content = adj_content.assign(1.0 / (loss_content + 1e-10))
update_adj_style = adj_style.assign(1.0 / (loss_style + 1e-10))
update_adj_denoise = adj_denoise.assign(1.0 / (loss_denoise + 1e-10))
loss_combined = weight_content * adj_content * loss_content + weight_style * adj_style * loss_style + weight_denoise * adj_denoise * loss_denoise
# tensor flow function.
gradient = tf.gradients(loss_combined, model.input)
run_list = [gradient, update_adj_content, update_adj_style, update_adj_denoise]
# The output image is started with a random noise.
mixed_image = np.random.rand(*content_image.shape) + 128
for i in range(num_iterations):
# creating a feed dictionary.
feed_dict = model.create_feed_dict(image=mixed_image)
grad, adj_content_val, adj_style_val, adj_denoise_val = session.run(run_list, feed_dict=feed_dict)
grad = np.squeeze(grad)
step_size_scaled = step_size / (np.std(grad) + 1e-8)
# Updating the image.
mixed_image -= grad * step_size_scaled
# Checking the image has pixel values between 0 and 255.
mixed_image = np.clip(mixed_image, 0.0, 255.0)
print(". ", end="")
# Displaying status.
if (i % 10 == 0) or (i == num_iterations - 1):
print()
print("Iteration:", i)
msg = "Weight Adj. for Content: {0:.2e}, Style: {1:.2e}, Denoise: {2:.2e}"
print(msg.format(adj_content_val, adj_style_val, adj_denoise_val))
# Plotting the images.
plot_images(content_image=content_image,
style_image=style_image,
mixed_image=mixed_image)
print()
print("Final image:")
plot_image_big(mixed_image)
# Closing the TensorFlow session.
session.close()
return mixed_image
if octave_index > 0:
# upscale details from the previous octave
detail_height, detail_width = detail.shape[:2]
# detail = scipy.ndimage.zoom(detail, (1.0/octave_scale, 1.0/octave_scale, 1), order=1)
print('resizing detail from %s to %s' %
(detail.shape, octave_base.shape))
detail = imresize(detail, octave_base.shape[:2])
x = preprocess_image(octave_base + detail)
dream = Input(shape=(3, img_shape[0], img_shape[1]))
if args.model == 'resnet50':
model = resnet50.ResNet50(include_top=False, input_tensor=dream)
elif args.model == 'vgg16':
model = vgg16.VGG16(include_top=False, input_tensor=dream)
elif args.model == 'vgg19':
model = vgg19.VGG19(include_top=False, input_tensor=dream)
elif args.model == 'inception_v3':
model = inception_v3.InceptionV3(include_top=False, input_tensor=dream)
else:
raise 'unknown model ' + args.model
print('Model loaded.')
loss_and_grads = create_loss_function(dream, settings, model, img_shape)
evaluator = Evaluator(loss_and_grads)
# run scipy-based optimization (L-BFGS) over the pixels of the generated image
# so as to minimize the loss
for i in range(10):
print('Start of iteration', i)
def style_transfer(content_image,
style_image,
content_layer_ids,
style_layer_ids,
weight_content=1.5,
weight_style=10.0,
weight_denoise=0.3,
num_iterations=120,
step_size=10.0):
# Create an instance of the VGG16-model. This is done
# in each call of this function, because we will add
# operations to the graph so it can grow very large
# and run out of RAM if we keep using the same instance.
model = vgg16.VGG16()
# Create a TensorFlow-session.
session = tf.InteractiveSession(graph=model.graph)
# Print the names of the content-layers.
print("Content layers:")
print(model.get_layer_names(content_layer_ids))
print()
# Print the names of the style-layers.
print("Style layers:")
print(model.get_layer_names(style_layer_ids))
print()
mod_value = int(num_iterations / 10)
# Create the loss-function for the content-layers and -image.
loss_content = create_content_loss(session=session,
model=model,
content_image=content_image,
layer_ids=content_layer_ids)
# Create the loss-function for the style-layers and -image.
loss_style = create_style_loss(session=session,
model=model,
style_image=style_image,
layer_ids=style_layer_ids)
# Create the loss-function for the denoising of the mixed-image.
loss_denoise = create_denoise_loss(model)
# Create TensorFlow variables for adjusting the values of
# the loss-functions. This is explained below.
adj_content = tf.Variable(1e-10, name='adj_content')
adj_style = tf.Variable(1e-10, name='adj_style')
adj_denoise = tf.Variable(1e-10, name='adj_denoise')
# Initialize the adjustment values for the loss-functions.
session.run([
adj_content.initializer, adj_style.initializer, adj_denoise.initializer
])
# Create TensorFlow operations for updating the adjustment values.
# These are basically just the reciprocal values of the
# loss-functions, with a small value 1e-10 added to avoid the
# possibility of division by zero.
update_adj_content = adj_content.assign(1.0 / (loss_content + 1e-10))
update_adj_style = adj_style.assign(1.0 / (loss_style + 1e-10))
update_adj_denoise = adj_denoise.assign(1.0 / (loss_denoise + 1e-10))
# This is the weighted loss-function that we will minimize
# below in order to generate the mixed-image.
# Because we multiply the loss-values with their reciprocal
# adjustment values, we can use relative weights for the
# loss-functions that are easier to select, as they are
# independent of the exact choice of style- and content-layers.
loss_combined = weight_content * adj_content * loss_content + \
weight_style * adj_style * loss_style + \
weight_denoise * adj_denoise * loss_denoise
# Use TensorFlow to get the mathematical function for the
# gradient of the combined loss-function with regard to
# the input image.
gradient = tf.gradients(loss_combined, model.input)
# List of tensors that we will run in each optimization iteration.
run_list = [gradient, update_adj_content, update_adj_style, \
update_adj_denoise]
# The mixed-image is initialized with random noise.
# It is the same size as the content-image.
mixed_image = np.random.rand(*content_image.shape) + 128
#mixed_image = content_image
for i in range(num_iterations):
print("Iteration:", i)
# Create a feed-dict with the mixed-image.
feed_dict = model.create_feed_dict(image=mixed_image)
# Use TensorFlow to calculate the value of the
# gradient, as well as updating the adjustment values.
grad, adj_content_val, adj_style_val, adj_denoise_val \
= session.run(run_list, feed_dict=feed_dict)
# Reduce the dimensionality of the gradient.
grad = np.squeeze(grad)
# Scale the step-size according to the gradient-values.
step_size_scaled = step_size / (np.std(grad) + 1e-8)
# Update the image by following the gradient.
mixed_image -= grad * step_size_scaled
# Ensure the image has valid pixel-values between 0 and 255.
mixed_image = np.clip(mixed_image, 0.0, 255.0)
# Print a little progress-indicator.
print(". ", end="")
# Display status once every 10 iterations, and the last.
if (i % mod_value == 0) or (i == num_iterations - 1):
print()
print("Iteration:", i)
# Print adjustment weights for loss-functions.
msg = "Weight Adj. for Content: {0:.2e}, Style: {1:.2e}, Denoise: {2:.2e}"
print(msg.format(adj_content_val, adj_style_val, adj_denoise_val))
# Plot the content-, style- and mixed-images.
plot_images(content_image=content_image,
style_image=style_image,
mixed_image=mixed_image)
'''
# Print adjustment weights for loss-functions.
msg = "Weight Adj. for Content: {0:.2e}, Style: {1:.2e}, Denoise: {2:.2e}"
print(msg.format(adj_content_val, adj_style_val, adj_denoise_val))
# Plot the content-, style- and mixed-images.
plot_images(content_image=content_image, style_image=style_image, mixed_image=mixed_image)
'''
print()
print("Final image:")
plot_image_big(mixed_image)
# Close the TensorFlow session to release its resources.
session.close()
# Return the mixed-image.
return mixed_image
def style_transfer(content_image,
style_image,
content_layer_ids,
style_layer_ids,
weight_content=1.5,
weight_style=10,
weight_denoise=0.3,
number_iterations=120,
step_size=10):
model = vgg16.VGG16()
session = tf.InteractiveSession(graph=model.graph)
# Create the loss-function for the content-layers and -image.
loss_content = create_content_loss(session=session,
model=model,
content_image=content_image,
layer_ids=content_layer_ids)
# Create the loss-function for the style-layers and -image.
loss_style = create_style_loss(session=session,
model=model,
style_image=style_image,
layer_ids=style_layer_ids)
# Create the loss-function for the denoising of the mixed-image.
loss_denoise = create_denoise_loss(model)
adj_content = tf.Variable(1e-10, name='adj_content')
adj_style = tf.Variable(1e-10, name='adj_style')
adj_denoise = tf.Variable(1e-10, name='adj_denoise')
session.run([
adj_content.initializer, adj_style.initializer, adj_denoise.initializer
])
update_adj_content = adj_content.assign(1.0 / (loss_content + 1e-10))
update_adj_style = adj_style.assign(1.0 / (loss_style + 1e-10))
update_adj_denoise = adj_denoise.assign(1.0 / (loss_denoise + 1e-10))
loss_combined = weight_content * adj_content * loss_content + weight_style * adj_style * loss_style + weight_denoise * adj_denoise * loss_denoise
gradient = tf.gradients(loss_combined, model.input)
run_list = [
gradient, update_adj_style, update_adj_denoise, update_adj_content
]
mixed_image = np.random.rand(*content_image.shape) + 128
for i in range(number_iterations):
feed_dict = model.create_feed_dict(image=mixed_image)
gradient, adj_content_val, adj_style_val, adj_denoise_val = session.run(
run_list, feed_dict=feed_dict)
grad = np.squeeze(gradient)
step_size_scaled = step_size / (np.std(grad) + 1e-10)
mixed_image -= grad * step_size_scaled
mixed_image = np.clip(mixed_image, 0.0, 255.0)
if (i % 10 == 0) or (i == number_iterations - 1):
print("iterations", i)
msg = "Weight Adj. for Content: {0:.2e}, Style: {1:.2e}, Denoise: {2:.2e}"
print(msg.format(adj_content_val, adj_style_val, adj_denoise_val))
plot_images(content_image=content_image,
style_image=style_image,
mixed_image=mixed_image)
print("final image")
plot_image_big(mixed_image)
session.close()
return mixed_image
def style_transfer(content_image, style_image, mask_image,final_image,
content_layer_ids, style_layer_ids,
weight_content, weight_style, num_iterations, step_size):
model = vgg16.VGG16()
session = tf.InteractiveSession(graph=model.graph)
print("Content layers:")
print(model.get_layer_names(content_layer_ids))
print()
print("Style layers:")
print(model.get_layer_names(style_layer_ids))
print()
loss_content = create_content_loss(session=session,
model=model,
content_image=content_image,
layer_ids=content_layer_ids)
loss_style = create_style_loss(session=session,
model=model,
style_image=style_image,
layer_ids=style_layer_ids)
adj_content = tf.Variable(1e-10, name='adj_content')
adj_style = tf.Variable(1e-10, name='adj_style')
session.run([adj_content.initializer,
adj_style.initializer])
update_adj_content = adj_content.assign(1.0/(loss_content+1e-10))
update_adj_style = adj_style.assign(1.0/(loss_style+1e-10))
loss_combined = weight_content * adj_content * loss_content + weight_style * adj_style * loss_style
gradient = tf.gradients(loss_combined, model.input)
run_list = [gradient, update_adj_content, update_adj_style]
mixed_image = np.random.rand(*content_image.shape) + 128
if mask_image is not None:
mixed_image *= mask_image
if final_image is not None:
mixed_image=final_image
#plot_image_big(mixed_image)
for i in range(num_iterations):
feed_dict = model.create_feed_dict(image=mixed_image)
grad, adj_content_val, adj_style_val = session.run(run_list,feed_dict=feed_dict)
grad = np.squeeze(grad)
step_size_scaled = step_size / (np.std(grad) + 1e-8)
if mask_image is not None:
mixed_image -= grad * step_size_scaled * mask_image
else:
mixed_image -= grad * step_size_scaled
mixed_image = np.clip(mixed_image, 0.0, 255.0)
print(".", end="")
if (i % 33 == 0 ) or (i == num_iterations -1):
print()
print("Iteration:",i)
msg = "Weight Adj. for Content: {0:.2e}, Style: {1:.2e}"
print(msg.format(adj_content_val, adj_style_val))
#plot_images(content_image=content_image,style_image=style_image,mixed_image=mixed_image)
print()
print("Transfer Finishi")
#plot_image_big(mixed_image)
session.close()
return mixed_image
def style_transfer(content_image, style_image,
content_layer_ids, style_layer_ids,
weight_content=1.5, weight_style=10.0,
weight_denoise=0.3,
num_iterations=120, step_size=10.0):
"""
Use gradient descent to find an image that minimizes the
loss-functions of the content-layers and style-layers. This
should result in a mixed-image that resembles the contours
of the content-image, and resembles the colours and textures
of the style-image.
Parameters:
content_image: Numpy 3-dim float-array with the content-image.
style_image: Numpy 3-dim float-array with the style-image.
content_layer_ids: List of integers identifying the content-layers.
style_layer_ids: List of integers identifying the style-layers.
weight_content: Weight for the content-loss-function.
weight_style: Weight for the style-loss-function.
weight_denoise: Weight for the denoising-loss-function.
num_iterations: Number of optimization iterations to perform.
step_size: Step-size for the gradient in each iteration.
"""
# Create an instance of the VGG16-model. This is done
# in each call of this function, because we will add
# operations to the graph so it can grow very large
# and run out of RAM if we keep using the same instance.
model = vgg16.VGG16()
# Create a TensorFlow-session.
session = tf.InteractiveSession(graph=model.graph)
# Print the names of the content-layers.
print("Content layers:")
print(model.get_layer_names(content_layer_ids))
print()
# Print the names of the style-layers.
print("Style layers:")
print(model.get_layer_names(style_layer_ids))
print()
# Create the loss-function for the content-layers and -image.
loss_content = create_content_loss(session=session,
model=model,
content_image=content_image,
layer_ids=content_layer_ids)
# Create the loss-function for the style-layers and -image.
loss_style = create_style_loss(session=session,
model=model,
style_image=style_image,
layer_ids=style_layer_ids)
# Create the loss-function for the denoising of the mixed-image.
loss_denoise = create_denoise_loss(model)
#adjust levels of loss functions, normalize them
#multiply them with a variable
#taking reciprocal values of loss values of content, style, denoising
#small constant to avoid divide by 0
#adjustment value normalizes loss so approximately 1
#weights should be set relative to each other dont depend on layers
#we are using
# Create TensorFlow variables for adjusting the values of
# the loss-functions. This is explained below.
adj_content = tf.Variable(1e-10, name='adj_content')
adj_style = tf.Variable(1e-10, name='adj_style')
adj_denoise = tf.Variable(1e-10, name='adj_denoise')
# Initialize the adjustment values for the loss-functions.
session.run([adj_content.initializer,
adj_style.initializer,
adj_denoise.initializer])
# Create TensorFlow operations for updating the adjustment values.
# These are basically just the reciprocal values of the
# loss-functions, with a small value 1e-10 added to avoid the
# possibility of division by zero.
update_adj_content = adj_content.assign(1.0 / (loss_content + 1e-10))
update_adj_style = adj_style.assign(1.0 / (loss_style + 1e-10))
update_adj_denoise = adj_denoise.assign(1.0 / (loss_denoise + 1e-10))
# This is the weighted loss-function that we will minimize
# below in order to generate the mixed-image.
# Because we multiply the loss-values with their reciprocal
# adjustment values, we can use relative weights for the
# loss-functions that are easier to select, as they are
# independent of the exact choice of style- and content-layers.
loss_combined = weight_content * adj_content * loss_content + \
weight_style * adj_style * loss_style + \
weight_denoise * adj_denoise * loss_denoise
# Use TensorFlow to get the mathematical function for the
# gradient of the combined loss-function with regard to
# the input image. (mixed)
gradient = tf.gradients(loss_combined, model.input)
# List of tensors that we will run in each optimization iteration.
run_list = [gradient, update_adj_content, update_adj_style, \
update_adj_denoise]
# The mixed-image is initialized with random noise.
# It is the same size as the content-image.
#where we first init it
mixed_image = np.random.rand(*content_image.shape) + 128
for i in range(num_iterations):
# Create a feed-dict with the mixed-image.
feed_dict = model.create_feed_dict(image=mixed_image)
# Use TensorFlow to calculate the value of the
# gradient, as well as updating the adjustment values.
grad, adj_content_val, adj_style_val, adj_denoise_val \
= session.run(run_list, feed_dict=feed_dict)
# Reduce the dimensionality of the gradient.
#Remove single-dimensional entries from the shape of an array.
grad = np.squeeze(grad)
# Scale the step-size according to the gradient-values.
#Ratio of weights:updates
#akin to learning rate
step_size_scaled = step_size / (np.std(grad) + 1e-8)
# Update the image by following the gradient.
#gradient descent
mixed_image -= grad * step_size_scaled
# Ensure the image has valid pixel-values between 0 and 255.
#Given an interval, values outside the interval are clipped
#to the interval edges.
mixed_image = np.clip(mixed_image, 0.0, 255.0)
# Display status once every 10 iterations, and the last.
if (i % 10 == 0) or (i == num_iterations - 1):
print("Iteration:", i)
# Print adjustment weights for loss-functions.
msg = "Weight Adj. for Content: {0:.2e}, Style: {1:.2e}, Denoise: {2:.2e}"
print(msg.format(adj_content_val, adj_style_val, adj_denoise_val))
#in larger resolution
# Plot the content-, style- and mixed-images.
#plot_images(content_image=content_image, style_image=style_image, mixed_image=mixed_image)
#print()
#print("Final image:")
#plot_image_big(mixed_image)
# Close the TensorFlow session to release its resources.
session.close()
# Return the mixed-image.
return mixed_image
def exportData(pathName, modelName):
global skipLinear
device = 'cuda' if torch.cuda.is_available() else 'cpu'
if modelName == "vgg16_c10":
model = vgg16.VGG16()
elif modelName == "vgg16_c10_old":
model = vgg16_old.VGG16()
elif modelName == "resnet50_c10":
model = resnet.ResNet50()
elif modelName == "inceptionv3_c10":
model = inception_v3_c10.inception_v3()
elif modelName == "alexnet_in":
#model = torch.hub.load('pytorch/vision', 'alexnet', pretrained=True)
model = alexnet.AlexNet()
skipLinear = True
elif modelName == "vgg16_in":
model = vgg_in.vgg16()
skipLinear = True
elif modelName == "resnet50_in":
model = resnet_in.resnet50()
elif modelName == "transformer_wmt":
translator = transformer_translate.create_model(
) # returns Transformer()
translator = translator.to(device)
model = translator.model
else:
print("Model not supported")
exit(1)
model = model.to(device)
if ("vgg16" in modelName or "alexnet" in modelName):
model = replace_vgg16(model, skipLinear)
elif "transformer" in modelName:
#transformer_train.replace_with_pruned(model, "model", prune_attention=True, prune_only_attention=False)
pass
else:
replace_with_pruned(model, "model", skipLinear)
if "_in" in modelName:
if not torch.distributed.is_initialized():
port = np.random.randint(10000, 65536)
torch.distributed.init_process_group(
backend='nccl',
init_method='tcp://127.0.0.1:%d' % port,
rank=0,
world_size=1)
model = torch.nn.parallel.DistributedDataParallel(model,
device_ids=[0],
output_device=0)
if not ("transformer" in modelName):
model.load_state_dict(torch.load(pathName))
model.eval()
#prune(model, method="cascade", q=1.0)
#layer_num = 0
#save_dir = "/root/hostCurUser/reproduce/DNNsim/net_traces/ResNet50_ImageNet_CSP/"
#lstModules = list( model.named_modules())
#for n, m in model.named_modules():
#for i in range(len(lstModules)):
# if isinstance(lstModules[i], nn.Conv2d) or isinstance(lstModules[i], nn.Linear):
# model[i] = DataExporter(lstModules[i], save_dir, layer_num)
# layer_num += 1
f = open(MODEL_PATH + "model.csv", "w")
fscale = open(SCALE_PATH, "w")
fscale.write(
"Layer name, IFMAP Height, IFMAP Width, Filter Height, Filter Width, Channels, Num Filter, Strides,\n"
)
layer_idx = 0
models = [] # for SparTen
layers = []
acts = []
weights = []
paddings = []
strides = []
idxs = []
def extract(module, input):
#if module.extracted:
# return
if len(input[0].shape) < 4:
if not ("transformer" in modelName):
return
try:
a = input[0].detach().cpu().reshape(1, module.in_features, 1,
1)
except:
a = input[0].detach().cpu().reshape(-1, 1, 1)
a = a[:module.in_features]
a = a.reshape(1, module.in_features, 1, 1)
else:
a = input[0].detach().cpu()
acts.append(a)
if isinstance(module, torch.nn.Conv2d):
layer = module.weight.view(
(module.out_channels, -1)).detach().cpu().numpy()
weight = module.weight.detach().cpu().numpy()
tp = "conv"
stride = str(max(module.stride[0], module.stride[1]))
padding = str(max(module.padding[0], module.padding[1]))
in_channels = module.in_channels
out_channels = module.out_channels
kernel_size = module.kernel_size
padding_st = module.padding
stride_st = module.stride
elif isinstance(module, torch.nn.Linear) and (not skipLinearExport):
layer = module.weight.view(
(module.out_features, -1)).detach().cpu().numpy()
weight = module.weight.detach().cpu().reshape(
module.weight.shape[0], module.weight.shape[1], 1, 1).numpy()
tp = "fc"
stride = str(1)
padding = str(0)
in_channels = module.in_features
out_channels = module.out_features
kernel_size = (1, 1)
padding_st = (0, 0)
stride_st = (1, 1)
else:
print("{} does not exist".format(module))
exit(1)
name = str(module.layer_idx)
weights.append(weight)
paddings.append(int(padding))
strides.append(int(stride))
f.write(name + "," + tp + "," + stride + "," + padding + ",\n")
layers.append(layer)
models.append({
'in_channels': in_channels,
'out_channels': out_channels,
'kernel': kernel_size,
'name': tp + name,
'padding': padding_st,
'weights': weight,
'IFM': a.cpu().numpy(),
'stride': stride_st
})
idxs.append(module.layer_idx)
#module.extracted = True
#replace_with_exporter(model, "model", f, fscale)
for n, m in model.named_modules():
if isinstance(m, torch.nn.Conv2d):
weight = m.weight.detach().cpu().numpy()
if weight.shape[2] != weight.shape[3]:
continue
#weights.append(weight)
#m.extracted = False
m.register_forward_pre_hook(extract)
#name = str(layer_idx)
#tp = "conv"
#stride = str(max(m.stride[0], m.stride[1]))
#padding = str(max(m.padding[0], m.padding[1]))
#paddings.append(int(padding))
#f.write(name+"," + tp+"," + stride+"," + padding + ",\n")
m.layer_idx = layer_idx
layer_idx += 1
elif isinstance(m, torch.nn.Linear):
if not ("transformer" in modelName):
continue
#weight = m.weight.detach().cpu().reshape(m.weight.shape[0], m.weight.shape[1], 1, 1).numpy()
#weights.append(weight)
#m.extracted = False
m.register_forward_pre_hook(extract)
#name = str(layer_idx)
#tp = "fc"
#stride = str(1)
#padding = str(0)
#paddings.append(int(padding))
#f.write(name+"," + tp+"," + stride+"," + padding + ",\n")
m.layer_idx = layer_idx
layer_idx += 1
if "_in" in modelName:
IFM = torch.rand(1, 3, 224, 224).cuda()
model(IFM)
elif "_c10" in modelName:
IFM = torch.rand(1, 3, 32, 32).cuda()
model(IFM)
elif "_wmt" in modelName:
src_seq = [4556, 4562, 4560, 4557, 4712, 1894, 15, 4564, 4620, 0, 5]
pred_seq = translator.translate_sentence(
torch.LongTensor([src_seq]).to(device))
print(pred_seq)
else:
print("Dataset not supported")
exit(1)
print(len(acts))
print(len(weights))
#layers = loadCoeffIdx(pathName, modelName)
with open(ST_PATH + modelName + ".h5", "wb") as f:
pickle.dump(models, f)
sq_ptrs = []
out_channels = []
for layer in layers:
sp, oc = squeezeCoeffIdxTotal(layer, len(layer))
out_channels.append(oc)
#"""
i = 0
for idx in range(len(acts)):
x_save = acts[idx]
weight_save = weights[idx]
np.save(EXPORT_PATH + "act-" + str(idx) + "-0.npy", x_save)
np.save(EXPORT_PATH + "wgt-" + str(idx) + ".npy", weight_save)
x_save[x_save == 0] = int(0)
x_save[x_save != 0] = int(1)
np.savetxt(CX_PATH + "act" + str(idx) + ".csv",
x_save.reshape(-1),
delimiter=",",
fmt="%d")
weight_save[weight_save == 0] = int(0)
weight_save[weight_save != 0] = int(1)
np.savetxt(CX_PATH + "Conv2D_" + str(idx) + ".csv",
weight_save.reshape(weight_save.shape[0], -1),
delimiter=",",
fmt="%d")
if x_save.shape[2] > 1:
name = "Conv" + str(idx) #str(idxs[idx])
IFM_height = str(x_save.shape[2] + (2 * paddings[idx]))
IFM_width = str(x_save.shape[3] + (2 * paddings[idx]))
filt_height = str(weight_save.shape[2])
filt_width = str(weight_save.shape[3])
else:
name = "FC" + str(idx) #str(idxs[idx])
IFM_height = str(1)
IFM_width = str(1)
filt_height = str(1)
filt_width = str(1)
channels = str(weight_save.shape[1])
if ("resnet50" in modelName) and (idx == 4 or idx == 15 or idx == 29
or idx == 49):
num_filt = str(weight_save.shape[0])
else:
num_filt = str(out_channels[i])
i += 1
fscale.write(name + ',\t' + IFM_height + ',\t' + IFM_width + ',\t' +
filt_height + ',\t' + filt_width + ',\t' + channels +
',\t' + num_filt + ',\t' + str(strides[idx]) + ',\n')
fscale.close()
f.close()
#"""
cxf = open(CX_PATH + "ModelShape.txt", "w")
cxf.write("LayerName\tLayerID\tInputShape\tOutputShape\tKernelShape\n")
cx_layers = []
layer_idx = 0
for n, m in model.named_modules():
if isinstance(m, torch.nn.Conv2d):
if m.weight.shape[2] != m.weight.shape[3]:
continue
curr = "Conv\t" + str(layer_idx) + "\t" + str(
tuple(acts[layer_idx].shape)).replace('(', '').replace(
')', '').replace(' ', '') + "\t" + str(
tuple(acts[layer_idx].shape)).replace('(', '').replace(
')', '').replace(' ', '') + "\t" + str(
tuple(
m.weight.shape)).replace('(', '').replace(
')', '').replace(' ', '') + "\n"
cxf.write(curr)
cx_layers.append(curr)
layer_idx += 1
if isinstance(m, torch.nn.Linear) and (not skipLinear):
curr = "FC\t" + str(idxs[layer_idx]) + "\t" + str(
tuple(acts[layer_idx].shape)).replace('(', '').replace(
')', '').replace(' ', '') + "\t" + str(
tuple(acts[layer_idx].shape)).replace('(', '').replace(
')', '').replace(' ', '') + "\t" + str(
tuple(
m.weight.shape)).replace('(', '').replace(
')', '').replace(' ', '') + ",1,1\n"
cxf.write(curr)
cx_layers.append(curr)
layer_idx += 1
if "transformer" in modelName:
for i in range(11):
for j in range(36, 97):
cxf.write(cx_layers[j])
cxf.close()
return
def style_transfer(content_image,
style_image,
content_layer_ids,
style_layer_ids,
weight_content=1.5,
weight_style=10.0,
weight_denoise=0.3,
num_iterations=120,
step_size=10.0):
#instantiate VGG16 model everytime otherwise when we add nodes it will eat up too much RAM
model = vgg16.VGG16()
#make your tf session
session = tf.InteractiveSession(graph=model.graph)
content_loss = create_content_loss(session=session,
model=model,
content_image=content_image,
layer_ids=content_layer_ids)
style_loss = create_style_loss(session=session,
model=model,
style_image=style_image,
layer_ids=style_layer_ids)
denoise_loss = create_denoise_loss(model)
#adjust levels of loss functions
content_adjusted = tf.Variable(1e-10, name='content_adjusted')
style_adjusted = tf.Variable(1e-10, name='style_adjusted')
denoise_adjusted = tf.Variable(1e-10, name='denoise_adjusted')
# Initialize the adjustment values for the loss-functions.
session.run([
content_adjusted.initializer, style_adjusted.initializer,
denoise_adjusted.initializer
])
#creating tf operations to update adjusted values with normalized adjusted values
update_content_adj = content_adjusted.assign(1.0 / (content_loss + 1e-10))
update_style_adj = style_adjusted.assign(1.0 / (style_loss + 1e-10))
update_denoise_adj = denoise_adjusted.assign(1.0 / (denoise_loss + 1e-10))
#weighted loss function we use for our mixed image
loss_combined = weight_content * content_adjusted * content_loss + \
weight_style * style_adjusted * style_loss + \
weight_denoise * denoise_adjusted * denoise_loss
# tensorflow operation to generate gradient for weighted loss function from model input image
gradient = tf.gradients(loss_combined, model.input)
# list of tensor for tensor flow session that we will run in each slow optimization iteration :(
run_tensors = [gradient, update_content_adj, update_style_adj, \
update_denoise_adj]
#initialize mixed image with noise
mixed_image = np.random.rand(*content_image.shape) + 128
for i in range(num_iterations):
feed_dict = model.create_feed_dict(image=mixed_image)
grad, content_adjusted, style_adjusted, denoise_adjusted \
= session.run(run_tensors, feed_dict = feed_dict)
grad = np.squeeze(grad)
#scale step size based on gradient
scaled_step = step_size / (np.std(grad) + 1e-8)
#gradient descent : update image based on gradient
mixed_image -= grad * scaled_step
# ensure the pixel values of numpy array mixed_image have 0 - 255 value
mixed_image = np.clip(mixed_image, 0.0, 255.0)
fname = 'img' + np.str(i) + '.jpeg'
save_image(mixed_image, fname)
#printing progress john each iteration
print(".", end="")
if (i % 10 == 0) or (i == num_iterations - 1):
print()
print("Iteration:", i)
# Print adjustment weights for loss-functions.
msg = "Weight Adj. for Content: {0:.2e}, Style: {1:.2e}, Denoise: {2:.2e}"
print(
msg.format(content_adjusted, style_adjusted, denoise_adjusted))
session.close()
return mixed_image
def style_transfer(content_image,
style_image,
content_layer_ids,
style_layer_ids,
weight_content=1.5,
weight_style=10.0,
weight_denoise=0.3,
num_iterations=120,
step_size=10.0):
model = vgg16.VGG16()
session = tf.InteractiveSession(graph=model.graph)
print("Content layers:")
print(model.get_layer_names(content_layer_ids))
print()
print("Style layers:")
print(model.get_layer_names(style_layer_ids))
print()
loss_content = create_content_loss(session=session,
model=model,
content_image=content_image,
layer_ids=content_layer_ids)
loss_style = create_style_loss(session=session,
model=model,
style_image=style_image,
layer_ids=style_layer_ids)
loss_denoise = create_denoise_loss(model)
adj_content = tf.Variable(1e-10, name='adj_content')
adj_style = tf.Variable(1e-10, name='adj_style')
adj_denoise = tf.Variable(1e-10, name='adj_denoise')
session.run([
adj_content.initializer, adj_style.initializer, adj_denoise.initializer
])
update_adj_content = adj_content.assign(1.0 / (loss_content + 1e-10))
update_adj_style = adj_style.assign(1.0 / (loss_style + 1e-10))
update_adj_denoise = adj_denoise.assign(1.0 / (loss_denoise + 1e-10))
loss_combined = weight_content * adj_content * loss_content + \
weight_style * adj_style * loss_style + \
weight_denoise * adj_denoise * loss_denoise
gradient = tf.gradients(loss_combined, model.input)
run_list = [gradient, update_adj_content, update_adj_style, \
update_adj_denoise]
mixed_image = np.random.rand(*content_image.shape) + 128
for i in range(num_iterations):
feed_dict = model.create_feed_dict(image=mixed_image)
grad, adj_content_val, adj_style_val, adj_denoise_val \
= session.run(run_list, feed_dict=feed_dict)
grad = np.squeeze(grad)
step_size_scaled = step_size / (np.std(grad) + 1e-8)
mixed_image -= grad * step_size_scaled
mixed_image = np.clip(mixed_image, 0.0, 255.0)
print(". ", end="")
if (i % 10 == 0) or (i == num_iterations - 1):
print()
print("Iteration:", i)
# Print adjustment weights for loss-functions.
msg = "Weight Adj. for Content: {0:.2e}, Style: {1:.2e}, Denoise: {2:.2e}"
print(msg.format(adj_content_val, adj_style_val, adj_denoise_val))
plot_images(content_image=content_image,
style_image=style_image,
mixed_image=mixed_image)
print()
print("Final image:")
plot_image_big(mixed_image)
session.close()
return mixed_image
content[:, :, 0] -= 103.939
content[:, :, 1] -= 116.779
content[:, :, 2] -= 123.68
content = content[:, :, ::-1]
style[:, :, 0] -= 103.939
style[:, :, 1] -= 116.779
style[:, :, 2] -= 123.68
style = style[:, :, ::-1]
content_weight = 0.025
style_weight = 5.0
total_variation_weight = 1.0
vgg_16 = vgg16.VGG16()
content_layer = vgg_16.get_content_layer_tensors()
def calculate_content_loss():
feed_dict_1 = vgg_16.create_feed_dict(image=content)
content_layer_values = sess.run(content_layer, feed_dict=feed_dict_1)
return tf.reduce_sum(tf.square(content_layer - content_layer_values))
def gram_matrix(tensor):
#matrix = tf.contrib.keras.backend.batch_flatten(tf.contrib.keras.backend.permute_dimensions(tensor, [2, 0, 1]))
num_channels = tf.shape(tensor)[3]
matrix = tf.reshape(tensor, shape=[-1, num_channels])
gram = tf.matmul(tf.transpose(matrix), matrix)
print input_mat.shape
incorrect_and_wrong_prediction_word2Vec_pearson_dict[word]=input_mat
print incorrect_and_wrong_prediction_word2Vec_pearson_dict
joblib.dump(incorrect_and_wrong_prediction_word2Vec_pearson_dict,"/Users/Dhanush/Desktop/CNNStudy/word2vecpearson.pkl")
"""
word2Vec_pearson_dict=joblib.load("/Users/Dhanush/Desktop/CNNStudy/word2vecpearson.pkl")
correct_vocab=joblib.load("/Users/Dhanush/Desktop/CNNStudy/correctwords.pkl")
incorrect_vocab=joblib.load("/Users/Dhanush/Desktop/CNNStudy/incorrectwords.pkl")
print len(incorrect_vocab)
j=0
correct_cnn_vector =[]
layer='block5_conv1'
print layer
model = vgg16.VGG16(include_top=True, weights='imagenet')
"""
for path in paths:
if vocab[j] not in correct_vocab:
j+=1
continue
print vocab[j],j
img = image.load_img(path, target_size=(224, 224))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
vec= get_act_vector(path,model,layer)[0]
vec=vec.flatten()
vec = vec.tolist()
correct_cnn_vector.append(vec)
def style_transfer(content_image,
style_image,
content_layer_ids,
style_layer_ids,
weight_content=1.5,
weight_style=10.0,
weight_denoise=0.3,
num_iterations=120,
step_size=10.0):
"""
Используется градиентный спуск, чтобы найти изображение
минимизирующее функцию потреь контенти стиль изображения.
Это должно привести к смешанному изображению, которое напоминает контуры
содержимого-изображения и состоит из цвета и текстур стиль изображения.
Параметры
content_image: Трехмерный массив содержащий контент изображение
style_image: Трехмерный массив содержащий стиль изображение
content_layer_ids: Спиок идентификаторов слоев для контента
style_layer_ids: Спиок идентификаторов слоев для стиля
weight_content: Вес функции потерь контента
weight_style: Вес функции потерь стиля
weight_denoise: Вес функции потерь шума
num_iterations: Количество итераций
step_size: Шаг для градиента на каждой итерации
"""
# Создание экземпляра VGG16
model = vgg16.VGG16()
# Создание интерактивной сессии Tensorflow
session = tf.InteractiveSession(graph=model.graph)
# Вывод названий слоев контента
print("Content layers:")
print(model.get_layer_names(content_layer_ids))
print()
# Вывод названий слоев стиля
print("Style layers:")
print(model.get_layer_names(style_layer_ids))
print()
# Вычисление функции потерь контент изображения
loss_content = create_content_loss(session=session,
model=model,
content_image=content_image,
layer_ids=content_layer_ids)
# Вычисление функции потерь стиль изображения
loss_style = create_style_loss(session=session,
model=model,
style_image=style_image,
layer_ids=style_layer_ids)
# Вычисление функции потерь шума
loss_denoise = create_denoise_loss(model)
# Создание переменных Tensorflow для отрегулированных функций потерь
adj_content = tf.Variable(1e-10, name='adj_content')
adj_style = tf.Variable(1e-10, name='adj_style')
adj_denoise = tf.Variable(1e-10, name='adj_denoise')
# Инизиализация отрегулированных занчений функций потерь
session.run([
adj_content.initializer, adj_style.initializer, adj_denoise.initializer
])
# Создание операции Tensorflow для обновления отрегулированных значений
update_adj_content = adj_content.assign(1.0 / (loss_content + 1e-10))
update_adj_style = adj_style.assign(1.0 / (loss_style + 1e-10))
update_adj_denoise = adj_denoise.assign(1.0 / (loss_denoise + 1e-10))
# Вычисление весов функции потерь для минимизации
loss_combined = weight_content * adj_content * loss_content + \
weight_style * adj_style * loss_style + \
weight_denoise * adj_denoise * loss_denoise
# Вычисление градиента для комбинированного значения функций потреь
gradient = tf.gradients(loss_combined, model.input)
# Список тензоров для запуска при оптимизации
run_list = [gradient, update_adj_content, update_adj_style, \
update_adj_denoise]
# Создание изображения для смешивания, как случайного шума
mixed_image = np.random.rand(*content_image.shape) + 128
for i in range(num_iterations):
# Создание feed-dict - словаря элементов графа и значений
feed_dict = model.create_feed_dict(image=mixed_image)
# Вычисление значений градиента и отрегулированных значений
grad, adj_content_val, adj_style_val, adj_denoise_val = session.run(
run_list, feed_dict=feed_dict)
# Уменьшение размерности градиента
grad = np.squeeze(grad)
# Масштабирование в соответствии с параметром step size
step_size_scaled = step_size / (np.std(grad) + 1e-8)
# Обновление изображения с помощью градиента
mixed_image -= grad * step_size_scaled
# Проверка на принадлежность значения пикселя к интервалу [0, 255]
mixed_image = np.clip(mixed_image, 0.0, 255.0)
# Индикатор прогресса
print(". ", end="")
# Отображение весов и изображений каждые 10 итераций
if (i % 10 == 0) or (i == num_iterations - 1):
print()
print("Iteration:", i)
msg = "Adjust weights for Content: {0:.2e}, Style: {1:.2e}, Denoise: {2:.2e}"
print(msg.format(adj_content_val, adj_style_val, adj_denoise_val))
plot_images(content_image=content_image,
style_image=style_image,
mixed_image=mixed_image)
print()
print("Final image:")
mixed_image = np.clip(mixed_image, 0.0, 255.0)
mixed_image = mixed_image.astype(np.uint8)
mixed_image = PIL.Image.fromarray(mixed_image)
# mixed_image=upscale(mixed_image,4)
imshow(mixed_image)
# plot_image_big(mixed_image)
# Закрытие сессии Tensorflow
session.close()
# Возвращение итогового изображения
return mixed_image
print "Backend must be theano or tensorflow"
import numpy as np
import time
import sys
from keras.optimizers import SGD
# vgg16.py from https://github.com/fchollet/deep-learning-models
sys.path.append("deep-learning-models")
import vgg16
width = 224
height = 224
batch_size = 16
model = vgg16.VGG16(include_top=True, weights=None)
sgd = SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd) # loss='hinge'
if args.backend == "theano":
x = np.zeros((batch_size, 3, width, height), dtype=np.float32)
elif args.backend == "tensorflow":
x = np.zeros((batch_size, width, height, 3), dtype=np.float32)
else:
print "Backend must be theano or tensorflow"
y = np.zeros((batch_size, 1000), dtype=np.float32)
# warmup
model.train_on_batch(x, y)
t0 = time.time()
def style_transfer(content_image,
style_image,
content_layer_ids,
style_layer_ids,
weight_content=1.5,
weight_style=10.0,
weight_denoise=0.3,
num_iterations=1000,
step_size=10.0):
model = vgg16.VGG16()
# Create a TensorFlow-session.
session = tf.InteractiveSession(graph=model.graph)
loss_content = create_content_loss(session=session,
model=model,
content_image=content_image,
layer_ids=content_layer_ids)
# Create the loss-function for the style-layers and -image.
loss_style = create_style_loss(session=session,
model=model,
style_image=style_image,
layer_ids=style_layer_ids)
# Create the loss-function for the denoising of the mixed-image.
loss_denoise = create_denoise_loss(model)
# Create TensorFlow variables for adjusting the values of
# the loss-functions. This is explained below.
adj_content = tf.Variable(1e-10, name='adj_content')
adj_style = tf.Variable(1e-10, name='adj_style')
adj_denoise = tf.Variable(1e-10, name='adj_denoise')
# Initialize the adjustment values for the loss-functions.
session.run([
adj_content.initializer, adj_style.initializer, adj_denoise.initializer
])
update_adj_content = adj_content.assign(1.0 / (loss_content + 1e-10))
update_adj_style = adj_style.assign(1.0 / (loss_style + 1e-10))
update_adj_denoise = adj_denoise.assign(1.0 / (loss_denoise + 1e-10))
loss_combined = weight_content * adj_content * loss_content + weight_style * adj_style * loss_style + weight_denoise * adj_denoise * loss_denoise
gradient = tf.gradients(loss_combined, model.input)
# List of tensors that we will run in each optimization iteration.
run_list = [
gradient, update_adj_content, update_adj_style, update_adj_denoise
]
# The mixed-image is initialized with random noise.
# It is the same size as the content-image.
mixed_image = np.random.rand(*content_image.shape) + 128
for i in range(num_iterations):
# Create a feed-dict with the mixed-image.
feed_dict = model.create_feed_dict(image=mixed_image)
# Use TensorFlow to calculate the value of the
# gradient, as well as updating the adjustment values.
grad, adj_content_val, adj_style_val, adj_denoise_val = session.run(
run_list, feed_dict=feed_dict)
# Reduce the dimensionality of the gradient.
grad = np.squeeze(grad)
# Scale the step-size according to the gradient-values.
step_size_scaled = step_size / (np.std(grad) + 1e-8)
# Update the image by following the gradient.
mixed_image -= grad * step_size_scaled
# Ensure the image has valid pixel-values between 0 and 255.
mixed_image = np.clip(mixed_image, 0.0, 255.0)
# Print a little progress-indicator.
# print(". ", end="")
# Display status once every 10 iterations, and the last.
# if (i % 10 == 0) or (i == num_iterations - 1):
# print()
# print("Iteration:", i)
#
# # Print adjustment weights for loss-functions.
# print(msg.format(adj_content_val, adj_style_val, adj_denoise_val))
#
# # Plot the content-, style- and mixed-images.
# plot_images(content_image=content_image,
# style_image=style_image,
# mixed_image=mixed_image)
# print()
# print("Final image:")
# plot_image_big(mixed_image)
# Close the TensorFlow session to release its resources.
session.close()
# Return the mixed-image.
return mixed_image
def optimize(content_img,
style_img,
content_layer_ids,
style_layer_ids,
weight_content=1.5,
weight_style=10.0,
weight_denoise=0.3,
num_iterations=120,
step_size=10.0):
model = vgg16.VGG16()
session = tf.InteractiveSession(graph=model.graph)
loss_content = content_loss(session=session,
model=model,
content_img=content_img,
layer_ids=content_layer_ids)
loss_style = style_loss(session=session,
model=model,
style_img=style_img,
layer_ids=style_layer_ids)
loss_denoise = denoise_loss(model)
adj_content = tf.Variable(1e-10, name='adj_content')
adj_style = tf.Variable(1e-10, name='adj_style')
adj_denoise = tf.Variable(1e-10, name='adj_denoise')
session.run([
adj_content.initializer, adj_style.initializer, adj_denoise.initializer
])
update_adj_content = adj_content.assign(1.0 / (loss_content + 1e-10))
update_adj_style = adj_style.assign(1.0 / (loss_style + 1e-10))
update_adj_denoise = adj_denoise.assign(1.0 / (loss_denoise + 1e-10))
# we make the loss independent of the exact choice of style- and content-layers.
loss_combined = weight_content * adj_content * loss_content + \
weight_style * adj_style * loss_style + \
weight_denoise * adj_denoise * loss_denoise
gradient = tf.gradients(loss_combined, model.input)
run_list = [
gradient, update_adj_content, update_adj_style, update_adj_denoise
]
# mixed img is initialized with random noise.
mixed_img = np.random.rand(*content_img.shape) + 128
# m = np.zeros(mixed_img.shape)
# v = np.zeros(mixed_img.shape)
for i in tqdm(range(num_iterations)):
feed_dict = model.create_feed_dict(image=mixed_img)
grad, adj_content_val, adj_style_val, adj_denoise_val = session.run(
run_list, feed_dict=feed_dict)
grad = np.squeeze(grad)
step_size_scaled = step_size / (np.std(grad) + 1e-8)
# Update the img by following the gradient.
# m,v = adam(grad,i+1,m,v)
# mixed_img -= step_size_scaled*m/(np.sqrt(v)+1e-8)
mixed_img -= step_size_scaled * grad
mixed_img = np.clip(mixed_img, 0.0, 255.0)
if (i % 10 == 0) or (i == num_iterations - 1):
print()
print("iter:", i)
msg = "weight adjustment for content: {0:.2e}, style: {1:.2e}, denoise: {2:.2e}"
print(msg.format(adj_content_val, adj_style_val, adj_denoise_val))
print()
plot_img(mixed_img)
session.close()
return mixed_img
def loadCoeffIdx(pathName, modelName):
skipLinear = False
device = 'cuda' if torch.cuda.is_available() else 'cpu'
if modelName == "vgg16_c10":
model = vgg16.VGG16()
elif modelName == "vgg16_c10_old":
model = vgg16_old.VGG16()
elif modelName == "resnet50_c10":
model = resnet.ResNet50()
elif modelName == "inceptionv3_c10":
model = inception_v3_c10.inception_v3()
elif modelName == "alexnet_in":
#model = torch.hub.load('pytorch/vision', 'alexnet', pretrained=True)
model = alexnet.AlexNet()
skipLinear = True
elif modelName == "vgg16_in":
model = vgg_in.vgg16()
elif modelName == "resnet50_in":
model = resnet_in.resnet50()
elif modelName == "transformer_wmt":
translator = transformer_translate.create_model(
) # returns Transformer()
translator = translator.to(device)
model = translator.model
else:
print("Model not supported")
exit(1)
model = model.to(device)
if ("vgg16" in modelName or "alexnet" in modelName):
model = replace_vgg16(model, skipLinear)
elif "transformer" in modelName:
#transformer_train.replace_with_pruned(model, "model", prune_attention=True, prune_only_attention=False)
pass
else:
replace_with_pruned(model, "model", skipLinear)
if "_in" in modelName:
if not torch.distributed.is_initialized():
port = np.random.randint(10000, 65536)
torch.distributed.init_process_group(
backend='nccl',
init_method='tcp://127.0.0.1:%d' % port,
rank=0,
world_size=1)
model = torch.nn.parallel.DistributedDataParallel(model)
if not ("transformer" in modelName):
model.load_state_dict(torch.load(pathName))
model.eval()
#prune(model, method="cascade", q=1.0)
layers = []
widths = [3]
numConv = 0
assert isinstance(model, nn.Module)
for n, m in model.named_modules():
print(type(m))
if isinstance(m, torch.nn.Conv2d):
numConv += 1
if isinstance(m, torch.nn.Linear):
numConv += 1
if isinstance(m, PrunedConv):
#print(m.mask)
#layer = m.mask.view((m.out_channels, -1)).detach().cpu().numpy()
layer = m.conv.weight.view(
(m.out_channels, -1)).detach().cpu().numpy()
elif isinstance(m, PrunedLinear) or isinstance(
m, CSP.pruned_layers.PrunedLinear) and (not skipLinear):
#print(m.mask)
#layer = m.mask.view((m.out_features, -1)).detach().cpu().numpy()
layer = m.linear.weight.view(
(m.out_features, -1)).detach().cpu().numpy()
else:
continue
#print(n)
#print(layer.shape)
#layer = layer > 0
widths.append(len(layer))
layers.append(layer)
#layerMask = layer > 0
#layerMask = np.transpose(layerMask)
#print("NUM CONV: {}".format(numConv))
#print("NUM EXTRACTED LAYERS: {}".format(len(layers)))
if "transformer" in modelName:
for i in range(11):
for j in range(36, 97):
layers.append(layers[j])
return layers
def style_transfer(content_image,
style_image,
content_layer_ids,
style_layer_ids,
weight_content=1.5,
weight_style=10.0,
weight_denoise=0.3,
num_iterations=120,
step_size=10.0):
"""
Use gradient descent to find an image that minimizes the
loss-functions of the content-layers and style-layers.
"""
# Create an instance of the VGG16-model.
model = vgg16.VGG16()
# Create a TensorFlow-session.
session = tf.InteractiveSession(graph=model.graph)
# create a log file
summary_writer = tf.summary.FileWriter('log', graph_def=session.graph_def)
# Print the names of the content-layers.
print("Content layers:")
print(model.get_layer_names(content_layer_ids))
print()
# Print the names of the style-layers.
print("Style layers:")
print(model.get_layer_names(style_layer_ids))
print()
loss_content = loss.create_content_loss(session=session,
model=model,
content_image=content_image,
layer_ids=content_layer_ids)
loss_style = loss.create_style_loss(session=session,
model=model,
style_image=style_image,
layer_ids=style_layer_ids)
loss_denoise = loss.create_denoise_loss(model)
adj_content = tf.Variable(1e-10, name='adj_content')
adj_style = tf.Variable(1e-10, name='adj_style')
adj_denoise = tf.Variable(1e-10, name='adj_denoise')
# Initialize the adjustment values for the loss-functions.
session.run([
adj_content.initializer, adj_style.initializer, adj_denoise.initializer
])
# add 1e-10 to avoid the possibility of division by zero.
update_adj_content = adj_content.assign(1.0 / (loss_content + 1e-10))
update_adj_style = adj_style.assign(1.0 / (loss_style + 1e-10))
update_adj_denoise = adj_denoise.assign(1.0 / (loss_denoise + 1e-10))
loss_combined = (weight_content * adj_content * loss_content +
weight_style * adj_style * loss_style +
weight_denoise * adj_denoise * loss_denoise)
# gradient decent
gradient = tf.gradients(loss_combined, model.input)
# run in each optimization iteration.
run_list = [
gradient, update_adj_content, update_adj_style, update_adj_denoise
]
mixed_image = np.random.rand(*content_image.shape) + 128
for i in range(num_iterations):
# Create a feed-dict with the mixed-image.
feed_dict = model.create_feed_dict(image=mixed_image)
grad, adj_content_val, adj_style_val, adj_denoise_val = (session.run(
run_list, feed_dict=feed_dict))
# Reduce the dimensionality of the gradient.
grad = np.squeeze(grad)
step_size_scaled = step_size / (np.std(grad) + 1e-8)
mixed_image -= grad * step_size_scaled
mixed_image = np.clip(mixed_image, 0.0, 255.0)
print(". ", end="")
# Display status every 10 iterations
if (i % 10 == 0) or (i == num_iterations - 1):
print()
print("Iteration:", i)
# Print adjustment weights for loss-functions.
msg = "Weight Adj. for Content: {0:.2e}, Style: {1:.2e}, Denoise: {2:.2e}"
print(msg.format(adj_content_val, adj_style_val, adj_denoise_val))
summary_writer.add_summary(adj_content_val, adj_style_val,
adj_denoise_val)
# Plot the content-, style- and mixed-images.
IO.plot_images(content_image=content_image,
style_image=style_image,
mixed_image=mixed_image)
print()
print("Final image:")
IO.plot_image_big(mixed_image)
session.close()
return mixed_image
def style_transfer(content_image,
style_image,
content_layer_ids,
style_layer_ids,
alpha=1.5,
beta=10.0,
weight_denoise=0.3,
num_iterations=120,
step_size=10.0):
"""
Use gradient descent to find an image that minimizes the
loss-functions of the content-layers and style-layers. This
should result in a mixed-image that resembles the contours
of the content-image, and resembles the colours and textures
of the style-image.
Parameters:
content_image: Numpy 3-dim float-array with the content-image.
style_image: Numpy 3-dim float-array with the style-image.
content_layer_ids: List of integers identifying the content-layers.
style_layer_ids: List of integers identifying the style-layers.
alpha: Weight for the content-loss-function.
beta: Weight for the style-loss-function.
weight_denoise: Weight for the denoising-loss-function.
num_iterations: Number of optimization iterations to perform.
step_size: Step-size for the gradient in each iteration.
"""
initializeVGG16()
model = vgg16.VGG16()
# Create a TensorFlow-session.
session = tf.InteractiveSession(graph=model.graph)
# Print the names of the content-layers.
print()
print("Content layers:")
print(model.get_layer_names(content_layer_ids))
print()
# Print the names of the style-layers.
print("Style layers:")
print(model.get_layer_names(style_layer_ids))
print()
# Create the loss-function for the content-layers and -image.
loss_content = create_content_loss(session=session,
model=model,
content_image=content_image,
layer_ids=content_layer_ids)
# Create the loss-function for the style-layers and -image.
loss_style = create_style_loss(session=session,
model=model,
style_image=style_image,
layer_ids=style_layer_ids)
# Create the loss-function for the denoising of the mixed-image.
loss_denoise = create_denoise_loss(model)
# Create TensorFlow variables for adjusting the values of
# the loss-functions. This is explained below.
## Think of these as bias
adj_content = tf.Variable(1e-10, name='adj_content')
adj_style = tf.Variable(1e-10, name='adj_style')
adj_denoise = tf.Variable(1e-10, name='adj_denoise')
# Initialize the adjustment values for the loss-functions.
session.run([
adj_content.initializer, adj_style.initializer, adj_denoise.initializer
])
# Create TensorFlow operations for updating the adjustment values.
# These are basically just the reciprocal values of the
# loss-functions, with a small value 1e-10 added to avoid the
# possibility of division by zero.
update_adj_content = adj_content.assign(1.0 / (loss_content + 1e-10))
update_adj_style = adj_style.assign(1.0 / (loss_style + 1e-10))
update_adj_denoise = adj_denoise.assign(1.0 / (loss_denoise + 1e-10))
loss_combined = alpha * adj_content * loss_content + \
beta * adj_style * loss_style + \
weight_denoise * adj_denoise * loss_denoise
# Compute gradient
gradient = tf.gradients(loss_combined, model.input)
# List of tensors that we will run in each optimization iteration.
run_list = [
gradient, update_adj_content, update_adj_style, update_adj_denoise
]
# The mixed-image is initialized with random noise.
# It is the same size as the content-image.
mixed_image = np.random.rand(*content_image.shape) + 128
for i in range(num_iterations):
# Create a feed-dict with the mixed-image.
feed_dict = model.create_feed_dict(image=mixed_image)
# Use TensorFlow to calculate the value of the
# gradient, as well as updating the adjustment values.
grad, adj_content_val, adj_style_val, adj_denoise_val = session.run(
run_list, feed_dict=feed_dict)
# Reduce the dimensionality of the gradient.
grad = np.squeeze(grad)
# Scale the step-size according to the gradient-values.
step_size_scaled = step_size / (np.std(grad) + 1e-8)
# Update the image by following the gradient.
## Gradient descent
mixed_image -= grad * step_size_scaled
# Ensure the image has valid pixel-values between 0 and 255.
mixed_image = np.clip(mixed_image, 0.0, 255.0)
# Display status once every 10 iterations, and the last.
if (i % 10 == 0) or (i == num_iterations - 1):
print()
print("Epoch:", i)
# Print adjustment weights for loss-functions.
msg = "Weight Adj. for Content: {0:.2e}, Style: {1:.2e}, Denoise: {2:.2e}"
print(msg.format(adj_content_val, adj_style_val, adj_denoise_val))
#imageUtils.save_image(mixed_image, "mixed" + str(i) + ".png")
# Close the TensorFlow session to release its resources.
session.close()
# Return the mixed-image.
return mixed_image
def style_transfer(content_image, style_image,
content_layer_ids, style_layer_ids,
weight_content=1.5, weight_style=10.0,
weight_denoise=0.3,
num_iter=120, step_size=10.0):
model = vgg16.VGG16()
sess = tf.InteractiveSession(graph=model.graph)
# Print the names of the content-layers.
print("Content layers:")
print(model.get_layer_names(content_layer_ids))
print()
# Print the names of the style-layers.
print("Style layers:")
print(model.get_layer_names(style_layer_ids))
print()
# Create the loss-function for the content-layers and -image.
loss_content = content_img_loss(sess=sess,
model=model,
content_img=content_image,
layer_ids=content_layer_ids)
# Create the loss-function for the style-layers and -image.
loss_style = style_img_loss(sess=sess,
model=model,
style_img=style_image,
layer_ids=style_layer_ids)
# Create the loss-function for the denoising of the mixed-image.
loss_denoise = create_denoise_loss(model)
adj_content = tf.Variable(1e-10, name='adj_content')
adj_style = tf.Variable(1e-10, name='adj_style')
adj_denoise = tf.Variable(1e-10, name='adj_denoise')
# Initialize the adjustment values for the loss-functions.
sess.run([adj_content.initializer,
adj_style.initializer,
adj_denoise.initializer])
update_adj_content = adj_content.assign(1.0 / (loss_content + 1e-10))
update_adj_style = adj_style.assign(1.0 / (loss_style + 1e-10))
update_adj_denoise = adj_denoise.assign(1.0 / (loss_denoise + 1e-10))
# define the loss function for applying gradient
loss = weight_content*adj_content*loss_content \
+ weight_style*adj_style*loss_style \
+ weight_denoise*adj_denoise*loss_denoise
gradient = tf.gradients(loss, model.input)
# List of tensors that we will run in each optimization iteration.
run_list = [gradient, update_adj_content, update_adj_style, \
update_adj_denoise]
# The mixed-image is initialized with random noise.
# It is the same size as the content-image.
mixed_image = np.random.rand(*content_image.shape) + 128
for i in range(num_iter):
print("Iteration:", i)
feed_dict = model.create_feed_dict(image=mixed_image)
grad, adj_content_val, adj_style_val, adj_denoise_val \
= sess.run(run_list, feed_dict=feed_dict)
# Reduce the dimensionality of the gradient.
grad = np.squeeze(grad)
# Scale the step-size according to the gradient-values.
step_size_scaled = step_size / (np.std(grad) + 1e-8)
# Update the image by following the gradient.
mixed_image -= grad * step_size_scaled
# Ensure the image has valid pixel-values between 0 and 255.
mixed_image = np.clip(mixed_image, 0.0, 255.0)
if (i % 10 == 0) or (i == num_iter - 1):
# Print adjustment weights for loss-functions.
msg = "Weight Adj. for Content: {0:.2e}, Style: {1:.2e}, Denoise: {2:.2e}"
print(msg.format(adj_content_val, adj_style_val, adj_denoise_val))
print()
print("Final image:")
img.plot_imgs(content_img=content_image,
style_img=style_image,
mixed_img=mixed_image)
img.plot_img(mixed_image)
# Close the TensorFlow session to release its resources.
sess.close()
# Return the mixed-image.
return mixed_image
class Tester:
def __init__(self, model):
self.model = model
device_name = tf.test.gpu_device_name()
if device_name == '/device:GPU:0':
print('Found GPU at:{}!'.format(device_name))
else:
print('Tensorflow have not found GPU,run on CPU mode!')
train_set = [".\\tfrecord\\train.tfrecord"]
val_set = [".\\tfrecord\\val.tfrecord"]
from dataprovider import dataprovider
# create dateprovide for model
dataset_train = dataprovider(train_set, 61, 1037, [224, 224, 3], 32739, 40)
dataset_val = dataprovider(val_set, 61, 100, [224, 224, 3], 4982, 40)
import vgg16
import vgg19
tester = Tester(
vgg16.VGG16(dataset_train, dataset_val, 0.01, 400,
os.path.join('.', 'checkpoints', 'vgg_16.ckpt')))
#tester = Tester(vgg19.VGG19(dataset,0.01,400,".\\checkpoints\\vgg_19.ckpt",))
tester.model.train_all_epochs()
content_layer_ids = [8]
# style_layer_ids = list(range(13))
style_layer_ids = [0, 2, 4, 7, 10]
#-----------------------------------------------------------------------------------------------------------------------
# Sytle Transfer
weight_content = 1.0
weight_style = 200.0
step_size = 10
# num_iterations = 200
num_iterations = 20
disp_interval = num_iterations
model = vgg16.VGG16()
session = tf.InteractiveSession(graph=model.graph)
# Print the names of the content-layers.
print("Content layers:")
print(model.get_layer_names(content_layer_ids))
print()
# Print the names of the style-layers.
print("Style layers:")
print(model.get_layer_names(style_layer_ids))
print()
# Create the loss-function for the content-layers and -image.
loss_content = create_content_loss(session=session,
def training(learn_rate=0.01, num_epochs=1000, save_model=False, debug=False):
# assert len(train_x.shape) == 4
# [num_images, img_height, img_width, num_channel] = train_x.shape
# num_classes = labels.shape[-1]
config = Config()
# config.num_classes = num_classes
num_steps = int(np.ceil(config.num_images / float(config.batch_size)))
with tf.Graph().as_default():
model = vgg16.VGG16(config)
voc2012 = VOC2012('../data', config.batch_size, config.test_size)
predicts = model.building(True)
# loss function
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
labels=model.labels, logits=predicts)
loss = tf.reduce_mean(cross_entropy)
model.loss_summary(loss)
# optimizer with decayed learning rate
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(learn_rate,
global_step,
num_steps * num_epochs,
0.1,
staircase=True)
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(
loss, global_step=global_step)
# prediction for the training data
predicts_result = tf.nn.softmax(predicts)
# Initializing operation
init_op = tf.global_variables_initializer()
saver = tf.train.Saver(max_to_keep=100)
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
# initialize parameters or restore from previous model
if not os.path.exists(config.params_dir):
os.makedirs(config.params_dir)
if os.listdir(config.params_dir) == [] or config.initialize:
print "Initializing Network"
sess.run(init_op)
else:
sess.run(init_op)
model.restore(sess, saver, config.load_filename)
merged = tf.summary.merge_all()
logdir = os.path.join(config.logdir,
datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
writer = tf.summary.FileWriter(logdir, sess.graph)
epoch_loss = 0.0
for epoch in range(num_epochs):
for step in range(num_steps):
with tf.device("/cpu:0"):
imgs, labels = voc2012.train.next_batch(
config.batch_size)
labels = one_hot(labels, 20)
feed_dict = {model.imgs: imgs, model.labels: labels}
with tf.device(config.gpu):
_, l, predictions = sess.run(
[optimizer, loss, predicts_result],
feed_dict=feed_dict)
print "batch loss: ", l
epoch_loss += l
print "each epoch Loss: %0.6f" % (epoch_loss / num_steps)
with tf.device("/cpu:0"):
# write summary
if epoch % config.summary_iters == 0:
tmp_global_step = model.global_step.eval()
summary = sess.run(merged, feed_dict=feed_dict)
writer.add_summary(summary, tmp_global_step)
# save checkpoint
if epoch % config.checkpoint_iters == 0:
tmp_global_step = model.global_step.eval()
model.save(sess, saver, config.save_filename,
tmp_global_step)
test_loss = 0.0
test_accuracy = 0.0
for i in range(4):
with tf.device("/cpu:0"):
valid_x, valid_y = voc2012.test.next_batch(
config.test_size)
valid_y = one_hot(test_labels, 20)
if valid_x is not None and valid_y is not None:
feed_dict = {model.imgs: valid_x, model.labels: valid_y}
l, predictions = sess.run([loss, predicts_result],
feed_dict=feed_dict)
test_loss += l
test_accuracy += predictions
print('Valid Loss = %.6f\t Accuracy = %.6f%%' %
(test_loss / 4, accuracy(predictions, valid_y) / 4))
