def test(epoch):
model.training = False
test_loss = 0
test_loader = generate_batch_loader(test_data, batch_size=batch_size)
for i, data in enumerate(test_loader):
# Dymanic Construction of Graph
dy.renew_cg()
x = dy.inputTensor(data.reshape(-1, 784).T)
recon_x, mu, logvar = model.forward(x)
loss = loss_function(recon_x, x, mu, logvar)
# Forward
loss_value = loss.value()
test_loss += loss_value
if i == 0:
n = min(data.shape[0], 8)
comparison = np.concatenate([data[:n],
recon_x.npvalue().T.reshape(batch_size, 1, 28, 28)[:n]])
save_image(comparison,
'results/reconstruction_' + str(epoch) + '.png', nrow=n)
test_loss /= len(test_data)
print('====> Test set loss: {:.4f}'.format(test_loss))
def generate(self, x_samples, root_path=None, idx=None):
if self.data_format == 'NCHW':
x_samples = to_nchw_numpy(x_samples)
generated = self.sess.run(self.yt_img, {self.xt: x_samples})
y_path = os.path.join(root_path, 'y_{}.png'.format(idx))
save_image(generated, y_path, nrow=self.b_num)
print("[*] Samples saved: {}".format(y_path))
def main():
# parse arguments
args = parse_args()
if args is None:
exit()
# load content image
content_image = utils.load_image(args.content, max_size=args.max_size)
# open session
soft_config = tf.ConfigProto(allow_soft_placement=True)
soft_config.gpu_options.allow_growth = True # to deal with large image
sess = tf.Session(config=soft_config)
# build the graph
transformer = style_transfer_tester.StyleTransferTester(session=sess,
model_path=args.style_model,
content_image=content_image,
)
# execute the graph
start_time = time.time()
output_image = transformer.test()
end_time = time.time()
# save result
utils.save_image(output_image, args.output)
# report execution time
shape = content_image.shape #(batch, width, height, channel)
print('Execution time for a %d x %d image : %f msec' % (shape[0], shape[1], 1000.*float(end_time - start_time)/60))
def stylize(args):
device = torch.device("cuda" if args.cuda else "cpu")
content_image = utils.load_image(args.content_image, scale=args.content_scale)
content_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Lambda(lambda x: x.mul(255))
])
content_image = content_transform(content_image)
content_image = content_image.unsqueeze(0).to(device)
if args.model.endswith(".onnx"):
output = stylize_onnx_caffe2(content_image, args)
else:
with torch.no_grad():
style_model = TransformerNet()
state_dict = torch.load(args.model)
# remove saved deprecated running_* keys in InstanceNorm from the checkpoint
for k in list(state_dict.keys()):
if re.search(r'in\d+\.running_(mean|var)$', k):
del state_dict[k]
style_model.load_state_dict(state_dict)
style_model.to(device)
if args.export_onnx:
assert args.export_onnx.endswith(".onnx"), "Export model file should end with .onnx"
output = torch.onnx._export(style_model, content_image, args.export_onnx).cpu()
else:
output = style_model(content_image).cpu()
utils.save_image(args.output_image, output[0])
def train(self, n_iters):
skip_step = 1
with tf.Session() as sess:
###############################
## TO DO:
## 1. initialize your variables
## 2. create writer to write your graph
sess.run(tf.global_variables_initializer())
writer = tf.summary.FileWriter('graphs/style_stranfer', sess.graph)
###############################
sess.run(self.input_img.assign(self.initial_img))
###############################
## TO DO:
## 1. create a saver object
## 2. check if a checkpoint exists, restore the variables
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(os.path.dirname('checkpoints/style_transfer/checkpoint'))
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
##############################
initial_step = self.gstep.eval()
start_time = time.time()
for index in range(initial_step, n_iters):
if index >= 5 and index < 20:
skip_step = 10
elif index >= 20:
skip_step = 20
sess.run(self.opt)
if (index + 1) % skip_step == 0:
###############################
## TO DO: obtain generated image, loss, and summary
gen_image, total_loss, summary = sess.run([self.input_img,
self.total_loss,
self.summary_op])
###############################
# add back the mean pixels we subtracted before
gen_image = gen_image + self.vgg.mean_pixels
writer.add_summary(summary, global_step=index)
print('Step {}\n Sum: {:5.1f}'.format(index + 1, np.sum(gen_image)))
print(' Loss: {:5.1f}'.format(total_loss))
print(' Took: {} seconds'.format(time.time() - start_time))
start_time = time.time()
filename = 'outputs/%d.png' % (index)
utils.save_image(filename, gen_image)
if (index + 1) % 20 == 0:
###############################
## TO DO: save the variables into a checkpoint
saver.save(sess, 'checkpoints/style_stranfer/style_transfer', index)
def _save_image(note, image): note.snap_path = _build_snap_path(image) note.image_path = _build_image_path(image) if not utils.save_image_snap(note.snap_path, image): note.snap_path = note.image_path note.photo_time = utils.get_photo_time(image) if note.photo_time is None: note.photo_time = note.create_time utils.save_image(note.image_path, image)
def find_edges(img, input, output, binary=False, textual=False, interactive=False, changed="", n8=False, search_width=16):
# create necessary image copies
changed_img = np.copy(img)
img_copy = np.copy(img)
img = np.apply_along_axis(kitti_labels.get_label, 2, img)
prob_img = np.zeros((img.shape[0], img.shape[1], kitti_labels.NUMBER_OF_LABELS))
# iterate through image and look at top and right neighbours (8-neighbourhood)
for i in xrange(img.shape[0]):
for j in xrange(img.shape[1]):
current_pixel = img[i,j]
if current_pixel in kitti_labels.DONT_CARE_LABELS:
# put uniform distribution if the label is unknown
prob_img[i,j,:] = search_width
continue
# set the probability for the read label
if np.sum(prob_img[i,j,:]) == 0:
prob_img[i,j,current_pixel] = search_width
# seach range in each direction
max_counter = search_width / 2
# check right neighbour
_search_right(i, j, img, current_pixel, max_counter, prob_img, changed_img, search_width)
# check top
_search_top(i, j, img, current_pixel, max_counter, prob_img, changed_img, search_width)
if n8:
# check top right
_search_top_right(i, j, img, current_pixel, max_counter, prob_img, changed_img, search_width)
# check top left
_search_top_left(i, j, img, current_pixel, max_counter, prob_img, changed_img, search_width)
if interactive:
plt.imshow(changed_img)
plt.figure()
plt.imshow(img_copy)
plt.show()
if changed:
print "Saving image with change pixels - {}".format(changed)
utils.save_image(changed_img, changed)
if binary:
print "Saving potentials - binary format - {}".format(output + ".bin")
utils.save_prob_map_bin(prob_img, output + ".bin", True)
if textual:
print "Saving potentials - textual format - {}".format(output + ".txt")
utils.save_prob_map_txt(prob_img, output + ".txt", True)
def write_grid(input_images, gt_projs, pred_projs, global_step,
input_voxels, output_voxels):
"""Native python function to call for writing images to files."""
grid = _build_image_grid(
input_images,
gt_projs,
pred_projs,
input_voxels=input_voxels,
output_voxels=output_voxels)
if global_step % summary_freq == 0:
img_path = os.path.join(log_dir, '%s.jpg' % str(global_step))
utils.save_image(grid, img_path)
return grid
def train(model, generated_image, initial_image):
""" Train your model.
Don't forget to create folders for checkpoints and outputs.
"""
skip_step = 1
with tf.Session() as sess:
saver = tf.train.Saver()
###############################
## TO DO:
## 1. initialize your variables
## 2. create writer to write your graph
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
writer = tf.summary.FileWriter(EXP + '/graphs', sess.graph)
###############################
sess.run(generated_image.assign(initial_image))
ckpt = tf.train.get_checkpoint_state(os.path.dirname('checkpoints/checkpoint'))
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
initial_step = model['global_step'].eval()
start_time = time.time()
for index in range(initial_step, ITERS):
if index >= 5 and index < 20:
skip_step = 10
elif index >= 20:
skip_step = 20
sess.run(model['optimizer'])
if (index + 1) % skip_step == 0:
###############################
## TO DO: obtain generated image and loss
gen_image, total_loss, summary = sess.run([generated_image, model['total_loss'],
model['summary_op']])
###############################
gen_image = gen_image + MEAN_PIXELS
writer.add_summary(summary, global_step=index)
print('Step {}\n Sum: {:5.1f}'.format(index + 1, np.sum(gen_image)))
print(' Loss: {:5.1f}'.format(total_loss))
print(' Time: {}'.format(time.time() - start_time))
start_time = time.time()
filename = 'outputs/%d.png' % (index)
utils.save_image(filename, gen_image)
if (index + 1) % 20 == 0:
saver.save(sess, 'checkpoints/style_transfer', index)
def main():
args = utils.get_args()
print("Prepare dataset...")
mnist = input_data.read_data_sets("mnist/", one_hot = True)
with tf.Graph().as_default(), tf.Session() as session:
autoencoder = Autoencoder(
784, args.hid_shape, args.lat_shape,
optimizer = tf.train.AdagradOptimizer(args.lr),
batch_size = args.batch_size,
dropout = args.dropout)
session.run(tf.initialize_all_variables())
if args.save_model or args.load_model:
saver = tf.train.Saver()
if args.load_model:
try:
saver.restore(session, utils.SAVER_FILE)
except ValueError:
print("Cant find model file")
sys.exit(1)
if args.make_imgs:
index = 0
print("Prepare images directory...")
utils.prepare_image_folder()
example = utils.get_example(args.digit, mnist.test)
print("Start training...")
for epoch in range(args.epoches):
for i, batch in enumerate(utils.gen_data(args.batch_size, mnist.train.images)):
autoencoder.fit_on_batch(session, batch)
if (i+1) % args.log_after == 0:
test_cost = autoencoder.evaluate(session, mnist.test.images)
print("Test error = {0:.4f} on {1} batch in {2} epoch".format(test_cost, i+1, epoch+1))
if args.make_imgs:
path = os.path.join(utils.IMG_FOLDER, "{0:03}.png".format(index))
autoencoded = autoencoder.encode_decode(session, example.reshape(1, 784))
utils.save_image(autoencoded.reshape((28, 28)), path)
index += 1
if args.save_model:
saver.save(session, utils.SAVER_FILE)
print("Model saved")
def generate(self, x_fixed, x_target_fixed, pose_target_fixed, root_path=None, path=None, idx=None, save=True):
G = self.sess.run(self.G, {self.x: x_fixed, self.pose_target: pose_target_fixed})
ssim_G_x_list = []
# x_0_255 = utils_wgan.unprocess_image(x_target_fixed, 127.5, 127.5)
for i in xrange(G.shape[0]):
# G_gray = rgb2gray((G[i,:]/127.5-1).clip(min=-1,max=1))
# x_target_gray = rgb2gray((x_target_fixed[i,:]).clip(min=-1,max=1))
G_gray = rgb2gray((G[i,:]).clip(min=0,max=255).astype(np.uint8))
x_target_gray = rgb2gray(((x_target_fixed[i,:]+1)*127.5).clip(min=0,max=255).astype(np.uint8))
ssim_G_x_list.append(ssim(G_gray, x_target_gray, data_range=x_target_gray.max() - x_target_gray.min(),
multichannel=False))
ssim_G_x_mean = np.mean(ssim_G_x_list)
if path is None and save:
path = os.path.join(root_path, '{}_G_ssim{}.png'.format(idx,ssim_G_x_mean))
save_image(G, path)
print("[*] Samples saved: {}".format(path))
return G
def train(self, epoches=300):
skip_step = 1
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
writer = tf.summary.FileWriter("graphs/style_transfer", sess.graph)
sess.run(self.input_img.assign(self.initial_img))
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(os.path.dirname("checkpoints/%s_%s_style_transfer/checkpoint" %
(self.content_name, self.style_name)))
if ckpt and ckpt.model_checkpoint_path:
print("You have pre-trained model, if you do not want to use this, please delete the existing one.")
saver.restore(sess, ckpt.model_checkpoint_path)
initial_step = self.gstep.eval()
for epoch in range(initial_step, epoches):
# 前面几轮每隔10个epoch生成一张图片
if epoch >= 5 and epoch < 20:
skip_step = 10
# 后面每隔20个epoch生成一张图片
elif epoch >= 20:
skip_step = 20
sess.run(self.optimizer)
if (epoch + 1) % skip_step == 0:
gen_image, total_loss, summary = sess.run([self.input_img,
self.total_loss,
self.summary_op])
# 对生成的图片逆向mean-center，即在每个channel上加上mean
gen_image = gen_image + self.vgg.mean_pixels
writer.add_summary(summary, global_step=epoch)
print("Step {}\n Sum: {:5.1f}".format(epoch + 1, np.sum(gen_image)))
print(" Loss: {:5.1f}".format(total_loss))
filename = "outputs/%s_%s/epoch_%d.png" % (self.content_name, self.style_name, epoch)
utils.save_image(filename, gen_image)
# 存储模型
if (epoch + 1) % 20 == 0:
saver.save(sess,
"checkpoints/%s_%s_style_transfer/style_transfer" %
(self.content_name, self.style_name), epoch)
def write_grid(input_images, gt_projs, pred_projs, pred_voxels,
global_step):
"""Native python function to call for writing images to files."""
grid = _build_image_grid(input_images, gt_projs, pred_projs, pred_voxels)
if global_step % summary_freq == 0:
img_path = os.path.join(log_dir, '%s.jpg' % str(global_step))
utils.save_image(grid, img_path)
with open(
os.path.join(log_dir, 'pred_voxels_%s' % str(global_step)),
'w') as fout:
np.save(fout, pred_voxels)
with open(
os.path.join(log_dir, 'input_images_%s' % str(global_step)),
'w') as fout:
np.save(fout, input_images)
return grid
def stylize(args):
content_image = utils.load_image(args.content_image, scale=args.content_scale)
content_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Lambda(lambda x: x.mul(255))
])
content_image = content_transform(content_image)
content_image = content_image.unsqueeze(0)
if args.cuda:
content_image = content_image.cuda()
content_image = Variable(content_image, volatile=True)
style_model = TransformerNet()
style_model.load_state_dict(torch.load(args.model))
if args.cuda:
style_model.cuda()
output = style_model(content_image)
if args.cuda:
output = output.cpu()
output_data = output.data[0]
utils.save_image(args.output_image, output_data)
def motion_detection(camera, folder, until):
"""
Uses 3 frames to look for motion, can't remember where
I found it but it gives better result than my first try
with comparing 2 frames.
"""
utils.clear_directory(folder)
# Need to get 2 images to start with
previous_image = cv2.cvtColor(camera.read()[1], cv2.cv.CV_RGB2GRAY)
current_image = cv2.cvtColor(camera.read()[1], cv2.cv.CV_RGB2GRAY)
purple = (140, 25, 71)
while True:
now = datetime.datetime.now()
_, image = camera.read()
gray_image = cv2.cvtColor(image, cv2.cv.CV_RGB2GRAY)
difference1 = cv2.absdiff(previous_image, gray_image)
difference2 = cv2.absdiff(current_image, gray_image)
result = cv2.bitwise_and(difference1, difference2)
# Basic threshold, turn the bitwise_and into a black or white (haha)
# result, white (255) being a motion
_, result = cv2.threshold(result, 40, 255, cv2.THRESH_BINARY)
# Let's show a square around the detected motion in the original pic
low_point, high_point = utils.find_motion_boundaries(result.tolist())
if low_point is not None and high_point is not None:
cv2.rectangle(image, low_point, high_point, purple, 3)
print 'Motion detected ! Taking picture'
utils.save_image(image, folder, now)
previous_image = current_image
current_image = gray_image
if utils.time_over(until, now):
break
del(camera)
def start_camera(camera, folder, interval, until=None):
"""
Start taking pictures every interval.
If until is specified, it will take pictures
until that time is reached (24h format).
Needs to be of the following format: HH:MM
"""
utils.clear_directory(folder)
number = 0
while True:
_, image = camera.read()
now = datetime.datetime.now()
number += 1
print 'Taking picture number %d at %s' % (number, now.isoformat())
utils.save_image(image, folder, now)
if utils.time_over(until, now):
break
time.sleep(interval)
del(camera)
def main(config):
prepare_dirs_and_logger(config)
batch_manager = BatchManager(config)
preprocess_path('data/kanji/train/0f9a8.svg_pre', 64, 64, batch_manager.rng)
preprocess_overlap('data/kanji/train/0f9a8.svg_pre', 64, 64, batch_manager.rng)
# thread test
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
sess_config.allow_soft_placement = True
sess_config.log_device_placement = False
sess = tf.Session(config=sess_config)
batch_manager.start_thread(sess)
x, y = batch_manager.batch()
if config.data_format == 'NCHW':
x = nhwc_to_nchw(x)
x_, y_ = sess.run([x, y])
batch_manager.stop_thread()
if config.data_format == 'NCHW':
x_ = x_.transpose([0, 2, 3, 1])
if config.archi == 'path':
b_ch = np.zeros([config.batch_size,config.height,config.width,1])
x_ = np.concatenate((x_*255, b_ch), axis=-1)
else:
x_ = x_*255
y_ = y_*255
save_image(x_, '{}/x_fixed.png'.format(config.model_dir))
save_image(y_, '{}/y_fixed.png'.format(config.model_dir))
# random pick from parameter space
x_samples, x_gt, y_gt, sample_list = batch_manager.random_list(8)
save_image(x_gt, '{}/x_gt.png'.format(config.model_dir))
save_image(y_gt, '{}/y_gt.png'.format(config.model_dir))
with open('{}/sample_list.txt'.format(config.model_dir), 'w') as f:
for sample in sample_list:
f.write(sample+'\n')
print('batch manager test done')
def get_image():
"""Gets an image file via POST request, feeds the image to the FaceNet model then saves both the original image
and its resulting embedding from the FaceNet model in their designated folders.
'uploads' folder: for image files
'embeddings' folder: for embedding numpy files.
"""
if request.method == 'POST':
if 'file' not in request.files:
return render_template(template_name_or_list="warning.html",
status="No 'file' field in POST request!")
file = request.files['file']
filename = file.filename
if filename == "":
return render_template(template_name_or_list="warning.html",
status="No selected file!")
if file and allowed_file(filename=filename, allowed_set=allowed_set):
filename = secure_filename(filename=filename)
# Read image file as numpy array of RGB dimension
img = imread(name=file, mode='RGB')
# Detect and crop a 160 x 160 image containing a human face in the image file
img = get_face(img=img,
pnet=pnet,
rnet=rnet,
onet=onet,
image_size=image_size)
# If a human face is detected
if img is not None:
embedding = forward_pass(
img=img,
session=facenet_persistent_session,
images_placeholder=images_placeholder,
embeddings=embeddings,
phase_train_placeholder=phase_train_placeholder,
image_size=image_size)
# Save cropped face image to 'uploads/' folder
save_image(img=img,
filename=filename,
uploads_path=uploads_path)
# Remove file extension from image filename for numpy file storage being based on image filename
filename = remove_file_extension(filename=filename)
# Save embedding to 'embeddings/' folder
save_embedding(embedding=embedding,
filename=filename,
embeddings_path=embeddings_path)
return render_template(
template_name_or_list="upload_result.html",
status="Image uploaded and embedded successfully!")
else:
return render_template(
template_name_or_list="upload_result.html",
status=
"Image upload was unsuccessful! No human face was detected!"
)
else:
return render_template(template_name_or_list="warning.html",
status="POST HTTP method required!")
def produce_art(content_image_path, style_image_path, model_path, model_type, width, alpha, beta, num_iters):
# The actual calculation
print "Read images..."
content_image = read_image(content_image_path, width)
style_image = read_image(style_image_path, width)
g = tf.Graph()
with g.device(device), g.as_default(), tf.Session(graph=g,
config=tf.ConfigProto(allow_soft_placement=True)) as sess:
print "Load content values..."
image = tf.constant(content_image)
model = getModel(image, model_path, model_type)
content_image_y_val = [sess.run(y_l) for y_l in model.y()] # sess.run(y_l) is a constant numpy array
print "Load style values..."
image = tf.constant(style_image)
model = getModel(image, model_path, model_type)
y = model.y()
style_image_st_val = []
for l in range(len(y)):
num_filters = content_image_y_val[l].shape[3]
st_shape = [-1, num_filters]
st_ = tf.reshape(y[l], st_shape)
st = tf.matmul(tf.transpose(st_), st_)
style_image_st_val.append(sess.run(st)) # sess.run(st) is a constant numpy array
print "Construct graph..."
# Start from white noise
# gen_image = tf.Variable(tf.truncated_normal(content_image.shape, stddev=20), trainable=True, name='gen_image')
# Start from the original image
gen_image = tf.Variable(tf.constant(np.array(content_image, dtype=np.float32)), trainable=True,
name='gen_image')
model = getModel(gen_image, model_path, model_type)
y = model.y()
L_content = 0.0
L_style = 0.0
for l in range(len(y)):
# Content loss
L_content += model.alpha[l] * tf.nn.l2_loss(y[l] - content_image_y_val[l])
# Style loss
num_filters = content_image_y_val[l].shape[3]
st_shape = [-1, num_filters]
st_ = tf.reshape(y[l], st_shape)
st = tf.matmul(tf.transpose(st_), st_)
N = np.prod(content_image_y_val[l].shape).astype(np.float32)
L_style += model.beta[l] * tf.nn.l2_loss(st - style_image_st_val[l]) / N ** 2 / len(y)
# The loss
L = alpha * L_content + beta * L_style
# The optimizer
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(learning_rate=2.0, global_step=global_step, decay_steps=100,
decay_rate=0.94, staircase=True)
train_step = tf.train.AdamOptimizer(learning_rate).minimize(L, global_step=global_step)
# A more simple optimizer
# train_step = tf.train.AdamOptimizer(learning_rate=2.0).minimize(L)
# Set up the summary writer (saving summaries is optional)
# (do `tensorboard --logdir=/tmp/na-logs` to view it)
tf.scalar_summary("L_content", L_content)
tf.scalar_summary("L_style", L_style)
gen_image_addmean = tf.Variable(tf.constant(np.array(content_image, dtype=np.float32)), trainable=False)
tf.image_summary("Generated image (TODO: add mean)", gen_image_addmean)
summary_op = tf.merge_all_summaries()
summary_writer = tf.train.SummaryWriter('/tmp/na-logs', graph_def=sess.graph_def)
print "Start calculation..."
# The optimizer has variables that require initialization as well
sess.run(tf.initialize_all_variables())
for i in range(num_iters):
if i % 10 == 0:
gen_image_val = sess.run(gen_image)
save_image(gen_image_val, i, out_dir)
print "L_content, L_style:", sess.run(L_content), sess.run(L_style)
# Increment summary
sess.run(tf.assign(gen_image_addmean, add_mean(gen_image_val)))
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, i)
print "Iter:", i
sess.run(train_step)
def main():
# make dirs
target_dirs = [args.logdir,
args.test_result_dir] #., args.test_LR_result_dir]
create_dirs(target_dirs)
# set logger
logflag = set_logger(args)
log(logflag, 'Test script start', 'info')
NLR_data = tf.placeholder(tf.float32,
shape=[None, None, None, args.channel],
name='NLR_input')
CLR_data = tf.placeholder(tf.float32,
shape=[None, None, None, args.channel],
name='CLR_input')
NHR_data = tf.placeholder(tf.float32,
shape=[None, None, None, args.channel],
name='NHR_input')
CHR_data = tf.placeholder(tf.float32,
shape=[None, None, None, args.channel],
name='CHR_input')
# build Generator and Discriminator
network = Network(args,
NLR_data=NLR_data,
CLR_data=CLR_data,
NHR_data=NHR_data,
CHR_data=CHR_data,
is_test=True)
CLR_C1, NLR_C1, CLR_C2, CHR_C3, NLR_C3, CHR_C4, CLR_I1, CHR_I1, CHR_I2 = network.train_generator(
)
# define optimizers
global_iter = tf.Variable(0, trainable=False)
gc.collect()
config = tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True, visible_device_list=args.gpu_dev_num))
# Start Session
with tf.Session(config=config) as sess:
log(logflag, 'Start Session', 'info')
sess.run(tf.global_variables_initializer())
sess.run(global_iter.initializer)
saver = tf.train.Saver(max_to_keep=10)
saver.restore(sess, tf.train.latest_checkpoint(args.checkpoint_dir))
validpathLR = np.sort(
np.asarray(glob(os.path.join(args.data_dir, '*.png'))))
validpathHR = np.sort(
np.asarray(glob(os.path.join(args.data_dir, '*.png'))))
import time
avgtime = 0
for valid_i in range(100):
validLR, validHR = generate_testset(validpathLR[valid_i],
validpathHR[valid_i], args)
name = validpathLR[valid_i].split('/')[-1]
validLR = np.transpose(validLR[:, :, :, np.newaxis], (3, 0, 1, 2))
validHR = np.transpose(validHR[:, :, :, np.newaxis], (3, 0, 1, 2))
starttime = time.time()
valid_out, valid_out_LR = sess.run(
[CHR_C3, CLR_C1],
feed_dict={
NLR_data: validLR,
NHR_data: validHR,
CLR_data: validLR,
CHR_data: validHR
})
validLR = np.rot90(validLR, 1, axes=(1, 2))
validHR = np.rot90(validHR, 1, axes=(1, 2))
valid_out90, valid_out_LR90 = sess.run(
[CHR_C3, CLR_C1],
feed_dict={
NLR_data: validLR,
NHR_data: validHR,
CLR_data: validLR,
CHR_data: validHR
})
valid_out += np.rot90(valid_out90, 3, axes=(1, 2))
valid_out_LR += np.rot90(valid_out_LR90, 3, axes=(1, 2))
validLR = np.rot90(validLR, 1, axes=(1, 2))
validHR = np.rot90(validHR, 1, axes=(1, 2))
valid_out90, valid_out_LR90 = sess.run(
[CHR_C3, CLR_C1],
feed_dict={
NLR_data: validLR,
NHR_data: validHR,
CLR_data: validLR,
CHR_data: validHR
})
valid_out += np.rot90(valid_out90, 2, axes=(1, 2))
valid_out_LR += np.rot90(valid_out_LR90, 2, axes=(1, 2))
validLR = np.rot90(validLR, 1, axes=(1, 2))
validHR = np.rot90(validHR, 1, axes=(1, 2))
valid_out90, valid_out_LR90 = sess.run(
[CHR_C3, CLR_C1],
feed_dict={
NLR_data: validLR,
NHR_data: validHR,
CLR_data: validLR,
CHR_data: validHR
})
valid_out += np.rot90(valid_out90, 1, axes=(1, 2))
valid_out_LR += np.rot90(valid_out_LR90, 1, axes=(1, 2))
validLR = np.rot90(validLR, 1, axes=(1, 2))
validHR = np.rot90(validHR, 1, axes=(1, 2))
validLR = validLR[:, ::-1, :, :]
validHR = validHR[:, ::-1, :, :]
valid_out90, valid_out_LR90 = sess.run(
[CHR_C3, CLR_C1],
feed_dict={
NLR_data: validLR,
NHR_data: validHR,
CLR_data: validLR,
CHR_data: validHR
})
valid_out += valid_out90[:, ::-1, :, :]
valid_out_LR += valid_out_LR90[:, ::-1, :, :]
validLR = np.rot90(validLR, 1, axes=(1, 2))
validHR = np.rot90(validHR, 1, axes=(1, 2))
valid_out90, valid_out_LR90 = sess.run(
[CHR_C3, CLR_C1],
feed_dict={
NLR_data: validLR,
NHR_data: validHR,
CLR_data: validLR,
CHR_data: validHR
})
valid_out90 = np.rot90(valid_out90, 3, axes=(1, 2))
valid_out_LR90 = np.rot90(valid_out_LR90, 3, axes=(1, 2))
valid_out += valid_out90[:, ::-1, :, :]
valid_out_LR += valid_out_LR90[:, ::-1, :, :]
validLR = np.rot90(validLR, 1, axes=(1, 2))
validHR = np.rot90(validHR, 1, axes=(1, 2))
valid_out90, valid_out_LR90 = sess.run(
[CHR_C3, CLR_C1],
feed_dict={
NLR_data: validLR,
NHR_data: validHR,
CLR_data: validLR,
CHR_data: validHR
})
valid_out90 = np.rot90(valid_out90, 2, axes=(1, 2))
valid_out_LR90 = np.rot90(valid_out_LR90, 2, axes=(1, 2))
valid_out += valid_out90[:, ::-1, :, :]
valid_out_LR += valid_out_LR90[:, ::-1, :, :]
validLR = np.rot90(validLR, 1, axes=(1, 2))
validHR = np.rot90(validHR, 1, axes=(1, 2))
valid_out90, valid_out_LR90 = sess.run(
[CHR_C3, CLR_C1],
feed_dict={
NLR_data: validLR,
NHR_data: validHR,
CLR_data: validLR,
CHR_data: validHR
})
valid_out90 = np.rot90(valid_out90, 1, axes=(1, 2))
valid_out_LR90 = np.rot90(valid_out_LR90, 1, axes=(1, 2))
valid_out += valid_out90[:, ::-1, :, :]
valid_out_LR += valid_out_LR90[:, ::-1, :, :]
valid_out /= 8.
valid_out_LR /= 8.
currtime = time.time() - starttime
print("time : %fs" % (currtime))
avgtime += currtime / 100
save_image(args, valid_out, 'test', name, save_max_num=1)
#save_image(args, valid_out_LR, 'test_LR', valid_i, save_max_num=5)
print("avg. time : %f" % avgtime)
def test(self):
x_fixed = self.get_image_from_loader()
save_image(x_fixed, '{}/x_fixed_test.png'.format(self.model_dir))
self.autoencode(x_fixed, self.model_dir, idx="test", x_fake=None)
def test(model, data_loader, num_train_batches, epoch, writer):
"""
Evaluate model on validation set
Args:
model: The CapsuleNet model.
data_loader: An interator over the dataset. It combines a dataset and a sampler.
"""
print('===> Evaluate mode')
# Switch to evaluate mode
model.eval()
if args.cuda:
# When we wrap a Module in DataParallel for multi-GPUs
model = model.module
loss = 0
margin_loss = 0
recon_loss = 0
correct = 0
num_batches = len(data_loader)
global_step = epoch * num_train_batches + num_train_batches
step['step'] = global_step
for data, target in data_loader:
batch_size = data.size(0)
target_indices = target
target_one_hot = utils.one_hot_encode(target_indices,
length=args.num_classes)
assert target_one_hot.size() == torch.Size([batch_size, 10])
data, target = Variable(data, volatile=True), Variable(target_one_hot)
if args.cuda:
data = data.cuda()
target = target.cuda()
# Output predictions
output = model(data) # output from DigitCaps (out_digit_caps)
# Sum up batch loss
t_loss, m_loss, r_loss = model.loss(data,
output,
target,
size_average=False)
loss += t_loss.data[0]
margin_loss += m_loss.data[0]
recon_loss += r_loss.data[0]
# Count number of correct predictions
# v_magnitude shape: [128, 10, 1, 1]
v_magnitude = torch.sqrt((output**2).sum(dim=2, keepdim=True))
# pred shape: [128, 1, 1, 1]
pred = v_magnitude.data.max(1, keepdim=True)[1].cpu()
correct += pred.eq(target_indices.view_as(pred)).sum()
# Get the reconstructed images of the last batch
if args.use_reconstruction_loss:
reconstruction = model.decoder(output, target)
# Input image size and number of channel.
# By default, for MNIST, the image width and height is 28x28 and 1 channel for black/white.
image_width = args.input_width
image_height = args.input_height
image_channel = args.num_conv_in_channel
recon_img = reconstruction.view(-1, image_channel, image_width,
image_height)
assert recon_img.size() == torch.Size(
[batch_size, image_channel, image_width, image_height])
# Save the image into file system
utils.save_image(
recon_img,
'results/recons_image_test_{}_{}.png'.format(epoch, global_step))
utils.save_image(
data,
'results/original_image_test_{}_{}.png'.format(epoch, global_step))
# Add and visualize the image in TensorBoard
recon_img = vutils.make_grid(recon_img.data,
normalize=True,
scale_each=True)
original_img = vutils.make_grid(data.data,
normalize=True,
scale_each=True)
writer.add_image('test/recons-image-{}-{}'.format(epoch, global_step),
recon_img, global_step)
writer.add_image(
'test/original-image-{}-{}'.format(epoch, global_step),
original_img, global_step)
# Log test losses
loss /= num_batches
margin_loss /= num_batches
recon_loss /= num_batches
# Log test accuracies
num_test_data = len(data_loader.dataset)
accuracy = correct / num_test_data
accuracy_percentage = 100. * accuracy
# TensorBoard logging
# 1) Log the scalar values
writer.add_scalar('test/total_loss', loss, global_step)
writer.add_scalar('test/margin_loss', margin_loss, global_step)
if args.use_reconstruction_loss:
writer.add_scalar('test/reconstruction_loss', recon_loss, global_step)
writer.add_scalar('test/accuracy', accuracy, global_step)
# Print test losses and accuracy
print('Test: [Loss: {:.6f},' \
'\tMargin loss: {:.6f},' \
'\tReconstruction loss: {:.6f}]'.format(
loss,
margin_loss,
recon_loss if args.use_reconstruction_loss else 0))
print('Test Accuracy: {}/{} ({:.0f}%)\n'.format(correct, num_test_data,
accuracy_percentage))
optimizer.zero_grad()
model(target_image)
style_score = 0
content_score = 0
for s1 in style_losses:
style_score += s1.loss
for c1 in content_losses:
content_score += c1.loss
style_score *= STYLE_WEIGHT
content_score *= CONTENT_WEIGHT
loss = style_score + content_score
loss.backward()
run[0] += 1
if run[0] % 10 == 0:
print('Run: {}'.format(run))
print('Style Loss: {:4f} Content Loss: {:4f}'.format(
style_score.item(), content_score.item()))
print()
return style_score + content_score
optimizer.step(closure)
target_image.data.clamp_(0, 1)
utils.save_image(target_image, OUTPUT_PATH)
def main():
global args, best_result, output_directory, train_csv, test_csv
# evaluation mode
start_epoch = 0
if args.evaluate:
assert os.path.isfile(args.evaluate), \
"=> no best model found at '{}'".format(args.evaluate)
print("=> loading best model '{}'".format(args.evaluate))
checkpoint = torch.load(args.evaluate)
output_directory = os.path.dirname(args.evaluate)
args = checkpoint['args']
start_epoch = checkpoint['epoch'] + 1
best_result = checkpoint['best_result']
model = checkpoint['model']
print("=> loaded best model (epoch {})".format(checkpoint['epoch']))
_, val_loader = create_data_loaders(args)
args.evaluate = True
validate(val_loader, model, checkpoint['epoch'], write_to_file=False)
return
# optionally resume from a checkpoint
elif args.resume:
chkpt_path = args.resume
assert os.path.isfile(chkpt_path), \
"=> no checkpoint found at '{}'".format(chkpt_path)
print("=> loading checkpoint '{}'".format(chkpt_path))
checkpoint = torch.load(chkpt_path)
args = checkpoint['args']
start_epoch = checkpoint['epoch'] + 1
best_result = checkpoint['best_result']
model = checkpoint['model']
optimizer = checkpoint['optimizer']
output_directory = os.path.dirname(os.path.abspath(chkpt_path))
print("=> loaded checkpoint (epoch {})".format(checkpoint['epoch']))
train_loader, val_loader = create_data_loaders(args)
args.resume = True
# create new model
else:
train_loader, val_loader = create_data_loaders(args)
print("=> creating Model ({}-{}) ...".format(args.arch, args.decoder))
in_channels = len(args.modality)
if args.arch == 'resnet50':
model = ResNet(layers=50,
decoder=args.decoder,
output_size=train_loader.dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet18':
model = ResNet(layers=18,
decoder=args.decoder,
output_size=train_loader.dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
print("=> model created.")
optimizer = torch.optim.SGD(model.parameters(), args.lr, \
momentum=args.momentum, weight_decay=args.weight_decay)
# model = torch.nn.DataParallel(model).cuda() # for multi-gpu training
model = model.cuda()
# define loss function (criterion) and optimizer
if args.criterion == 'l2':
criterion = criteria.MaskedMSELoss().cuda()
elif args.criterion == 'l1':
criterion = criteria.MaskedL1Loss().cuda()
# create results folder, if not already exists
output_directory = utils.get_output_directory(args)
if not os.path.exists(output_directory):
os.makedirs(output_directory)
train_csv = os.path.join(output_directory, 'train.csv')
test_csv = os.path.join(output_directory, 'test.csv')
best_txt = os.path.join(output_directory, 'best.txt')
# create new csv files with only header
if not args.resume:
with open(train_csv, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
with open(test_csv, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for epoch in range(start_epoch, args.epochs):
utils.adjust_learning_rate(optimizer, epoch, args.lr)
train(train_loader, model, criterion, optimizer,
epoch) # train for one epoch
result, img_merge = validate(val_loader, model,
epoch) # evaluate on validation set
# remember best rmse and save checkpoint
is_best = result.rmse < best_result.rmse
if is_best:
best_result = result
with open(best_txt, 'w') as txtfile:
txtfile.write(
"epoch={}\nmse={:.3f}\nrmse={:.3f}\nabsrel={:.3f}\nlg10={:.3f}\nmae={:.3f}\ndelta1={:.3f}\nt_gpu={:.4f}\n"
.format(epoch, result.mse, result.rmse, result.absrel,
result.lg10, result.mae, result.delta1,
result.gpu_time))
if img_merge is not None:
img_filename = output_directory + '/comparison_best.png'
utils.save_image(img_merge, img_filename)
utils.save_checkpoint(
{
'args': args,
'epoch': epoch,
'arch': args.arch,
'model': model,
'best_result': best_result,
'optimizer': optimizer,
}, is_best, epoch, output_directory)
def validate(val_loader, model, epoch, write_to_file=True):
average_meter = AverageMeter()
model.eval() # switch to evaluate mode
end = time.time()
for i, (input, target) in enumerate(val_loader):
if configuration_file.GPU == True:
input, target = input.cuda(), target.cuda()
# torch.cuda.synchronize()
data_time = time.time() - end
# compute output
end = time.time()
with torch.no_grad():
pred = model(input)
# torch.cuda.synchronize()
gpu_time = time.time() - end
# measure accuracy and record loss
result = Result()
result.evaluate(pred.data, target.data)
average_meter.update(result, gpu_time, data_time, input.size(0))
end = time.time()
# save 8 images for visualization
skip = 50
if args.modality == 'rgb':
rgb = input
if i == 0:
img_merge = utils.merge_into_row(rgb, target, pred)
elif (i < 8 * skip) and (i % skip == 0):
row = utils.merge_into_row(rgb, target, pred)
img_merge = utils.add_row(img_merge, row)
elif i == 8 * skip:
filename = output_directory + '/comparison_' + str(epoch) + '.png'
utils.save_image(img_merge, filename)
if (i + 1) % args.print_freq == 0:
print('Test: [{0}/{1}]\t'
't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t'
'RMSE={result.rmse:.2f}({average.rmse:.2f}) '
'MAE={result.mae:.2f}({average.mae:.2f}) '
'Delta1={result.delta1:.3f}({average.delta1:.3f}) '
'REL={result.absrel:.3f}({average.absrel:.3f}) '
'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format(
i + 1,
len(val_loader),
gpu_time=gpu_time,
result=result,
average=average_meter.average()))
avg = average_meter.average()
print('\n*\n'
'RMSE={average.rmse:.3f}\n'
'MAE={average.mae:.3f}\n'
'Delta1={average.delta1:.3f}\n'
'REL={average.absrel:.3f}\n'
'Lg10={average.lg10:.3f}\n'
't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time))
if write_to_file:
with open(test_csv, 'a') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writerow({
'mse': avg.mse,
'rmse': avg.rmse,
'absrel': avg.absrel,
'lg10': avg.lg10,
'mae': avg.mae,
'delta1': avg.delta1,
'delta2': avg.delta2,
'delta3': avg.delta3,
'data_time': avg.data_time,
'gpu_time': avg.gpu_time
})
return avg, img_merge
from MIDIUtil.Melody import Melody
from MIDIUtil.Note import Note
from MIDIUtil.MIDIReader import MIDIReader
from utils import save_image
import numpy as np
reader = MIDIReader()
melody = reader.read_file(
'/home/miguel/src/GeneticComposition/data/training_test/mary_had_a_little_lamb.mid'
)[0][0]
image = np.zeros((len(melody.notes), 129), dtype=np.int32)
for i, note in enumerate(melody.notes):
image[i, note.getMIDIIndex()] = 1
image = image.T
save_image(
image,
'/home/miguel/src/GeneticComposition/data/training_test/mary_had_a_little_lamb.png'
)
def run_one_order(padded_input, order, adj, loc, all_patches, patch_size,
model, context, final_outdir, style_name, disp_name,
binary_thresh):
styledResults = {}
full_destyled = (patch_size - context) * (patch_size - context)
nump = len(all_patches)
rf_score = np.zeros((nump, 1))
h, w, c = padded_input.shape
res = copy.deepcopy(padded_input)
mask = np.ones([h, w])
transform = get_transform(opt) #, grayscale=True)
all_can = []
all_can.append(copy.deepcopy(res))
adj += adj.transpose()
visited = np.zeros((adj.shape[0]), dtype=int)
margin = []
for idx, o in enumerate(order):
if idx == 0:
first = o
visited[o] = 1
tmp = adj[o, :]
nidx = [x for x in range(len(tmp)) if tmp[x] > 0]
noverlap = tmp[nidx]
vnidx = visited[nidx]
vnidx2 = [ix for ix, x in enumerate(vnidx) if x == 1]
o_dep = np.asarray([nidx[index] for index in vnidx2])
o_dep_overlap = np.asarray([noverlap[index] for index in vnidx2])
curPatch = all_patches[o]
cloc = loc[o]
c = cloc
hybrid = res[c[0]:c[0] + patch_size, c[1]:c[1] + patch_size, :]
cmask = mask[c[0]:c[0] + patch_size, c[1]:c[1] + patch_size]
msk = 255 - np.sum(hybrid, axis=2) / 3
msk = msk.astype(int)
unstyled = msk[cmask == 1]
nz = np.count_nonzero(unstyled)
if nz == 0:
margin = margin + [o]
mask[c[0]:c[0] + patch_size, c[1]:c[1] + patch_size] = -1
continue
cres = translate_patch(hybrid, cmask, o, c, model)
x1, x2, y1, y2 = 0, 0, 0, 0
buf = round(context / 2)
buf2 = round(buf / 2)
for dp in o_dep:
if dp == o - 1:
y1 = buf
olreg1 = res[c[0]:c[0] + patch_size, c[1]:c[1] + context, :]
olreg2 = cres[:, 0:context, :]
blended = blend_overlap(olreg1, olreg2)
cres[:, 0:context, :] = blended
if dp == o + 1:
y2 = -buf
olreg2 = res[c[0]:c[0] + patch_size,
c[1] + patch_size - context:c[1] + patch_size, :]
olreg1 = cres[:, patch_size - context:patch_size, :]
blended = blend_overlap(olreg1, olreg2)
cres[:, patch_size - context:patch_size, :] = blended
if dp < o - 1:
x1 = buf
olreg1 = res[c[0]:c[0] + context, c[1]:c[1] + patch_size, :]
olreg2 = cres[0:context, :, :]
blended = blend_overlap(olreg1, olreg2)
cres[0:context, :, :] = blended
if dp > o + 1:
x2 = -buf
olreg2 = res[c[0] + patch_size - context:c[0] + patch_size,
c[1]:c[1] + patch_size, :]
olreg1 = cres[patch_size - context:patch_size, :, :]
blended = blend_overlap(olreg1, olreg2)
cres[patch_size - context:patch_size, :, :] = blended
res[c[0]:c[0] + patch_size, c[1]:c[1] + patch_size] = cres
mask[c[0]:c[0] + patch_size, c[1]:c[1] + patch_size] = -1
all_can.append(copy.deepcopy(res))
name_str = '%s_%s_%d' % (style_name, disp_name, first)
file_name = os.path.join(final_outdir, name_str + '.png')
samename = glob.glob(file_name)
currnum = str(len(samename) + 1)
sname = os.path.join(final_outdir, name_str + '_' + currnum + '.png')
print(sname)
utils.save_image(res, sname)
def train(self):
print('\n{}\n'.format(self.config))
# Save some fixed images once.
z_fixed = np.random.normal(0, 1, size=(self.batch_size, self.z_dim))
x_fixed = self.get_images_from_loader()
save_image(x_fixed, '{}/x_fixed.png'.format(self.model_dir))
# Use tensorflow tutorial set for conveniently labeled mnist.
# NOTE: Data originally on [0,1], but immediately converted to [0, 255].
self.mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
self.images_train = self.prep_data(split='train', n=8000)
# Train generator.
for step in trange(self.start_step, self.max_step):
# For MMDGAN training, use data with predicted weights.
batch_train = self.get_n_images(self.batch_size, self.images_train)
# Let training focus on ae_loss in case it is high.
ae_loss_out = self.sess.run(self.ae_loss,
feed_dict={
self.x: batch_train,
self.lambda_mmd: self.lambda_mmd_setting,
self.lambda_ae: self.lambda_ae_setting})
# Set up basket of items to be run. Occasionally fetch items
# useful for logging and saving.
if ae_loss_out > 0.5:
fetch_dict = {
'd_optim': self.d_optim,
}
else:
fetch_dict = {
'd_optim': self.d_optim,
'g_optim': self.g_optim,
}
if step % self.log_step == 0:
fetch_dict.update({
'summary': self.summary_op,
'ae_loss_real': self.ae_loss_real,
'ae_loss_fake': self.ae_loss_fake,
'mmd': self.mmd,
'ncmd_k': self.ncmd_k,
'x_enc': self.x_enc,
'g_enc': self.g_enc,
})
# Run full training step on pre-fetched data and simulations.
result = self.sess.run(fetch_dict,
feed_dict={
self.x: batch_train,
self.lambda_mmd: self.lambda_mmd_setting,
self.lambda_ae: self.lambda_ae_setting})
if step % self.lr_update_step == self.lr_update_step - 1:
self.sess.run([self.g_lr_update, self.d_lr_update])
# Log and save as needed.
if step % self.log_step == 0:
self.summary_writer.add_summary(result['summary'], step)
self.summary_writer.flush()
ae_loss_real = result['ae_loss_real']
ae_loss_fake = result['ae_loss_fake']
mmd = result['mmd']
ncmd_k = result['ncmd_k']
print(('[{}/{}] LOSSES: ae_real/fake: {:.3f}, {:.3f} '
'mmd: {:.3f}, ncmd_k: {:.3f}').format(
step, self.max_step, ae_loss_real, ae_loss_fake, mmd,
ncmd_k))
print(np.round(np.percentile(result['x_enc'],
[0, 20, 50, 80, 100]), 2))
print(np.round(np.percentile(result['g_enc'],
[0, 20, 50, 80, 100]), 2))
if step % (self.save_step) == 0:
# First save a sample.
if step == 0:
x_samp = self.get_n_images(1, self.images_train)
save_image(x_samp, '{}/x_samp.png'.format(self.model_dir))
# Save images for fixed and random z.
gen_fixed = self.generate(
z_fixed, root_path=self.model_dir, step='fix'+str(step),
save=True)
z = np.random.normal(0, 1, size=(self.batch_size, self.z_dim))
gen_rand = self.generate(
z, root_path=self.model_dir, step='rand'+str(step),
save=True)
# Save image of interpolation of z.
self.interpolate_z(step)
def validate(val_loader, model, epoch, write_to_file=True):
average_meter = AverageMeter()
# switch to evaluate mode
model.eval()
end = time.time()
for i, (input, target) in enumerate(val_loader):
input, target = input.to(device), target.to(device)
torch.cuda.synchronize()
data_time = time.time() - end
# compute output
end = time.time()
depth_pred = model(input)
torch.cuda.synchronize()
gpu_time = time.time() - end
# measure accuracy and record loss
result = Result()
output1 = torch.index_select(depth_pred, 1,
torch.LongTensor([0]).to(device))
result.evaluate(output1, target)
average_meter.update(result, gpu_time, data_time, input.size(0))
end = time.time()
# save 8 images for visualization
skip = 50
if args.modality == 'd':
img_merge = None
else:
if args.modality == 'rgb':
rgb = input
elif args.modality == 'rgbd':
rgb = input[:, :3, :, :]
depth = input[:, 3:, :, :]
if i == 0:
if args.modality == 'rgbd':
img_merge = utils.merge_into_row_with_gt(
rgb, depth, target, depth_pred)
else:
img_merge = utils.merge_into_row(rgb, target, depth_pred)
elif (i < 8 * skip) and (i % skip == 0):
if args.modality == 'rgbd':
row = utils.merge_into_row_with_gt(rgb, depth, target,
depth_pred)
else:
row = utils.merge_into_row(rgb, target, depth_pred)
img_merge = utils.add_row(img_merge, row)
elif i == 8 * skip:
filename = output_directory + '/comparison_' + str(
epoch) + '.png'
utils.save_image(img_merge, filename)
if (i + 1) % args.print_freq == 0:
print('Test: [{0}/{1}]\t'
't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\t'
'RMSE={result.rmse:.2f}({average.rmse:.2f}) '
'MAE={result.mae:.2f}({average.mae:.2f}) '
'Delta1={result.delta1:.3f}({average.delta1:.3f}) '
'REL={result.absrel:.3f}({average.absrel:.3f}) '
'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format(
i + 1,
len(val_loader),
gpu_time=gpu_time,
result=result,
average=average_meter.average()))
avg = average_meter.average()
print('\n*\n'
'RMSE={average.rmse:.3f}\n'
'MAE={average.mae:.3f}\n'
'Delta1={average.delta1:.3f}\n'
'REL={average.absrel:.3f}\n'
'Lg10={average.lg10:.3f}\n'
't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time))
if write_to_file:
with open(test_csv, 'a') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writerow({
'mse': avg.mse.item(),
'rmse': avg.rmse.item(),
'absrel': avg.absrel.item(),
'lg10': avg.lg10.item(),
'mae': avg.mae.item(),
'delta1': avg.delta1.item(),
'delta2': avg.delta2.item(),
'delta3': avg.delta3.item(),
'gpu_time': avg.gpu_time,
'data_time': avg.data_time
})
return avg, img_merge
def main():
global args, best_result, output_directory, train_csv, test_csv, device
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
args = parser.parse_args()
if args.modality == 'rgb' and args.num_samples != 0:
print("number of samples is forced to be 0 when input modality is rgb")
args.num_samples = 0
if args.modality == 'rgb' and args.max_depth != 0.0:
print("max depth is forced to be 0.0 when input modality is rgb/rgbd")
args.max_depth = 0.0
sparsifier = None
max_depth = args.max_depth if args.max_depth >= 0.0 else np.inf
if args.sparsifier == UniformSampling.name:
sparsifier = UniformSampling(num_samples=args.num_samples,
max_depth=max_depth)
elif args.sparsifier == SimulatedStereo.name:
sparsifier = SimulatedStereo(num_samples=args.num_samples,
max_depth=max_depth)
# create results folder, if not already exists
output_directory = os.path.join(
'results',
'{}.sparsifier={}.modality={}.arch={}.decoder={}.criterion={}.lr={}.bs={}'
.format(args.data, sparsifier, args.modality, args.arch, args.decoder,
args.criterion, args.lr, args.batch_size))
if not os.path.exists(output_directory):
os.makedirs(output_directory)
train_csv = os.path.join(output_directory, 'train.csv')
test_csv = os.path.join(output_directory, 'test.csv')
best_txt = os.path.join(output_directory, 'best.txt')
# define loss function (criterion) and optimizer
if args.criterion == 'l2':
criterion = criteria.MaskedMSELoss().cuda()
elif args.criterion == 'l1':
criterion = criteria.MaskedL1Loss().cuda()
out_channels = 1
# Data loading code
print("=> creating data loaders ...")
traindir = os.path.join('data', args.data, 'train')
valdir = os.path.join('data', args.data, 'val')
train_dataset = NYUDataset(traindir,
type='train',
modality=args.modality,
sparsifier=sparsifier)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True,
sampler=None)
# set batch size to be 1 for validation
val_dataset = NYUDataset(valdir,
type='val',
modality=args.modality,
sparsifier=sparsifier)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
print("=> data loaders created.")
# evaluation mode
if args.evaluate:
best_model_filename = os.path.join(output_directory,
'model_best.pth.tar')
if os.path.isfile(best_model_filename):
print("=> loading best model '{}'".format(best_model_filename))
checkpoint = torch.load(best_model_filename)
args.start_epoch = checkpoint['epoch']
best_result = checkpoint['best_result']
model = checkpoint['model']
print("=> loaded best model (epoch {})".format(
checkpoint['epoch']))
else:
print("=> no best model found at '{}'".format(best_model_filename))
validate(val_loader, model, checkpoint['epoch'], write_to_file=False)
return
# optionally resume from a checkpoint
elif args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch'] + 1
best_result = checkpoint['best_result']
model = checkpoint['model']
optimizer = checkpoint['optimizer']
print("=> loaded checkpoint (epoch {})".format(
checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
return
# create new model
else:
# define model
print("=> creating Model ({}-{}) ...".format(args.arch, args.decoder))
in_channels = len(args.modality)
if args.arch == 'resnet50':
model = ResNet(layers=50,
decoder=args.decoder,
in_channels=in_channels,
out_channels=out_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet18':
model = ResNet(layers=18,
decoder=args.decoder,
in_channels=in_channels,
out_channels=out_channels,
pretrained=args.pretrained)
print("=> model created.")
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# create new csv files with only header
with open(train_csv, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
with open(test_csv, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
# model = torch.nn.DataParallel(model).cuda()
model = model.to(device)
print(model)
print("=> model transferred to GPU.")
for epoch in range(args.start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
# train for one epoch
train(train_loader, model, criterion, optimizer, epoch)
# evaluate on validation set
result, img_merge = validate(val_loader, model, epoch)
# remember best rmse and save checkpoint
is_best = result.rmse < best_result.rmse
if is_best:
best_result = result
with open(best_txt, 'w') as txtfile:
txtfile.write(
"epoch={}\nmse={:.3f}\nrmse={:.3f}\nabsrel={:.3f}\nlg10={:.3f}\nmae={:.3f}\ndelta1={:.3f}\nt_gpu={:.4f}\n"
.format(epoch, result.mse, result.rmse, result.absrel,
result.lg10, result.mae, result.delta1,
result.gpu_time))
if img_merge is not None:
img_filename = output_directory + '/comparison_best.png'
utils.save_image(img_merge, img_filename)
save_checkpoint(
{
'epoch': epoch,
'arch': args.arch,
'model': model,
'best_result': best_result,
'optimizer': optimizer,
}, is_best, epoch)
def train(args):
# GPU enabling
if (args.gpu != None):
use_cuda = True
dtype = torch.cuda.FloatTensor
torch.cuda.set_device(args.gpu)
print("Current device: %d" %torch.cuda.current_device())
# visualization of training controlled by flag
visualize = (args.visualize != None)
if (visualize):
img_transform_512 = transforms.Compose([
transforms.Scale(512), # scale shortest side to image_size
transforms.CenterCrop(512), # crop center image_size out
transforms.ToTensor(), # turn image from [0-255] to [0-1]
utils.normalize_tensor_transform() # normalize with ImageNet values
])
testImage_amber = utils.load_image("content_imgs/amber.jpg")
testImage_amber = img_transform_512(testImage_amber)
testImage_amber = Variable(testImage_amber.repeat(1, 1, 1, 1), requires_grad=False).type(dtype)
testImage_dan = utils.load_image("content_imgs/dan.jpg")
testImage_dan = img_transform_512(testImage_dan)
testImage_dan = Variable(testImage_dan.repeat(1, 1, 1, 1), requires_grad=False).type(dtype)
testImage_maine = utils.load_image("content_imgs/maine.jpg")
testImage_maine = img_transform_512(testImage_maine)
testImage_maine = Variable(testImage_maine.repeat(1, 1, 1, 1), requires_grad=False).type(dtype)
# define network
image_transformer = ImageTransformNet().type(dtype)
optimizer = Adam(image_transformer.parameters(), LEARNING_RATE)
loss_mse = torch.nn.MSELoss()
# load vgg network
vgg = Vgg16().type(dtype)
# get training dataset
dataset_transform = transforms.Compose([
transforms.Scale(IMAGE_SIZE), # scale shortest side to image_size
transforms.CenterCrop(IMAGE_SIZE), # crop center image_size out
transforms.ToTensor(), # turn image from [0-255] to [0-1]
utils.normalize_tensor_transform() # normalize with ImageNet values
])
train_dataset = datasets.ImageFolder(args.dataset, dataset_transform)
train_loader = DataLoader(train_dataset, batch_size = BATCH_SIZE)
# style image
style_transform = transforms.Compose([
transforms.ToTensor(), # turn image from [0-255] to [0-1]
utils.normalize_tensor_transform() # normalize with ImageNet values
])
style = utils.load_image(args.style_image)
style = style_transform(style)
style = Variable(style.repeat(BATCH_SIZE, 1, 1, 1)).type(dtype)
style_name = os.path.split(args.style_image)[-1].split('.')[0]
# calculate gram matrices for style feature layer maps we care about
style_features = vgg(style)
style_gram = [utils.gram(fmap) for fmap in style_features]
for e in range(EPOCHS):
# track values for...
img_count = 0
aggregate_style_loss = 0.0
aggregate_content_loss = 0.0
aggregate_tv_loss = 0.0
# train network
image_transformer.train()
for batch_num, (x, label) in enumerate(train_loader):
img_batch_read = len(x)
img_count += img_batch_read
# zero out gradients
optimizer.zero_grad()
# input batch to transformer network
x = Variable(x).type(dtype)
y_hat = image_transformer(x)
# get vgg features
y_c_features = vgg(x)
y_hat_features = vgg(y_hat)
# calculate style loss
y_hat_gram = [utils.gram(fmap) for fmap in y_hat_features]
style_loss = 0.0
for j in range(4):
style_loss += loss_mse(y_hat_gram[j], style_gram[j][:img_batch_read])
style_loss = STYLE_WEIGHT*style_loss
aggregate_style_loss += style_loss.data.item()
# calculate content loss (h_relu_2_2)
recon = y_c_features[1]
recon_hat = y_hat_features[1]
content_loss = CONTENT_WEIGHT*loss_mse(recon_hat, recon)
aggregate_content_loss += content_loss.data.item()
# calculate total variation regularization (anisotropic version)
# https://www.wikiwand.com/en/Total_variation_denoising
diff_i = torch.sum(torch.abs(y_hat[:, :, :, 1:] - y_hat[:, :, :, :-1]))
diff_j = torch.sum(torch.abs(y_hat[:, :, 1:, :] - y_hat[:, :, :-1, :]))
tv_loss = TV_WEIGHT*(diff_i + diff_j)
aggregate_tv_loss += tv_loss.data.item()
# total loss
total_loss = style_loss + content_loss + tv_loss
# backprop
total_loss.backward()
optimizer.step()
# print out status message
if ((batch_num + 1) % 100 == 0):
status = "{} Epoch {}: [{}/{}] Batch:[{}] agg_style: {:.6f} agg_content: {:.6f} agg_tv: {:.6f} style: {:.6f} content: {:.6f} tv: {:.6f} ".format(
time.ctime(), e + 1, img_count, len(train_dataset), batch_num+1,
aggregate_style_loss/(batch_num+1.0), aggregate_content_loss/(batch_num+1.0), aggregate_tv_loss/(batch_num+1.0),
style_loss.item(), content_loss.item(), tv_loss.item()
)
print(status)
if ((batch_num + 1) % 1000 == 0) and (visualize):
image_transformer.eval()
if not os.path.exists("visualization"):
os.makedirs("visualization")
if not os.path.exists("visualization/%s" %style_name):
os.makedirs("visualization/%s" %style_name)
outputTestImage_amber = image_transformer(testImage_amber).cpu()
amber_path = "visualization/%s/amber_%d_%05d.jpg" %(style_name, e+1, batch_num+1)
utils.save_image(amber_path, outputTestImage_amber.data[0])
outputTestImage_dan = image_transformer(testImage_dan).cpu()
dan_path = "visualization/%s/dan_%d_%05d.jpg" %(style_name, e+1, batch_num+1)
utils.save_image(dan_path, outputTestImage_dan.data[0])
outputTestImage_maine = image_transformer(testImage_maine).cpu()
maine_path = "visualization/%s/maine_%d_%05d.jpg" %(style_name, e+1, batch_num+1)
utils.save_image(maine_path, outputTestImage_maine.data[0])
print("images saved")
image_transformer.train()
# save model
image_transformer.eval()
if use_cuda:
image_transformer.cpu()
if not os.path.exists("models"):
os.makedirs("models")
filename = "models/" + str(style_name) + "_" + str(time.ctime()).replace(' ', '_') + ".model"
torch.save(image_transformer.state_dict(), filename)
if use_cuda:
image_transformer.cuda()
def build_graph(content_feat_map, style_grams, content_image, max_steps,
output_dir, output_image_name, option_weights, scope):
print "Make graph for new image..."
# initial = tf.random_normal(content_image.shape) * 0.256
initial = tf.truncated_normal(content_image.shape, stddev=20)
image = tf.Variable(initial, trainable=True)
image_vars_set = set(tf.all_variables())
net = VGG19({'data': image}, scope=scope, trainable=False)
with tf.Session() as sess:
# Load the data
net.load(neural_config.model_path, sess)
# Forward pass
feature_maps = [net.layers[tensor] for tensor in neural_config.new_image_layers]
img_content_feat_map = feature_maps[0]
utils.activation_summary(img_content_feat_map, "gen_image_content_map")
img_style_grams = []
for index, layer in enumerate(neural_config.style_layers):
feat_map = feature_maps[index + 1]
layer_shape = feat_map.get_shape().dims
size = layer_shape[1].value * layer_shape[1].value * layer_shape[3].value
features = tf.reshape(feat_map, (-1, layer_shape[3].value))
gram = tf.matmul(tf.transpose(features), features) / size
img_style_grams.append(gram)
utils.activation_summary(gram, "gen_image_style_gram%d" % index)
# content loss
content_loss = option_weights[0] * (2 * tf.nn.l2_loss(
img_content_feat_map - content_feat_map) / content_feat_map.size)
tf.scalar_summary("content_loss", content_loss)
# style loss
style_loss = 0
style_losses = []
for index, style_layer in enumerate(neural_config.style_layers):
style_losses.append(2 * tf.nn.l2_loss( img_style_grams[index] - style_grams[index]) / style_grams[index].size)
style_loss += option_weights[1] * reduce(tf.add, style_losses)
tf.scalar_summary("style_loss", style_loss)
# total variation denoising
tv_y_size = utils.getTensorSize(image[:,1:,:,:])
tv_x_size = utils.getTensorSize(image[:,:,1:,:])
tv_loss = option_weights[2] * 2 * (
(tf.nn.l2_loss(image[:,1:,:,:] - image[:,:content_image.shape[1]-1,:,:]) /
tv_y_size) +
(tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:content_image.shape[2]-1,:]) /
tv_x_size))
tf.scalar_summary("tv_loss", tv_loss)
# overall loss
loss = content_loss + style_loss + tv_loss
tf.scalar_summary("total_loss", loss)
temp = set(tf.all_variables())
# optimizer setup
# opt = tf.train.AdamOptimizer(neural_config.learning_rate).minimize(loss)
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(learning_rate=neural_config.learning_rate, global_step=global_step,
decay_steps=neural_config.decay_steps,
decay_rate=neural_config.decay_rate,
staircase=True)
opt = tf.train.AdamOptimizer(learning_rate).minimize(loss, global_step=global_step)
vars = set(tf.all_variables()) - temp
vars = vars.union(image_vars_set)
# vars.add(image)
init = tf.initialize_variables(vars)
sess.run(init)
# optimization
best_loss = float('inf')
for step in xrange(max_steps):
_, content_loss_value, style_loss_value, loss_value = sess.run([opt, content_loss, style_loss, loss])
format_str = ('step %d, content_loss_value = %.2f, style_loss_value = %.2f, loss = %.2f')
print (format_str % (step, content_loss_value, style_loss_value, loss_value))
last_step = step == neural_config.max_iter - 1
if (neural_config.checkpoint_steps and step % neural_config.checkpoint_steps == 0) or last_step:
if loss_value < best_loss:
best_loss = loss_value
best = sess.run(image)
# best_image = utils.save_image(best, step, output_dir)
utils.add_mean(best)
tf.image_summary(("images/step%d" % step), best, name="gen_image")
summary_op = tf.merge_all_summaries()
summary_writer = tf.train.SummaryWriter(neural_config.train_dir)
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, step)
# if last_step:
utils.save_image(best, step, output_dir, output_image_name)
def main():
global args, best_result, output_directory, train_csv, test_csv
print(args)
start = 0
# evaluation mode
if args.evaluate:
datasets = configuration_file.datasets_path
valdir = os.path.join(datasets, args.data, 'val')
print("Validation directory ", valdir)
if args.data == 'nyudepthv2':
from dataloaders.nyu import NYUDataset
val_dataset = NYUDataset(valdir,
split='val',
modality=args.modality)
else:
raise RuntimeError('Dataset not found.')
#set batch size to be 1 for validation
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
print("=> validation loaders created.")
assert os.path.isfile(args.evaluate), \
"=> no model found at '{}'".format(args.evaluate)
print("=> loading model '{}'".format(args.evaluate))
checkpoint = torch.load(args.evaluate)
if type(checkpoint) is dict:
args.start_epoch = checkpoint['epoch']
best_result = checkpoint['best_result']
model = checkpoint['model']
print("=> loaded best model (epoch {})".format(
checkpoint['epoch']))
else:
model = checkpoint
args.start_epoch = 0
output_directory = os.path.dirname(args.evaluate)
validate(val_loader, model, args.start_epoch, write_to_file=False)
return
# resume from a particular check point
elif args.resume:
chkpt_path = args.resume
assert os.path.isfile(
chkpt_path), "=> no checkpoint found at '{}'".format(chkpt_path)
print("=> loading checkpoint '{}'".format(chkpt_path))
checkpoint = torch.load(chkpt_path)
args = checkpoint['args']
start_epoch = checkpoint['epoch'] + 1 # load epoch number
start = start_epoch # resume from the checkpoint epoch
best_result = checkpoint['best_result'] # load best result
model = checkpoint['model'] # load model
optimizer = checkpoint['optimizer'] # load optimizer
output_directory = os.path.dirname(os.path.abspath(chkpt_path))
print("=> loaded checkpoint (epoch {})".format(checkpoint['epoch']))
train_loader, val_loader = create_data_loaders(
args) # create data loader
args.resume = True
# create new model if checkpoint does not exist
elif args.train:
train_loader, val_loader = create_data_loaders(
args) # load train and validation data
print("=> creating Model ({}-{}) ...".format(args.arch, args.decoder))
in_channels = len(args.modality)
if args.arch == 'MobileNet': # if encoder is MobileNet
model = models.MobileNetSkipAdd(
output_size=train_loader.dataset.output_size
) # MobileNet model is created
else:
model = models.MobileNetSkipAdd(
output_size=train_loader.dataset.output_size
) # by default MobileNet
print("=> model created.")
optimizer = torch.optim.SGD(model.parameters(), args.lr, \
momentum=args.momentum, weight_decay=args.weight_decay) # configure optimizer
if configuration_file.GPU == True:
if configuration_file.MULTI_GPU == True: # training on multiple GPU
model = torch.nn.DataParallel(model).cuda()
else: # training on single GPU
model = model.cuda()
else:
pass
# define loss function and optimizer
if args.criterion == 'l2':
if configuration_file.GPU == True:
criterion = MaskedMSELoss().cuda()
else:
criterion = MaskedMSELoss()
elif args.criterion == 'l1':
if configuration_file.GPU == True:
criterion = MaskedL1Loss().cuda()
else:
criterion = MaskedL1Loss()
# create results folder, if not already exists
output_directory = utils.get_output_directory(args)
if not os.path.exists(output_directory): # create new directory
os.makedirs(output_directory)
train_csv = os.path.join(output_directory,
'train.csv') # store training result
test_csv = os.path.join(output_directory, 'test.csv') # store test result
best_txt = os.path.join(output_directory, 'best.txt') # store best result
# create new csv files with only header
if not args.resume:
with open(train_csv, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
with open(test_csv, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
# training is strarted from here
for epoch in range(start, args.epochs):
utils.adjust_learning_rate(optimizer, epoch, args.lr)
train(train_loader, model, criterion, optimizer,
epoch) # train for one epoch
result, img_merge = validate(val_loader, model,
epoch) # evaluate on validation set
# remember best rmse and save checkpoint
is_best = result.rmse < best_result.rmse # compare result of the current epoch and best result
if is_best:
best_result = result
with open(best_txt, 'w') as txtfile:
txtfile.write(
"epoch={}\nmse={:.3f}\nrmse={:.3f}\nabsrel={:.3f}\nlg10={:.3f}\nmae={:.3f}\ndelta1={:.3f}\nt_gpu={:.4f}\n"
.format(epoch, result.mse, result.rmse, result.absrel,
result.lg10, result.mae, result.delta1,
result.gpu_time))
if img_merge is not None:
img_filename = output_directory + '/comparison_best.png'
utils.save_image(img_merge, img_filename)
utils.save_checkpoint(
{
'args': args,
'epoch': epoch,
'arch': args.arch,
'model': model,
'best_result': best_result,
'optimizer': optimizer,
}, is_best, epoch, output_directory)
def run(self):
self.slotsStartWatch.start()
self.slotsEndWatch.start()
while not self.stopped_thread():
self.check_timeouts()
new_timeout = self.get_next_timeout()
readable, writable, exceptional = select.select(
self.inputs, [], self.inputs, new_timeout)
for s in readable:
if s is self.sock_tcp:
(connection, address) = self.sock_tcp.accept()
connection.setblocking(False)
self.logger.info('TCP CON (%s, %d)' %
(address[0], address[1]))
self.inputs.append(connection)
else:
buffer = utils.read_data_from_socket(s)
if buffer:
self.logger.debug(
'Req_data = %s\t client = (%s, %d)' %
(buffer, s.getpeername()[0], s.getpeername()[1]))
action, data = utils.segment_packet(buffer)
if action == utils.GATEWAY_NODES_FLASH:
image_name = self.get_image_name(s)
result = []
for node in data[db_utils.NODES]:
if node['status'] == utils.FLASHED:
gateway_id = self.dbConnector.find_gateway_by_addr(
s.getpeername())
node_uid = self.dbConnector.get_node_uid_by_gateway_id_and_node_id(
gateway_id, node['node_id'])
self.dbConnector.node_update_flash_info(
node_uid, 'FINISHED', image_name)
_node = {
'_id': node_uid,
'status': node['status']
}
else: # node['status'] = utils.ERROR
gateway_id = self.dbConnector.find_gateway_by_addr(
s.getpeername())
node_uid = self.dbConnector.get_node_uid_by_gateway_id_and_node_id(
gateway_id, node['node_id'])
_node = {
'_id': node_uid,
'status': node['status']
}
result.append(_node)
self.handle_gateway_request(s, result)
self.inputs.remove(s)
s.close()
elif action == utils.GATEWAY_NODES_ERASE:
result = []
for node in data[db_utils.NODES]:
if node['status'] == utils.ERASED:
gateway_id = self.dbConnector.find_gateway_by_addr(
s.getpeername())
node_uid = self.dbConnector.get_node_uid_by_gateway_id_and_node_id(
gateway_id, node['node_id'])
self.dbConnector.node_update_flash_info(
node_uid, 'NOT_STARTED', None)
_node = {
'_id': node_uid,
'status': node['status']
}
else: # node['status'] = utils.ERROR
gateway_id = self.dbConnector.find_gateway_by_addr(
s.getpeername())
node_uid = self.dbConnector.get_node_uid_by_gateway_id_and_node_id(
gateway_id, node['node_id'])
_node = {
'_id': node_uid,
'status': node['status']
}
result.append(_node)
self.handle_gateway_request(s, result)
self.inputs.remove(s)
s.close()
elif action == utils.GATEWAY_NODES_RESET:
result = []
for node in data[db_utils.NODES]:
gateway_id = self.dbConnector.find_gateway_by_addr(
s.getpeername())
node_uid = self.dbConnector.get_node_uid_by_gateway_id_and_node_id(
gateway_id, node['node_id'])
_node = {
'_id': node_uid,
'status': node['status']
}
result.append(_node)
self.handle_gateway_request(s, result)
self.inputs.remove(s)
s.close()
elif action == utils.IMAGES_GET: # { token }
token = data[db_utils.TOKEN]
decoded_token = utils.decode_auth_token(token)
if not decoded_token:
utils.send_invalid_token_error(s)
else:
nodetypes_images = utils.get_nodetypes_images(
decoded_token[db_utils.USER])
pck = utils.create_response_packet(
json.dumps({
'data': nodetypes_images,
'status': 200
}).encode())
# OnSuccess: { data, status: 200 }
s.sendall(pck)
elif action == utils.IMAGE_SAVE: # { token, image_name, image_data, nodetype_id }
token = data[db_utils.TOKEN]
decoded_token = utils.decode_auth_token(token)
if not decoded_token:
utils.send_invalid_token_error(s)
else:
utils.save_image(decoded_token[db_utils.USER],
data[db_utils.NODETYPE_ID],
data[db_utils.IMAGE_NAME],
data[db_utils.IMAGE_DATA])
pck = utils.create_response_packet(
json.dumps({
'status': 200
}).encode())
# OnSuccess: { status: 200 }
s.sendall(pck)
elif action == utils.IMAGE_DELETE: # { token, image_name }
token = data[db_utils.TOKEN]
decoded_token = utils.decode_auth_token(token)
if not decoded_token:
utils.send_invalid_token_error(s)
else:
nodetype_id = utils.get_nodetype_by_user_and_image_name(
decoded_token[db_utils.USER],
data[db_utils.IMAGE_NAME])
res = utils.delete_image(
decoded_token[db_utils.USER],
data[db_utils.IMAGE_NAME], nodetype_id)
pck = utils.create_response_packet(
json.dumps(res).encode())
# OnSuccess: { status: 204 }
# OnError : { status: 404 }
s.sendall(pck)
elif action == utils.NODES_GET: # { token }
token = data[db_utils.TOKEN]
decoded_token = utils.decode_auth_token(token)
if not decoded_token:
utils.send_invalid_token_error(s)
else:
nodes = self.dbConnector.get_nodes()
pck = utils.create_response_packet(
json.dumps({
'nodes': nodes,
'status': 200
}).encode())
# OnSuccess: { nodes, status: 200 }
s.sendall(pck)
elif action == utils.NODES_FLASH: # { token, slot_id, image_name, node_uids}
token = data[db_utils.TOKEN]
slot_id = data[db_utils.TIMESLOT_ID]
decoded_token = utils.decode_auth_token(token)
if not decoded_token:
utils.send_invalid_token_error(s)
elif not self.dbConnector.get_slot_by_id(slot_id):
utils.send_invalid_slot_error(s)
else:
gateway_socks = self.send_flash_request(
decoded_token[db_utils.USER],
data[db_utils.IMAGE_NAME],
data[db_utils.NODE_UIDS])
self.inputs.extend(gateway_socks)
self.gateway_sockets.append(gateway_socks)
self.gateway_sockets_info.append({
'web_socket':
s,
'data': [],
db_utils.IMAGE_NAME:
data[db_utils.IMAGE_NAME]
})
elif action == utils.NODES_ERASE: # { token, slot_id, node_uids }
token = data[db_utils.TOKEN]
slot_id = data[db_utils.TIMESLOT_ID]
decoded_token = utils.decode_auth_token(token)
if not decoded_token:
utils.send_invalid_token_error(s)
elif not self.dbConnector.get_slot_by_id(slot_id):
utils.send_invalid_slot_error(s)
else:
gateway_socks = self.send_erase_request(
data[db_utils.NODE_UIDS])
self.inputs.extend(gateway_socks)
self.gateway_sockets.append(gateway_socks)
self.gateway_sockets_info.append({
'web_socket':
s,
'data': [],
db_utils.IMAGE_NAME:
''
})
elif action == utils.NODES_RESET: # { token, slot_id, node_uids }
token = data[db_utils.TOKEN]
slot_id = data[db_utils.TIMESLOT_ID]
decoded_token = utils.decode_auth_token(token)
if not decoded_token:
utils.send_invalid_token_error(s)
elif not self.dbConnector.get_slot_by_id(slot_id):
utils.send_invalid_slot_error(s)
else:
gateway_socks = self.send_reset_request(
data[db_utils.NODE_UIDS])
self.inputs.extend(gateway_socks)
self.gateway_sockets.append(gateway_socks)
self.gateway_sockets_info.append({
'web_socket':
s,
'data': [],
db_utils.IMAGE_NAME:
''
})
elif action == utils.TIMESLOTS_SAVE: # { token, slots: [{start, end}] }
token = data[db_utils.TOKEN]
decoded_token = utils.decode_auth_token(token)
if not decoded_token:
utils.send_invalid_token_error(s)
else:
slots = db_utils.convert_isoformat_to_datetime(
data[db_utils.SLOTS])
slots_saved = self.dbConnector.save_timeslots(
decoded_token[db_utils.USER], slots)
slots = db_utils.convert_datetime_to_isoformat(
slots_saved)
pck = utils.create_response_packet(
json.dumps({
'slots': slots,
'status': 200
}).encode())
# OnSuccess: { slots: [{start, end}], status: 200 }
s.sendall(pck)
elif action == utils.TIMESLOTS_GET_DAYSLOTS: # { token, date }
token = data[db_utils.TOKEN]
decoded_token = utils.decode_auth_token(token)
if not decoded_token:
utils.send_invalid_token_error(s)
else:
slots_day = self.dbConnector.get_day_slots(
data[db_utils.DATE])
slots = db_utils.convert_datetime_to_isoformat(
slots_day)
pck = utils.create_response_packet(
json.dumps({
'slots': slots,
'status': 200
}).encode())
# OnSuccess: { slots: [{start, end, user_id}], status: 200 }
s.sendall(pck)
elif action == utils.TIMESLOTS_GET_USERSLOTS: # { token }
token = data[db_utils.TOKEN]
decoded_token = utils.decode_auth_token(token)
if not decoded_token:
utils.send_invalid_token_error(s)
else:
user_slots = self.dbConnector.get_user_slots(
decoded_token[db_utils.USER])
slots = db_utils.convert_datetime_to_isoformat(
user_slots)
pck = utils.create_response_packet(
json.dumps({
'slots': slots,
'status': 200
}).encode())
# OnSuccess: { slots: [{slot_id, start, end}], status: 200 }
s.sendall(pck)
elif action == utils.NODETYPES_GET: # { token }
token = data[db_utils.TOKEN]
decoded_token = utils.decode_auth_token(token)
if not decoded_token:
utils.send_invalid_token_error(s)
else:
nodetypes = self.dbConnector.get_nodetypes()
pck = utils.create_response_packet(
json.dumps({
'nodetypes': nodetypes,
'status': 200
}).encode())
# OnSuccess: { nodetypes, status: 201 }
s.sendall(pck)
elif action == utils.USERS_SIGNUP: # { email, username, password }
res = self.dbConnector.create_user(data)
pck = utils.create_response_packet(
json.dumps(res).encode())
# OnSuccess: { status: 201 }
# OnError : { message, status: 403 }
s.sendall(pck)
elif action == utils.USERS_LOGIN: # { email, username }
res = self.dbConnector.login_user(data)
pck = utils.create_response_packet(
json.dumps(res).encode())
# OnSuccess: { token, status: 200}
# OnError : { message, status: 401 }
s.sendall(pck)
elif action == utils.DEBUG_START: # { token, slot_id }
token = data[db_utils.TOKEN]
slot_id = data[db_utils.TIMESLOT_ID]
decoded_token = utils.decode_auth_token(token)
if not decoded_token:
utils.send_invalid_token_error(s)
elif not self.dbConnector.get_slot_by_id(slot_id):
utils.send_invalid_slot_error(s)
else:
if self.sock_debug:
utils.remove_from_list(
self.inputs, self.sock_debug)
self.sock_debug.close()
log_data = [
'=== DEBUG CHANNEL START ===\n===========================\n'
]
pck = utils.create_response_packet(
json.dumps({
'data': log_data
}).encode())
self.sock_debug = s
self.sock_debug.sendall(pck)
elif action == utils.DEBUG_END: # { token, slot_id }
token = data[db_utils.TOKEN]
slot_id = data[db_utils.TIMESLOT_ID]
decoded_token = utils.decode_auth_token(token)
if not decoded_token:
utils.send_invalid_token_error(s)
elif not self.dbConnector.get_slot_by_id(slot_id):
utils.send_invalid_slot_error(s)
else:
log_data = [
'=== DEBUG CHANNEL END ===\n=========================\n'
]
if self.sock_debug:
self.experiment_info = {}
pck = utils.create_response_packet(
json.dumps({
'data': log_data,
'message': 'STOP DEBUG'
}).encode())
self.sock_debug.sendall(pck)
utils.remove_from_list(
self.inputs, self.sock_debug)
self.sock_debug.close()
self.sock_debug = None
# { status: 204 }
pck = utils.create_response_packet(
json.dumps({
'status': 204
}).encode())
s.sendall(pck)
elif action == utils.DEBUG_CLEAR_LOG: # { token, slot_id }
token = data[db_utils.TOKEN]
slot_id = data[db_utils.TIMESLOT_ID]
decoded_token = utils.decode_auth_token(token)
if not decoded_token:
utils.send_invalid_token_error(s)
elif not self.dbConnector.get_slot_by_id(slot_id):
utils.send_invalid_slot_error(s)
else:
self.clear_debug_log(
decoded_token[db_utils.USER], slot_id)
pck = utils.create_response_packet(
json.dumps({
'status': 204
}).encode())
# OnSuccess: { status: 204 }
s.sendall(pck)
elif action == utils.DEBUG_GET_LOG: # { token, slot_id }
token = data[db_utils.TOKEN]
slot_id = data[db_utils.TIMESLOT_ID]
decoded_token = utils.decode_auth_token(token)
if not decoded_token:
utils.send_invalid_token_error(s)
elif not self.dbConnector.get_slot_by_id(slot_id):
utils.send_invalid_slot_error(s)
else:
self.send_debug_log(
s, decoded_token[db_utils.USER], slot_id)
elif action == utils.DEBUG_GATEWAY: # [ TIMESTAMP, NODE_ID, DATA ]
print(data[0], '|', data[1], '|', data[2])
if self.experiment_info:
self.write_debug_log(data)
if self.sock_debug:
pck = utils.create_response_packet(
json.dumps({
'data': data
}).encode())
self.sock_debug.sendall(pck)
else:
self.logger.info(
'TCP DISCON (%s, %d)' %
(s.getpeername()[0], s.getpeername()[1]))
self.inputs.remove(s)
s.close()
for s in exceptional:
self.inputs.remove(s)
s.close()
for sock in self.inputs:
self.logger.debug('Exiting. . . Closing [%s]' % sock)
sock.close()
self.log_timer.cancel()
self.sock_tcp.close()
sys.exit('Exiting. . .')
def train(args, model, device):
"""
train the model
:model: image enhancer model
:device: cuda or cpu
"""
extractor = get_feature_extractor(device)
true_labels = torch.ones(config.batch_size, dtype=torch.long).to(device)
false_labels = torch.zeros(config.batch_size, dtype=torch.long).to(device)
settime = str(int(time.time()))
logs = open('logs/' + settime + '.txt', "w+")
logs.close()
for idx in range(args.resume_iter, config.train_iters):
train_original, train_style = load_train_dataset(
config.data_path, config.batch_size,
config.height * config.width * config.channels)
x = torch.from_numpy(train_original).float()
y_real = torch.from_numpy(train_style).float()
x = x.view(-1, config.height, config.width,
config.channels).permute(0, 3, 1, 2).to(device)
y_real = y_real.view(-1, config.height, config.width,
config.channels).permute(0, 3, 1, 2).to(device)
# --------------------------------------------------------------------------------------------------------------
# Train generators
# --------------------------------------------------------------------------------------------------------------
y_fake = model.gen_g(x)
x_rec = model.gen_f(y_fake)
# content loss
feat_x = get_feature(extractor, x, config.feature_id, device)
feat_x_rec = get_feature(extractor, x_rec, config.feature_id, device)
loss_content = torch.pow(feat_x.detach() - feat_x_rec, 2).mean()
# color loss
# gaussian blur image for discriminator_c
fake_blur = gaussian_blur(y_fake, config.kernel_size, config.sigma,
config.channels, device)
logits_fake_blur = model.dis_c(fake_blur)
if args.model_type == 'DCGAN':
loss_c = model.criterion(logits_fake_blur, true_labels)
elif args.model_type == 'WGAN':
loss_c = model.criterion(logits_fake_blur)
# texture loss
# gray-scale image for discriminator_t
fake_gray = gray_scale(y_fake)
logits_fake_gray = model.dis_t(fake_gray)
if args.model_type == 'DCGAN':
loss_t = model.criterion(logits_fake_gray, true_labels)
elif args.model_type == 'WGAN':
loss_t = model.criterion(logits_fake_gray)
# total variation loss
height_tv = torch.pow(
y_fake[:, :, 1:, :] - y_fake[:, :, :config.height - 1, :],
2).mean()
width_tv = torch.pow(
y_fake[:, :, :, 1:] - y_fake[:, :, :, :config.width - 1],
2).mean()
loss_tv = height_tv + width_tv
# total generator loss
gen_loss = loss_content + config.lambda_c * loss_c + config.lambda_t * loss_t + config.lambda_tv * loss_tv
model.g_optimizer.zero_grad()
model.f_optimizer.zero_grad()
gen_loss.backward()
model.g_optimizer.step()
model.f_optimizer.step()
# --------------------------------------------------------------------------------------------------------------
# Train discriminators
# --------------------------------------------------------------------------------------------------------------
y_fake = model.gen_g(x)
# color loss
fake_blur = gaussian_blur(y_fake, config.kernel_size, config.sigma,
config.channels, device)
real_blur = gaussian_blur(y_real, config.kernel_size, config.sigma,
config.channels, device)
logits_fake_blur = model.dis_c(fake_blur.detach())
logits_real_blur = model.dis_c(real_blur.detach())
if args.model_type == 'DCGAN':
loss_dc = model.criterion(logits_real_blur,
true_labels) + model.criterion(
logits_fake_blur, false_labels)
elif args.model_type == 'WGAN':
loss_dc = model.criterion(logits_real_blur) - model.criterion(
logits_fake_blur)
# texture loss
fake_gray = gray_scale(y_fake)
real_gray = gray_scale(y_real)
logits_fake_gray = model.dis_t(fake_gray.detach())
logits_real_gray = model.dis_t(real_gray.detach())
if args.model_type == 'DCGAN':
loss_dt = model.criterion(logits_real_gray,
true_labels) + model.criterion(
logits_fake_gray, false_labels)
elif args.model_type == 'WGAN':
loss_dt = model.criterion(logits_real_gray) - model.criterion(
logits_fake_gray)
# total discriminator loss
dis_loss = config.lambda_c * loss_dc + config.lambda_t * loss_dt
model.c_optimizer.zero_grad()
model.t_optimizer.zero_grad()
dis_loss.backward()
model.c_optimizer.step()
model.t_optimizer.step()
# Add weight clamping for WGAN
if args.model_type == 'WGAN':
for param in model.dis_c.parameters():
param.data.clamp_(-config.clamp, config.clamp)
for param in model.dis_t.parameters():
param.data.clamp_(-config.clamp, config.clamp)
print('Iteration : {}/{}, Gen_loss : {:.4f}, Dis_loss : {:.4f}'.format(
idx + 1, config.train_iters, gen_loss.data, dis_loss.data))
print(
'Loss_content : {:.4f}, Loss_c : {:.4f}, Loss_t : {:.4f}, Loss_tv: {:.4f}'
.format(loss_content.data, loss_c.data, loss_t.data, loss_tv.data))
print('Loss_dc : {:.4f}, Loss_dt : {:.4f}'.format(
loss_dc.data, loss_dt.data))
if (idx + 1) % 50 == 0:
log1 = 'Iteration : {}/{}, Gen_loss : {:.4f}, Dis_loss : {:.4f}'.format(
idx + 1, config.train_iters, gen_loss.data, dis_loss.data)
log2 = 'Loss_content : {:.4f}, Loss_c : {:.4f}, Loss_t : {:.4f}, Loss_tv: {:.4f}'.format(
loss_content.data, loss_c.data, loss_t.data, loss_tv.data)
log3 = 'Loss_dc : {:.4f}, Loss_dt : {:.4f}'.format(
loss_dc.data, loss_dt.data)
logs = open('logs/' + settime + '.txt', "a")
logs.write(log1)
logs.write('\n')
logs.write(log2)
logs.write('\n')
logs.write(log3)
logs.write('\n')
logs.close()
if (idx + 1) % 1000 == 0:
sample_path = os.path.join(config.sample_path, args.model_type)
checkpoint_path = os.path.join(config.checkpoint_path,
args.model_type)
utils.save_image(
x, os.path.join(sample_path, '{}-x.jpg'.format(idx + 1)))
utils.save_image(
x_rec, os.path.join(sample_path,
'{}-x_rec.jpg'.format(idx + 1)))
utils.save_image(
y_fake,
os.path.join(sample_path, '{}-y_fake.jpg'.format(idx + 1)))
utils.save_image(
y_real,
os.path.join(sample_path, '{}-y_real.jpg'.format(idx + 1)))
utils.save_image(
fake_blur,
os.path.join(sample_path, '{}-fake_blur.jpg'.format(idx + 1)))
utils.save_image(
real_blur,
os.path.join(sample_path, '{}-real_blur.jpg'.format(idx + 1)))
utils.save_image(
fake_gray,
os.path.join(sample_path, '{}-fake_gray.jpg'.format(idx + 1)))
utils.save_image(
real_gray,
os.path.join(sample_path, '{}-real_gray.jpg'.format(idx + 1)))
torch.save(
model.gen_g.state_dict(),
os.path.join(checkpoint_path, '{}-Gen_g.ckpt'.format(idx + 1)))
torch.save(
model.gen_f.state_dict(),
os.path.join(checkpoint_path, '{}-Gen_f.ckpt'.format(idx + 1)))
torch.save(
model.dis_c.state_dict(),
os.path.join(checkpoint_path, '{}-Dis_c.ckpt'.format(idx + 1)))
torch.save(
model.dis_t.state_dict(),
os.path.join(checkpoint_path, '{}-Dis_t.ckpt'.format(idx + 1)))
print('Saved intermediate images and model checkpoints.')
# The optimizer
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(learning_rate=2.0, global_step=global_step, decay_steps=100, decay_rate=0.94, staircase=True)
train_step = tf.train.AdamOptimizer(learning_rate).minimize(L, global_step=global_step)
# A more simple optimizer
# train_step = tf.train.AdamOptimizer(learning_rate=2.0).minimize(L)
# Set up the summary writer (saving summaries is optional)
# (do `tensorboard --logdir=/tmp/na-logs` to view it)
tf.scalar_summary("L_content", L_content)
tf.scalar_summary("L_style", L_style)
gen_image_addmean = tf.Variable(tf.constant(np.array(content_image, dtype=np.float32)), trainable=False)
tf.image_summary("Generated image (TODO: add mean)", gen_image_addmean)
summary_op = tf.merge_all_summaries()
summary_writer = tf.train.SummaryWriter('/tmp/na-logs', graph_def=sess.graph_def)
print "Start calculation..."
# The optimizer has variables that require initialization as well
sess.run(tf.initialize_all_variables())
for i in range(num_iters):
if i % 10 == 0:
gen_image_val = sess.run(gen_image)
save_image(gen_image_val, i)
print "L_content, L_style:", sess.run(L_content), sess.run(L_style)
# Increment summary
sess.run(tf.assign(gen_image_addmean, add_mean(gen_image_val)))
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, i)
print "Iter:", i
sess.run(train_step)
def test_full(args, model, device):
"""
test the trained model with full images
:model: image enhancer model
:device: cuda or cpu
"""
test_path = '../data/full/original/'
generate_path = '../data/full/generate/'
test_image_num = len([
name for name in os.listdir(test_path)
if os.path.isfile(os.path.join(test_path, name))
])
score_psnr, score_ssim_skimage, score_ssim_minstar, score_msssim_minstar = 0.0, 0.0, 0.0, 0.0
ind = 0
for name in os.listdir(test_path):
if os.path.isfile(os.path.join(test_path, name)):
ind += 1
test_original, test_style, image_height, image_width = load_test_dataset(
name)
x = torch.from_numpy(test_original).float()
y_real = torch.from_numpy(test_style).float()
x = x.view(image_height, image_width,
config.channels).permute(2, 0, 1).to(device)
y_real = y_real.view(image_height, image_width,
config.channels).permute(2, 0, 1).to(device)
y_fake = model.gen_g(
x.view(-1, config.channels, image_height, image_width))
y_fake = y_fake.view(config.channels, image_height, image_width)
# Calculate PSNR & SSIM scores
score_psnr += psnr_full(y_fake, y_real)
y_fake_np = y_fake.detach().cpu().numpy().transpose(1, 2, 0)
y_real_np = y_real.cpu().numpy().transpose(1, 2, 0)
temp_ssim, _ = compare_ssim(y_fake_np,
y_real_np,
multichannel=True,
gaussian_weights=True,
full=True)
score_ssim_skimage += temp_ssim
temp_ssim, _ = ssim(y_fake,
y_real,
kernel_size=11,
kernel_sigma=1.5)
score_ssim_minstar += temp_ssim
score_msssim_minstar += multi_scale_ssim(y_fake,
y_real,
kernel_size=11,
kernel_sigma=1.5)
print(
'PSNR & SSIM scores of {} images are calculated.'.format(ind))
utils.save_image(
y_fake,
os.path.join(generate_path,
'{}-x.jpg'.format(name[:5] + args.model_type)))
score_psnr /= test_image_num
score_ssim_skimage /= test_image_num
score_ssim_minstar /= test_image_num
score_msssim_minstar /= test_image_num
print(
'PSNR : {:.4f}, SSIM_skimage : {:.4f}, SSIM_minstar : {:.4f}, SSIM_msssim: {:.4f}'
.format(score_psnr, score_ssim_skimage, score_ssim_minstar,
score_msssim_minstar))
def generate(self, inputs, path, idx=None):
path = '{}/{}_G.png'.format(path, idx)
x = self.sess.run(self.G, {self.z: inputs})
save_image(x, path)
print("[*] Samples saved: {}".format(path))
return x
return Variable(x) # End of Helper routines # Load synthetic dataset #syn_image1, syn_image2, syn_label = dataLoading.load_synthetic_ldan_data(synthetic_image_dataset_path) #real_image, sirfs_normal, sirfs_SH, sirfs_shading, real_image_val, sirfs_sh_val, sirfs_normal_val, sirfs_shading_val = dataLoading.load_real_images_celebA(real_image_dataset_path, validation = True) #real_image_mask = dataLoading.getMask(real_image_mask, global_batch_size) #real_image, sirfs_normal, sirfs_SH, sirfs_shading, tNormal, real_image_mask, tSH, real_image_val, sirfs_sh_val, sirfs_normal_val, sirfs_shading_val, true_normal_val, mask_val, true_lighting_val = dataLoading.load_SfSNet_data(sfs_net_path, validation = True, twoLevel = True) syn_image, syn_normal, syn_sh, syn_shading, syn_mask, syn_sirfs_shading, syn_sirfs_normal, syn_sirfs_sh, syn_image_val, syn_normal_val, syn_lighting_val, syn_shading_val, syn_mask_val, syn_sirfs_shading_val, syn_sirfs_normal_val, syn_sirfs_sh_val = dataLoading.load_SfSNet_data(synthetic_data_path, twoLevel = True) real_image, real_normal, real_sh, real_shading, real_mask, real_sirfs_shading, real_sirfs_normal, real_sirfs_sh, real_image_val, real_normal_val, real_lighting_val, real_shading_val, real_mask_val, real_sirfs_shading_val, real_sirfs_normal_val, real_sirfs_sh_val = dataLoading.load_SfSNet_data(real_data_path, validation = True, twoLevel = True) # Transforms being used # if SHOW_IMAGES: ''' syn_image_mask_test = next(iter(mask_val)) utils.save_image(torchvision.utils.make_grid(syn_image_mask_test*255, padding=1), output_path+'images/MASK_TEST.png') tmp = next(iter(syn_image1)) utils.save_image(torchvision.utils.make_grid(tmp, padding=1), output_path+'images/test_synthetic_img.png') tmp = var(next(iter(syn_image_val))) tmp = denorm(tmp) print(tmp.data.shape) tmp = applyMask(tmp, syn_image_mask_test) utils.save_image(torchvision.utils.make_grid(tmp.data, padding=1), output_path+'images/test_syn_image.png') tmp = var(next(iter(syn_normal_val))) tmp = denorm(tmp) tmp = applyMask(tmp, syn_image_mask_test)
def create_model_and_train(self):
inputs = self.input
targets = self.target
def create_discriminator(discrim_inputs, discrim_targets):
n_layers = 3
layers = []
# 2x [batch, height, width, in_channels] => [batch, height, width, in_channels * 2]
input = tf.concat([discrim_inputs, discrim_targets], axis=3)
with tf.variable_scope("layer_1"):
convolved = self.discrim_conv(input, a.ndf, stride=2)
rectified = self.lrelu(convolved, 0.2)
layers.append(rectified)
for i in range(n_layers):
with tf.variable_scope("layer_%d" % (len(layers) + 1)):
out_channels = a.ndf * min(2**(i + 1), 8)
stride = 1 if i == n_layers - 1 else 2 # last layer here has stride 1
convolved = self.discrim_conv(layers[-1],
out_channels,
stride=stride)
normalized = self.batchnorm(convolved)
rectified = self.lrelu(normalized, 0.2)
layers.append(rectified)
with tf.variable_scope("layer_%d" % (len(layers) + 1)):
convolved = self.discrim_conv(rectified,
out_channels=1,
stride=1)
output = tf.sigmoid(convolved)
layers.append(output)
return layers[-1]
with tf.variable_scope("generator"):
out_channels = int(targets.get_shape()[-1])
outputs = self.create_generator(inputs, out_channels)
with tf.name_scope("real_discriminator"):
with tf.variable_scope("discriminator"):
predict_real = create_discriminator(inputs, targets)
with tf.name_scope("fake_discriminator"):
with tf.variable_scope("discriminator", reuse=True):
predict_fake = create_discriminator(inputs, outputs)
with tf.name_scope("discriminator_loss"):
# minimizing -tf.log will try to get inputs to 1
# predict_real => 1
# predict_fake => 0
discrim_loss = tf.reduce_mean(-(tf.log(predict_real + EPS) +
tf.log(1 - predict_fake + EPS)))
with tf.name_scope("generator_loss"):
# predict_fake => 1
# abs(targets - outputs) => 0
gen_loss_GAN = tf.reduce_mean(-tf.log(predict_fake + EPS))
if a.is_L2:
gen_loss_L1 = tf.reduce_mean(tf.square(targets - outputs))
else:
gen_loss_L1 = tf.reduce_mean(tf.abs(targets - outputs))
if a.gan_weight == 0:
gen_loss = gen_loss_L1
else:
gen_loss = gen_loss_GAN * a.gan_weight + gen_loss_L1 * a.l1_weight
with tf.name_scope("discriminator_train"):
discrim_tvars = [
var for var in tf.trainable_variables()
if var.name.startswith("discriminator")
]
discrim_optim = tf.train.AdamOptimizer(a.lr, a.beta1)
optimizer_d = discrim_optim.minimize(discrim_loss,
var_list=discrim_tvars)
with tf.name_scope("generator_train"):
gen_tvars = [
var for var in tf.trainable_variables()
if var.name.startswith("generator")
]
gen_optim = tf.train.AdamOptimizer(a.lr, a.beta1)
optimizer_g = gen_optim.minimize(gen_loss, var_list=gen_tvars)
self.sess.run(tf.global_variables_initializer())
with tf.name_scope("parameter_count"):
parameter_count = tf.reduce_sum([
tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()
])
print("parameter_count =", self.sess.run(parameter_count))
ckpt = tf.train.latest_checkpoint(a.checkpoint_dir)
saver = tf.train.Saver()
if ckpt:
print('syccessfully loaded model from: ' + ckpt)
saver.restore(self.sess, ckpt)
image_input, image_gt = utils.read_image(a.image_path)
if a.is_test:
images_path = utils.get_all_image_path(a.test_image_path)
for image_path in images_path:
image = utils.imread(image_path, half_size=64, image_size=128)
image = image[np.newaxis, :, :, np.newaxis]
output_image = self.sess.run(outputs,
feed_dict={self.input: image})
utils.save_image_output(output_image, image_path, 'train')
for i in range(a.max_epochs):
print('Train epoch {}:-----'.format(i + 1))
start_time = time.time()
batch = 1
for batch_input, batch_ouput in utils.batch_iter(image_input,
image_gt,
a.batch_size,
shuffle=True):
if np.random.randint(2, size=1)[0] == 1: # random flip
batch_input = np.flip(batch_input, axis=1)
batch_ouput = np.flip(batch_ouput, axis=1)
if np.random.randint(2, size=1)[0] == 1: # random flip
batch_input = np.flip(batch_input, axis=1)
batch_ouput = np.flip(batch_ouput, axis=1)
train_dict = {
self.input: batch_input,
self.target: batch_ouput
}
if a.gan_weight == 0:
gen_loss_L1_value, _ = self.sess.run(
[gen_loss_L1, optimizer_g], feed_dict=train_dict)
if batch % 10 == 0:
print('Train batch {}:-----'.format(batch))
print('gen_loss_L1_value {0:.6}'.format(
gen_loss_L1_value))
else:
discrim_loss_value, gen_loss_GAN_value, gen_loss_L1_value, _, _ = self.sess.run(
[
discrim_loss, gen_loss_GAN, gen_loss_L1,
optimizer_d, optimizer_g
],
feed_dict=train_dict)
if batch % 4 == 0:
print('Train batch {}:-----'.format(batch))
print(
'discrim_loss {0:.6} gen_loss_GAN_value {1:.6} gen_loss_L1_value {2:.6}'
.format(discrim_loss_value, gen_loss_GAN_value,
gen_loss_L1_value))
batch += 1
end_time = time.time()
time_dif = end_time - start_time
print("Time usage: " +
str(timedelta(seconds=int(round(time_dif)))))
if i % 100 == 0:
output_image = self.sess.run(
outputs, feed_dict={self.input: batch_input})
utils.save_image(batch_input, output_image, i)
saver.save(self.sess,
os.path.join(a.checkpoint_dir, 'step_' + str(i)))
def predict(self, file_path):
# convert svg to raster image
img, num_paths, path_list = self.batch_manager.read_svg(file_path)
file_name = os.path.splitext(os.path.basename(file_path))[0]
input_img_path = os.path.join(self.model_dir, '%s_0_input.png' % file_name)
save_image((1-img[np.newaxis,:,:,np.newaxis])*255, input_img_path, padding=0)
# # debug
# print(num_paths)
# plt.imshow(img, cmap=plt.cm.gray)
# plt.show()
pm = Param()
# predict paths through pathnet
start_time = time.time()
paths, path_pixels = self.extract_path(img)
num_path_pixels = len(path_pixels[0])
pids = self.rng.randint(num_path_pixels, size=8)
path_img_path = os.path.join(self.model_dir, '%s_1_path.png' % file_name)
save_image((1 - paths[pids,:,:,:])*255, path_img_path, padding=0)
# # debug
# plt.imshow(paths[0,:,:,0], cmap=plt.cm.gray)
# plt.show()
duration = time.time() - start_time
print('%s: %s, predict paths (#pixels:%d) through pathnet (%.3f sec)' % (datetime.now(), file_name, num_path_pixels, duration))
pm.duration_pred = duration
pm.duration = duration
dup_dict = {}
dup_rev_dict = {}
dup_id = num_path_pixels # start id of duplicated pixels
if self.find_overlap:
# predict overlap using overlap net
start_time = time.time()
ov = self.overlap(img)
overlap_img_path = os.path.join(self.model_dir, '%s_2_overlap.png' % file_name)
ov_img = ov[np.newaxis,:,:,np.newaxis]
save_image((1-ov_img)*255, overlap_img_path, padding=0)
# # debug
# plt.imshow(ov, cmap=plt.cm.gray)
# plt.show()
for i in range(num_path_pixels):
if ov[path_pixels[0][i], path_pixels[1][i]]:
dup_dict[i] = dup_id
dup_rev_dict[dup_id] = i
dup_id += 1
# debug
# print(dup_dict)
# print(dup_rev_dict)
duration = time.time() - start_time
print('%s: %s, predict overlap (#:%d) through ovnet (%.3f sec)' % (datetime.now(), file_name, dup_id-num_path_pixels, duration))
pm.duration_ov = duration
pm.duration += duration
else:
pm.duration_ov = 0
# write config file for graphcut
start_time = time.time()
tmp_dir = os.path.join(self.model_dir, 'tmp')
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
pred_file_path = os.path.join(tmp_dir, file_name+'.pred')
f = open(pred_file_path, 'w')
# info
f.write(pred_file_path + '\n')
f.write(self.data_path + '\n')
f.write('%d\n' % self.max_label)
f.write('%d\n' % self.label_cost)
f.write('%f\n' % self.sigma_neighbor)
f.write('%f\n' % self.sigma_predict)
# f.write('%d\n' % num_path_pixels)
f.write('%d\n' % dup_id)
# support only symmetric edge weight
radius = self.sigma_neighbor*2
nb = sklearn.neighbors.NearestNeighbors(radius=radius)
nb.fit(np.array(path_pixels).transpose())
high_spatial = 100000
for i in range(num_path_pixels-1):
p1 = np.array([path_pixels[0][i], path_pixels[1][i]])
pred_p1 = np.reshape(paths[i,:,:,:], [self.height, self.width])
# see close neighbors and some far neighbors (stochastic sampling)
rng = nb.radius_neighbors([p1])
num_close = len(rng[1][0])
far = np.setdiff1d(range(i+1,num_path_pixels),rng[1][0])
num_far = len(far)
num_far = int(num_far * self.neighbor_sample)
if num_far > 0:
far_ids = self.rng.choice(far, size=num_far)
nb_ids = np.concatenate((rng[1][0],far_ids))
else:
nb_ids = rng[1][0]
for rj, j in enumerate(nb_ids): # ids
if j <= i:
continue
p2 = np.array([path_pixels[0][j], path_pixels[1][j]])
if rj < num_close: d12 = rng[0][0][rj]
else: d12 = np.linalg.norm(p1-p2, 2)
# for j in range(i+1, num_path_pixels): # see entire neighbors
# p2 = np.array([path_pixels[0][j], path_pixels[1][j]])
# d12 = np.linalg.norm(p1-p2, 2)
pred_p2 = np.reshape(paths[j,:,:,:], [self.height, self.width])
pred = (pred_p1[p2[0],p2[1]] + pred_p2[p1[0],p1[1]]) * 0.5
pred = np.exp(-0.5 * (1.0-pred)**2 / self.sigma_predict**2)
spatial = np.exp(-0.5 * d12**2 / self.sigma_neighbor**2)
f.write('%d %d %f %f\n' % (i, j, pred, spatial))
dup_i = dup_dict.get(i)
if dup_i is not None:
f.write('%d %d %f %f\n' % (j, dup_i, pred, spatial)) # as dup is always smaller than normal id
f.write('%d %d %f %f\n' % (i, dup_i, 0, high_spatial)) # shouldn't be labeled together
dup_j = dup_dict.get(j)
if dup_j is not None:
f.write('%d %d %f %f\n' % (i, dup_j, pred, spatial)) # as dup is always smaller than normal id
f.write('%d %d %f %f\n' % (j, dup_j, 0, high_spatial)) # shouldn't be labeled together
if dup_i is not None and dup_j is not None:
f.write('%d %d %f %f\n' % (dup_i, dup_j, pred, spatial)) # dup_i < dup_j
f.close()
duration = time.time() - start_time
print('%s: %s, prediction computed (%.3f sec)' % (datetime.now(), file_name, duration))
pm.duration_map = duration
pm.duration += duration
pm.num_paths = num_paths
pm.path_list = path_list
pm.path_pixels = path_pixels
pm.dup_dict = dup_dict
pm.dup_rev_dict = dup_rev_dict
pm.img = img
pm.file_path = file_path
pm.model_dir = self.model_dir
pm.height = self.height
pm.width = self.width
pm.max_label = self.max_label
pm.sigma_neighbor = self.sigma_neighbor
pm.sigma_predict = self.sigma_predict
return pm
def transfer(num_iterations=200):
# Reset the graph
tf.reset_default_graph()
# Start interactive session
sess = tf.InteractiveSession()
# Initialize a noisy image by adding random noise to the content_image
generated_image = generate_noise_image(content_image)
# load VGG19 model
model = load_vgg_model("imagenet-vgg-verydeep-19.mat")
# Assign the content image to be the input of the VGG model.
sess.run(model['input'].assign(content_image))
# Select the output tensor of layer conv4_2
out = model['conv4_2']
# Set a_C to be the hidden layer activation from the layer we have selected
a_C = sess.run(out)
# Set a_G to be the hidden layer activation from same layer. Here, a_G references model['conv4_2']
# and isn't evaluated yet. Later in the code, we'll assign the image G as the model input, so that
# when we run the session, this will be the activations drawn from the appropriate layer, with G as input.
a_G = out
# Compute the content cost
J_content = compute_content_cost(a_C, a_G)
# Assign the input of the model to be the "style" image
sess.run(model['input'].assign(style_image))
# Compute the style cost
J_style = compute_style_cost(sess, model)
J = total_cost(J_content, J_style, alpha, beta) # 10,40
# define optimizer (1 line)
optimizer = tf.train.AdamOptimizer(2.0)
# define train_step (1 line)
train_step = optimizer.minimize(J)
# Initialize global variables (you need to run the session on the initializer)
sess.run(tf.global_variables_initializer())
# Run the noisy input image (initial generated image) through the model. Use assign().
sess.run(model["input"].assign(generated_image))
for i in range(num_iterations):
# Run the session on the train_step to minimize the total cost
sess.run(train_step)
# Compute the generated image by running the session on the current model['input']
generated_image = sess.run(model["input"])
# Print every 20 iteration.
if i % 20 == 0:
Jt, Jc, Js = sess.run([J, J_content, J_style])
print("Iteration " + str(i) + " :")
print("total cost = " + str(Jt))
print("content cost = " + str(Jc))
print("style cost = " + str(Js))
# save current generated image in the "/output" directory
save_image("images/" + str(i) + ".png", generated_image)
# save last generated image
save_image("images/output.png", generated_image)
return generated_image
def train(self):
x_list, xs, ys, sample_list = self.batch_manager.random_list(self.b_num)
save_image(xs, '{}/x_gt.png'.format(self.model_dir))
save_image(ys, '{}/y_gt.png'.format(self.model_dir))
with open('{}/gt.txt'.format(self.model_dir), 'w') as f:
for sample in sample_list:
f.write(sample + '\n')
# call once
summary_once = self.sess.run(self.summary_once)
self.summary_writer.add_summary(summary_once, 0)
self.summary_writer.flush()
for step in trange(self.start_step, self.max_step):
fetch_dict = {
"optim": self.optim,
"loss": self.loss,
}
if step % self.log_step == 0 or step == self.max_step-1:
fetch_dict.update({
"summary": self.summary_op,
})
if step % self.test_step == self.test_step-1 or step == self.max_step-1:
l1, l2, iou, nb = 0, 0, 0, 0
for x, y in self.batch_manager.test_batch():
if self.data_format == 'NCHW':
x = to_nchw_numpy(x)
y = to_nchw_numpy(y)
tl1, tl2, y_ = self.sess.run([self.tl1, self.tl2, self.yt_], {self.xt: x, self.yt: y})
l1 += tl1
l2 += tl2
nb += 1
# iou
y_I = np.logical_and(y>0, y_>0)
y_I_sum = np.sum(y_I, axis=(1, 2, 3))
y_U = np.logical_or(y>0, y_>0)
y_U_sum = np.sum(y_U, axis=(1, 2, 3))
# print(y_I_sum, y_U_sum)
nonzero_id = np.where(y_U_sum != 0)[0]
if nonzero_id.shape[0] == 0:
acc = 1.0
else:
acc = np.average(y_I_sum[nonzero_id] / y_U_sum[nonzero_id])
iou += acc
if nb > 500:
break
l1 /= float(nb)
l2 /= float(nb)
iou /= float(nb)
summary_test = self.sess.run(self.summary_test,
{self.test_acc_l1: l1, self.test_acc_l2: l2, self.test_acc_iou: iou})
self.summary_writer.add_summary(summary_test, step)
self.summary_writer.flush()
result = self.sess.run(fetch_dict)
if step % self.log_step == 0 or step == self.max_step-1:
self.summary_writer.add_summary(result['summary'], step)
self.summary_writer.flush()
loss = result['loss']
assert not np.isnan(loss), 'Model diverged with loss = NaN'
print("\n[{}/{}] Loss: {:.6f}".format(step, self.max_step, loss))
if step % (self.log_step * 10) == 0 or step == self.max_step-1:
self.generate(x_list, self.model_dir, idx=step)
if step % self.lr_update_step == self.lr_update_step - 1:
self.sess.run(self.lr_update)
# save last checkpoint..
save_path = os.path.join(self.model_dir, 'model.ckpt')
self.saver.save(self.sess, save_path, global_step=self.step)
self.batch_manager.stop_thread()
print("current batch size:", len(filenames))
adversarial_samples = sess.run(gen_adversarial_sample,
feed_dict={inceptionV3.x_input:
images})[:len(filenames)]
predictions = sess.run([inceptionV3.predicts, sm],
feed_dict={inceptionV3.x_input: images})
adversarial_predictions = sess.run(
inceptionV3.predicts,
feed_dict={inceptionV3.x_input: adversarial_samples})[:len(filenames)]
imageIds += list(map(lambda x: x[:-4], filenames))
predictedIds += list(predictions[0][:len(filenames)])
adversarialPredictedIds += list(adversarial_predictions)
for adv, fn in zip(adversarial_samples, filenames):
save_image("output/Attack_" + fn, adv)
# generate saliency maps
# sms = sess.run(sm, feed_dict={inceptionV3.x_input: images})[:len(filenames)]
# for sm_image,fn in zip(sms,filenames):
# save_image("output/sm_"+fn,(sm_image-1.0)*2.)
summary_frame = pd.DataFrame({"ImageId":imageIds,"predictedLabel":predictedIds,"adversarialLabel":adversarialPredictedIds})\
.merge(image_lc,how='left',on='ImageId')
summary_frame = summary_frame.merge(Id2name,
how='left',
left_on='predictedLabel',
right_on='CategoryId',
suffixes=['_t', '_p']).drop('CategoryId',
if optimize_simply:
# A more simple optimizer
train_step = tf.train.AdamOptimizer(learning_rate=2.0).minimize(L)
else:
# Complicated optimizer
train_step = tf.train.AdamOptimizer(learning_rate).minimize(L, global_step=global_step)
# Set up the summary writer (saving summaries is optional)
# (do `tensorboard --logdir=/tmp/na-logs` to view it)
tf.scalar_summary("L_content", L_content)
tf.scalar_summary("L_style", L_style)
gen_image_addmean = tf.Variable(tf.constant(np.array(content_image, dtype=np.float32)), trainable=False)
tf.image_summary("Generated image (TODO: add mean)", gen_image_addmean)
summary_op = tf.merge_all_summaries()
summary_writer = tf.train.SummaryWriter('/tmp/na-logs', graph_def=sess.graph_def)
print "Start calculation..."
# The optimizer has variables that require initialization as well
sess.run(tf.initialize_all_variables())
for i in range(num_iters):
if i % 10 == 0:
gen_image_val = sess.run(gen_image)
save_image(gen_image_val, i, args.out_dir)
print "L_content, L_style:", sess.run(L_content), sess.run(L_style)
# Increment summary
sess.run(tf.assign(gen_image_addmean, add_mean(gen_image_val)))
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, i)
print "Iter:", i
sess.run(train_step)
def train(self, n_iters):
skip_step = 1
with tf.Session() as sess:
###############################
# TO DO:
# 1. initialize your variables
# 2. create writer to write your graph
###############################
sess.run(tf.global_variables_initializer())
writer = tf.summary.FileWriter('graphs', sess.graph)
sess.run(self.input_img.assign(self.initial_img))
###############################
# TO DO:
# 1. create a saver object
# 2. check if a checkpoint exists, restore the variables
##############################
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(
os.path.dirname('checkpoints/checkpoint'))
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
initial_step = self.gstep.eval()
start_time = time.time()
for index in range(initial_step, n_iters):
if 5 <= index < 20:
skip_step = 10
elif index >= 20:
skip_step = 20
sess.run(self.opt)
if (index + 1) % skip_step == 0:
###############################
# TO DO: obtain generated image, loss, and summary
###############################
gen_image, total_loss, summary = sess.run(
[self.input_img, self.total_loss, self.summary_op])
###############################
# add back the mean pixels we subtracted before
gen_image = gen_image + self.vgg.mean_pixels
writer.add_summary(summary, global_step=index)
print('Step {}\n Sum: {:5.1f}'.format(
index + 1, np.sum(gen_image)))
print(' Loss: {:5.1f}'.format(total_loss))
print(' Took: {} seconds'.format(time.time() -
start_time))
start_time = time.time()
filename = './outputs/%d.png' % index
utils.save_image(filename, gen_image)
if (index + 1) % 20 == 0:
###############################
# TO DO: save the variables into a checkpoint
###############################
saver.save(sess, './checkpoints/style_transfer', index)
(x % self.overrect_lava.width, y % self.overrect_lava.height)
)
over_rock = self.overimage_rocks.get_at(
(x % self.overrect_rocks.width, y % self.overrect_rocks.height)
)
# Processed colours should be between 1 and 250 (if it's 0,0,0 it'll be keyed)
new_colour.r = max(1, min(new_colour.r + over_lava.r * 2 - over_rock.r / 2, 250))
new_colour.g = max(1, min(new_colour.g - over_lava.r - over_rock.r / 2, 250))
new_colour.b = max(1, min(new_colour.b - over_lava.r - over_rock.r / 2, 250))
"""
this draws a ratio*ratio grid in the level, useful for some debugging
if x % self.ratio == 0 or y % self.ratio == 0:
new_colour = (0,0,255)
"""
self.set_at((x, y), new_colour)
utils.save_image("{0}_processed.png".format(stage_file), self)
level_md5 = utils.file_md5("{0}.gif".format(stage_file))
print "MD5 of stage: {0}".format(level_md5)
utils.set_option("{0}_hash".format(stage_file), level_md5)
def calculate_limits(self):
# TODO: cache limits, it takes a while to process them
for x in range(self.rect.width - 1):
# will store top and bottom limits and append it to self.limits
# to control the ship not getting over this
x_limits = [-1, -1]
for y in range(self.rect.height):
# Calculate top and bottom limits
if x_limits[0] == -1 and self.level_data.get_at((x, y)) != self.colors["grass"]:
x_limits[0] = y
if x_limits[0] != -1 and x_limits[1] == -1 and self.level_data.get_at((x, y)) == self.colors["grass"]:
def train(self):
print("训练")
x_fixed, x_target_fixed, pose_fixed, pose_target_fixed, mask_fixed, mask_target_fixed = \
self.get_image_from_loader()
# 以下是灰度图
# print("x_fixed:", x_fixed.shape)
# print("x_target_fixed:", x_target_fixed.shape)
# print("mask_fixed:", mask_fixed.shape)
# print("mask_target_fixed:", mask_target_fixed.shape)
# print("pose_fixed[0][40][40]:", np.array(pose_fixed[0][40][40]))
# print("pose_target_fixed:", pose_target_fixed.shape)
save_image(x_fixed, '{}/.png'.format(self.model_dir))
save_image(x_target_fixed, '{}/x_target_fixed.png'.format(self.model_dir))
save_image((np.amax(pose_fixed, axis=-1, keepdims=True)+1.0)*127.5, '{}/pose_fixed.png'.format(self.model_dir))
save_image((np.amax(pose_target_fixed, axis=-1, keepdims=True)+1.0)*127.5, '{}/pose_target_fixed.png'.format(self.model_dir))
save_image(mask_fixed, '{}/mask_fixed.png'.format(self.model_dir))
save_image(mask_target_fixed, '{}/mask_target_fixed.png'.format(self.model_dir))
for step in trange(self.start_step, self.max_step):
if step < 22000:
self.sess.run(self.g_optim1)
else:
# Train generator
if step > 0:
self.sess.run(self.g_optim2)
# Train critic
if (self.wgan_gp.MODE == 'dcgan') or (self.wgan_gp.MODE == 'lsgan'):
disc_ITERS = 1
else:
disc_ITERS = self.wgan_gp.CRITIC_ITERS
for i in xrange(disc_ITERS):
self.sess.run(self.d_optim)
if self.wgan_gp.MODE == 'wgan':
self.sess.run(self.clip_disc_weights)
fetch_dict = {}
if step % self.log_step == self.log_step-1:
fetch_dict.update({"summary": self.summary_op}) # "k_t": self.k_t,
result = self.sess.run(fetch_dict)
if step % self.log_step == self.log_step-1:
self.summary_writer.add_summary(result['summary'], step)
self.summary_writer.flush()
if step % (self.log_step * 3) == (self.log_step * 3)-1:
# if self.data_format == 'NCHW':
# x = x_fixed.transpose([0, 3, 1, 2])
# else:
# x = x_fixed
x = utils_wgan.process_image(x_fixed, 127.5, 127.5)
x_target = utils_wgan.process_image(x_target_fixed, 127.5, 127.5)
self.generate(x, x_target, pose_target_fixed, self.model_dir, idx=step)
if step % self.lr_update_step == self.lr_update_step - 1:
self.sess.run([self.g_lr_update, self.d_lr_update])
if step % (self.log_step * 30) == (self.log_step * 30)-1:
self.saver.save(self.sess, os.path.join(self.model_dir, 'model.ckpt'), global_step=step)
def write_grid(grid, global_step):
"""Native python function to call for writing images to files."""
if global_step % summary_freq == 0:
img_path = os.path.join(log_dir, '%s.jpg' % str(global_step))
utils.save_image(grid, img_path)
return 0
def write_grid(grid, global_step):
"""Native python function to call for writing images to files."""
if global_step % summary_freq == 0:
img_path = os.path.join(log_dir, '%s.jpg' % str(global_step))
utils.save_image(grid, img_path)
return 0
index_record = defaultdict(int)
for index, row in labels_df.iterrows():
image_name = row['Frame']
full_path_image_name = os.path.join(data_path, image_name)
if os.path.isfile(full_path_image_name):
image_name = image_name.split('.')[0]
image = utils.load_image(full_path_image_name)
xmin = row['xmin'] #if row['xmin'] < row['xmax'] else row['xmax']
xmax = row['xmax'] #if row['xmin'] < row['xmax'] else row['xmin']
ymin = row['ymin'] #if row['ymin'] < row['ymax'] else row['ymax']
ymax = row['ymax'] #if row['ymin'] < row['ymax'] else row['ymin']
cropped_image = image[xmax:ymax, xmin:ymin, :]
label = row['Label']
key = '{}_{}'.format(image_name, label)
obj_index = index_record[key]
obj_index += 1
save_image_full_path = '{path}{image_name}_{obj_index}_{label}.jpg'.format(
path=save_path,
image_name=image_name,
obj_index=obj_index,
label=label)
index_record[key] = obj_index
try:
resized_image = utils.resize_image(cropped_image, image_size)
utils.save_image(resized_image, save_image_full_path)
except Exception as e:
print 'failed image {}'.format(image_name)
import cv2
import numpy as np
import matplotlib.pyplot as plt
from utils import vis_hybrid_image, load_image, save_image, im_range
from student_code import my_imfilter, create_hybrid_image
image1 = load_image('../data/dog.bmp')
image2 = load_image('../data/cat.bmp')
cutoff_frequency = 7
filter = cv2.getGaussianKernel(ksize=cutoff_frequency * 4 + 1,
sigma=cutoff_frequency)
filter = np.dot(filter, filter.T)
low_frequencies, high_frequencies, hybrid_image = create_hybrid_image(
image1, image2, filter)
vis = vis_hybrid_image(hybrid_image)
save_image('../results/hybrid_image.jpg', hybrid_image)
save_image('../results/hybrid_image_scales.jpg', vis)
# input_real, input_cropped = input_real.cuda(),input_cropped.cuda()
# criterionMSE.cuda()
# real_center = real_center.cuda()
input_real = Variable(input_real)
input_cropped = Variable(input_cropped)
real_center = Variable(real_center)
input_real.data.resize_(image.size()).copy_(image)
input_cropped.data.resize_(image.size()).copy_(image)
real_center_cpu = image[:,:,opt.imageSize/4:opt.imageSize/4+opt.imageSize/2,opt.imageSize/4:opt.imageSize/4+opt.imageSize/2]
real_center.data.resize_(real_center_cpu.size()).copy_(real_center_cpu)
input_cropped.data[:,0,opt.imageSize/4+opt.overlapPred:opt.imageSize/4+opt.imageSize/2-opt.overlapPred,opt.imageSize/4+opt.overlapPred:opt.imageSize/4+opt.imageSize/2-opt.overlapPred] = 2*117.0/255.0 - 1.0
input_cropped.data[:,1,opt.imageSize/4+opt.overlapPred:opt.imageSize/4+opt.imageSize/2-opt.overlapPred,opt.imageSize/4+opt.overlapPred:opt.imageSize/4+opt.imageSize/2-opt.overlapPred] = 2*104.0/255.0 - 1.0
input_cropped.data[:,2,opt.imageSize/4+opt.overlapPred:opt.imageSize/4+opt.imageSize/2-opt.overlapPred,opt.imageSize/4+opt.overlapPred:opt.imageSize/4+opt.imageSize/2-opt.overlapPred] = 2*123.0/255.0 - 1.0
fake = netG(input_cropped)
errG = criterionMSE(fake,real_center)
recon_image = input_cropped.clone()
recon_image.data[:,:,opt.imageSize/4:opt.imageSize/4+opt.imageSize/2,opt.imageSize/4:opt.imageSize/4+opt.imageSize/2] = fake.data
utils.save_image('val_real_samples.png',image[0])
utils.save_image('val_cropped_samples.png',input_cropped.data[0])
utils.save_image('val_recon_samples.png',recon_image.data[0])
print('%.4f' % errG.data[0])
n = min(data.shape[0], 8)
comparison = np.concatenate([data[:n],
recon_x.npvalue().T.reshape(batch_size, 1, 28, 28)[:n]])
save_image(comparison,
'results/reconstruction_' + str(epoch) + '.png', nrow=n)
test_loss /= len(test_data)
print('====> Test set loss: {:.4f}'.format(test_loss))
import time
tictocs = []
for epoch in range(1, args.epochs + 1):
tic = time.time()
train(epoch)
test(epoch)
sample = dy.inputTensor(np.random.randn(20, 64))
sample = model.decode(sample)
save_image(sample.npvalue().T.reshape(64, 1, 28, 28),
'results/sample_' + str(epoch) + '.png')
toc = time.time()
tictocs.append(toc - tic)
print('############\n\n')
print('Total Time Cost:', np.sum(tictocs))
print('Epoch Time Cost', np.average(tictocs), '+-', np.std(tictocs) / np.sqrt(len(tictocs)))
print('\n\n############')
def main():
# parse arguments
args = parse_args()
if args is None:
exit()
# initiate VGG19 model
model_file_path = args.model_path + '/' + vgg19.MODEL_FILE_NAME
vgg_net = vgg19.VGG19(model_file_path)
# load content image and style image
content_image = utils.load_image(args.content, max_size=args.max_size)
style_image = utils.load_image(args.style,
shape=(content_image.shape[1],
content_image.shape[0]))
############## (w*h*3)으로 입력이 들어와야 가능
content_image = np.load('cont.npy')
content_image = np.transpose(content_image, (1, 2, 0))
# content_image = np.expand_dims(content_image,axis=0)
# content_image = np.zeros((256,256,3))
print(content_image.shape)
###########3##
# initial guess for output
if args.initial_type == 'content':
init_image = content_image
elif args.initial_type == 'style':
init_image = style_image
elif args.initial_type == 'random':
init_image = np.random.normal(size=content_image.shape,
scale=np.std(content_image))
# check input images for style-transfer
# utils.plot_images(content_image,style_image, init_image)
# create a map for content layers info
CONTENT_LAYERS = {}
for layer, weight in zip(args.content_layers, args.content_layer_weights):
CONTENT_LAYERS[layer] = weight
# create a map for style layers info
STYLE_LAYERS = {}
for layer, weight in zip(args.style_layers, args.style_layer_weights):
STYLE_LAYERS[layer] = weight
# open session
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
# build the graph
st = style_transfer.StyleTransfer(
session=sess,
content_layer_ids=CONTENT_LAYERS,
style_layer_ids=STYLE_LAYERS,
init_image=add_one_dim(init_image),
content_image=add_one_dim(content_image),
style_image=add_one_dim(style_image),
net=vgg_net,
num_iter=args.num_iter,
loss_ratio=args.loss_ratio,
content_loss_norm_type=args.content_loss_norm_type,
)
# launch the graph in a session
result_image = st.update()
# close session
sess.close()
# remove batch dimension
shape = result_image.shape
result_image = np.reshape(result_image, shape[1:])
# save result
utils.save_image(result_image, args.output)
metric.update([real_label, ], [output, ])
# train with fake image
fake_image = g_net(noise)
output = d_net(fake_image.detach()).reshape((-1, 1))
errD_fake = loss(output, fake_label)
errD = errD_real + errD_fake
errD.backward()
metric.update([fake_label, ], [output, ])
d_trainer.step(BATCH_SIZE)
# update G
with autograd.record():
fake_image = g_net(noise)
output = d_net(fake_image).reshape(-1, 1)
errG = loss(output, real_label)
errG.backward()
g_trainer.step(BATCH_SIZE)
# print log infomation every 100 batches
if i % 100 == 0:
name, acc = metric.get()
logging.info('discriminator loss = %f, generator loss = %f, \
binary training acc = %f at iter %d epoch %d',
nd.mean(errD).asscalar(), nd.mean(errG).asscalar(), acc, i, epoch)
if i == 0:
save_image(fake_image, epoch, IMAGE_SIZE, BATCH_SIZE, OUTPUT_DIR)
metric.reset()
def train(self, n_iters):
skip_step = 1
with tf.Session() as sess:
###############################
## TO DO:
## 1. initialize your variables
## 2. create writer to write your grapp
sess.run(tf.global_variables_initializer())
writer = tf.summary.FileWriter("graphs", sess.graph)
# writer.add_summary(self.summary_op)
###############################
sess.run(self.input_img.assign(self.initial_img))
###############################
## TO DO:
## 1. create a saver object
## 2. check if a checkpoint exists, restore the variables
saver = tf.train.Saver()
try:
saver.restore(sess, "graphs/style_transfer.ckpt")
except:
pass
##############################
initial_step = self.gstep.eval()
start_time = time.time()
for index in range(initial_step, n_iters):
if index >= 5 and index < 20:
skip_step = 10
elif index >= 20:
skip_step = 20
sess.run(self.opt)
if (index + 1) % skip_step == 0:
###############################
## TO DO: obtain generated image, loss, and summary
gen_image, total_loss, summary = sess.run(
[self.input_img, self.total_loss, self.summary_op])
###############################
# add back the mean pixels we subtracted before
gen_image = gen_image + self.vgg.mean_pixels
writer.add_summary(summary, global_step=index)
print('Step {}\n Sum: {:5.1f}'.format(
index + 1, np.sum(gen_image)))
print(' Loss: {:5.1f}'.format(total_loss))
print(' Took: {} seconds'.format(time.time() -
start_time))
start_time = time.time()
filename = 'outputs/%d.png' % (index)
utils.save_image(filename, gen_image)
if (index + 1) % 20 == 0:
###############################
## TO DO: save the variables into a checkpoint
###############################
saver.save(sess, "graph/style_transfer.ckpt")
def stylize(args):
device = torch.device("cuda" if args.cuda else "cpu")
content_image = utils.load_image(args.content_image,
scale=args.content_scale)
content_transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Lambda(lambda x: x.mul(255))])
content_image = content_transform(content_image)
content_image = content_image.unsqueeze(0).to(device)
if args.model.endswith(".onnx"):
output = stylize_onnx_caffe2(content_image, args)
else:
with torch.no_grad():
if args.distilled:
style_model = SmallTransformerNet()
else:
style_model = TransformerNet()
# Load model from checkpoint
state_dict = torch.load(args.model)
if 'state_dict' in state_dict:
state_dict = state_dict['state_dict']
# remove saved deprecated running_* keys in InstanceNorm from the checkpoint
for k in list(state_dict.keys()):
if re.search(r'in\d+\.running_(mean|var)$', k):
del state_dict[k]
style_model.load_state_dict(state_dict)
style_model.to(device)
if args.export_onnx:
assert args.export_onnx.endswith(
".onnx"), "Export model file should end with .onnx"
output = torch.onnx._export(style_model, content_image,
args.export_onnx).cpu()
else:
output = style_model(content_image).cpu()
utils.save_image(args.output_image, output[0])
# If required, print the loss function of the generated image
if args.print_loss:
with torch.no_grad():
vgg = Vgg16(requires_grad=False).to(device)
features_x = vgg(content_image)
features_y = vgg(output)
# Content loss
mse_loss = torch.nn.MSELoss()
content_loss = (args.content_weight *
mse_loss(features_y.relu2_2, features_x.relu2_2))
# Style loss
style_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Lambda(lambda x: x.mul(255))
])
style = utils.load_image(args.style_image, size=args.style_size)
style = style_transform(style)
style = style.repeat(1, 1, 1, 1).to(device)
feature_style = vgg(style)
gram_style = utils.gram_matrix(feature_style[0])
gram_y = utils.gram_matrix(features_y[0])
style_loss = mse_loss(gram_y, gram_style)
style_loss *= args.style_weight
# Print the losses
total_loss = content_loss + style_loss
print("Style: {:.4} ; Content: {:.4} ; Total: {:.4}".format(
style_loss, content_loss, total_loss))
def test(self):
print("测试")
test_result_dir = os.path.join(self.model_dir, 'test_result')
test_result_dir_x = os.path.join(test_result_dir, 'x')
test_result_dir_x_target = os.path.join(test_result_dir, 'x_target')
test_result_dir_G = os.path.join(test_result_dir, 'G')
test_result_dir_pose = os.path.join(test_result_dir, 'pose')
test_result_dir_pose_target = os.path.join(test_result_dir, 'pose_target')
test_result_dir_mask = os.path.join(test_result_dir, 'mask')
test_result_dir_mask_target = os.path.join(test_result_dir, 'mask_target')
if not os.path.exists(test_result_dir):
os.makedirs(test_result_dir)
if not os.path.exists(test_result_dir_x):
os.makedirs(test_result_dir_x)
if not os.path.exists(test_result_dir_x_target):
os.makedirs(test_result_dir_x_target)
if not os.path.exists(test_result_dir_G):
os.makedirs(test_result_dir_G)
if not os.path.exists(test_result_dir_pose):
os.makedirs(test_result_dir_pose)
if not os.path.exists(test_result_dir_pose_target):
os.makedirs(test_result_dir_pose_target)
if not os.path.exists(test_result_dir_mask):
os.makedirs(test_result_dir_mask)
if not os.path.exists(test_result_dir_mask_target):
os.makedirs(test_result_dir_mask_target)
for i in xrange(400):
x_fixed, x_target_fixed, pose_fixed, pose_target_fixed, mask_fixed, mask_target_fixed = self.get_image_from_loader()
x = utils_wgan.process_image(x_fixed, 127.5, 127.5)
x_target = utils_wgan.process_image(x_target_fixed, 127.5, 127.5)
if 0==i:
x_fake = self.generate(x, x_target, pose_target_fixed, test_result_dir, idx=self.start_step, save=True)
else:
x_fake = self.generate(x, x_target, pose_target_fixed, test_result_dir, idx=self.start_step, save=False)
p = (np.amax(pose_fixed, axis=-1, keepdims=False)+1.0)*127.5
pt = (np.amax(pose_target_fixed, axis=-1, keepdims=False)+1.0)*127.5
for j in xrange(self.batch_size):
idx = i*self.batch_size+j
im = Image.fromarray(x_fixed[j,:].astype(np.uint8))
im.save('%s/%05d.png'%(test_result_dir_x, idx))
im = Image.fromarray(x_target_fixed[j,:].astype(np.uint8))
im.save('%s/%05d.png'%(test_result_dir_x_target, idx))
im = Image.fromarray(x_fake[j,:].astype(np.uint8))
im.save('%s/%05d.png'%(test_result_dir_G, idx))
im = Image.fromarray(p[j,:].astype(np.uint8))
im.save('%s/%05d.png'%(test_result_dir_pose, idx))
im = Image.fromarray(pt[j,:].astype(np.uint8))
im.save('%s/%05d.png'%(test_result_dir_pose_target, idx))
im = Image.fromarray(mask_fixed[j,:].squeeze().astype(np.uint8))
im.save('%s/%05d.png'%(test_result_dir_mask, idx))
im = Image.fromarray(mask_target_fixed[j,:].squeeze().astype(np.uint8))
im.save('%s/%05d.png'%(test_result_dir_mask_target, idx))
if 0==i:
save_image(x_fixed, '{}/x_fixed.png'.format(test_result_dir))
save_image(x_target_fixed, '{}/x_target_fixed.png'.format(test_result_dir))
save_image(mask_fixed, '{}/mask_fixed.png'.format(test_result_dir))
save_image(mask_target_fixed, '{}/mask_target_fixed.png'.format(test_result_dir))
save_image((np.amax(pose_fixed, axis=-1, keepdims=True)+1.0)*127.5, '{}/pose_fixed.png'.format(test_result_dir))
save_image((np.amax(pose_target_fixed, axis=-1, keepdims=True)+1.0)*127.5, '{}/pose_target_fixed.png'.format(test_result_dir))
