def test(epoch): model.training = False test_loss = 0 test_loader = generate_batch_loader(test_data, batch_size=batch_size) for i, data in enumerate(test_loader): # Dymanic Construction of Graph dy.renew_cg() x = dy.inputTensor(data.reshape(-1, 784).T) recon_x, mu, logvar = model.forward(x) loss = loss_function(recon_x, x, mu, logvar) # Forward loss_value = loss.value() test_loss += loss_value if i == 0: n = min(data.shape[0], 8) comparison = np.concatenate([data[:n], recon_x.npvalue().T.reshape(batch_size, 1, 28, 28)[:n]]) save_image(comparison, 'results/reconstruction_' + str(epoch) + '.png', nrow=n) test_loss /= len(test_data) print('====> Test set loss: {:.4f}'.format(test_loss))
def generate(self, x_samples, root_path=None, idx=None): if self.data_format == 'NCHW': x_samples = to_nchw_numpy(x_samples) generated = self.sess.run(self.yt_img, {self.xt: x_samples}) y_path = os.path.join(root_path, 'y_{}.png'.format(idx)) save_image(generated, y_path, nrow=self.b_num) print("[*] Samples saved: {}".format(y_path))
def main(): # parse arguments args = parse_args() if args is None: exit() # load content image content_image = utils.load_image(args.content, max_size=args.max_size) # open session soft_config = tf.ConfigProto(allow_soft_placement=True) soft_config.gpu_options.allow_growth = True # to deal with large image sess = tf.Session(config=soft_config) # build the graph transformer = style_transfer_tester.StyleTransferTester(session=sess, model_path=args.style_model, content_image=content_image, ) # execute the graph start_time = time.time() output_image = transformer.test() end_time = time.time() # save result utils.save_image(output_image, args.output) # report execution time shape = content_image.shape #(batch, width, height, channel) print('Execution time for a %d x %d image : %f msec' % (shape[0], shape[1], 1000.*float(end_time - start_time)/60))
def stylize(args): device = torch.device("cuda" if args.cuda else "cpu") content_image = utils.load_image(args.content_image, scale=args.content_scale) content_transform = transforms.Compose([ transforms.ToTensor(), transforms.Lambda(lambda x: x.mul(255)) ]) content_image = content_transform(content_image) content_image = content_image.unsqueeze(0).to(device) if args.model.endswith(".onnx"): output = stylize_onnx_caffe2(content_image, args) else: with torch.no_grad(): style_model = TransformerNet() state_dict = torch.load(args.model) # remove saved deprecated running_* keys in InstanceNorm from the checkpoint for k in list(state_dict.keys()): if re.search(r'in\d+\.running_(mean|var)$', k): del state_dict[k] style_model.load_state_dict(state_dict) style_model.to(device) if args.export_onnx: assert args.export_onnx.endswith(".onnx"), "Export model file should end with .onnx" output = torch.onnx._export(style_model, content_image, args.export_onnx).cpu() else: output = style_model(content_image).cpu() utils.save_image(args.output_image, output[0])
def train(self, n_iters): skip_step = 1 with tf.Session() as sess: ############################### ## TO DO: ## 1. initialize your variables ## 2. create writer to write your graph sess.run(tf.global_variables_initializer()) writer = tf.summary.FileWriter('graphs/style_stranfer', sess.graph) ############################### sess.run(self.input_img.assign(self.initial_img)) ############################### ## TO DO: ## 1. create a saver object ## 2. check if a checkpoint exists, restore the variables saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(os.path.dirname('checkpoints/style_transfer/checkpoint')) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) ############################## initial_step = self.gstep.eval() start_time = time.time() for index in range(initial_step, n_iters): if index >= 5 and index < 20: skip_step = 10 elif index >= 20: skip_step = 20 sess.run(self.opt) if (index + 1) % skip_step == 0: ############################### ## TO DO: obtain generated image, loss, and summary gen_image, total_loss, summary = sess.run([self.input_img, self.total_loss, self.summary_op]) ############################### # add back the mean pixels we subtracted before gen_image = gen_image + self.vgg.mean_pixels writer.add_summary(summary, global_step=index) print('Step {}\n Sum: {:5.1f}'.format(index + 1, np.sum(gen_image))) print(' Loss: {:5.1f}'.format(total_loss)) print(' Took: {} seconds'.format(time.time() - start_time)) start_time = time.time() filename = 'outputs/%d.png' % (index) utils.save_image(filename, gen_image) if (index + 1) % 20 == 0: ############################### ## TO DO: save the variables into a checkpoint saver.save(sess, 'checkpoints/style_stranfer/style_transfer', index)
def _save_image(note, image): note.snap_path = _build_snap_path(image) note.image_path = _build_image_path(image) if not utils.save_image_snap(note.snap_path, image): note.snap_path = note.image_path note.photo_time = utils.get_photo_time(image) if note.photo_time is None: note.photo_time = note.create_time utils.save_image(note.image_path, image)
def find_edges(img, input, output, binary=False, textual=False, interactive=False, changed="", n8=False, search_width=16): # create necessary image copies changed_img = np.copy(img) img_copy = np.copy(img) img = np.apply_along_axis(kitti_labels.get_label, 2, img) prob_img = np.zeros((img.shape[0], img.shape[1], kitti_labels.NUMBER_OF_LABELS)) # iterate through image and look at top and right neighbours (8-neighbourhood) for i in xrange(img.shape[0]): for j in xrange(img.shape[1]): current_pixel = img[i,j] if current_pixel in kitti_labels.DONT_CARE_LABELS: # put uniform distribution if the label is unknown prob_img[i,j,:] = search_width continue # set the probability for the read label if np.sum(prob_img[i,j,:]) == 0: prob_img[i,j,current_pixel] = search_width # seach range in each direction max_counter = search_width / 2 # check right neighbour _search_right(i, j, img, current_pixel, max_counter, prob_img, changed_img, search_width) # check top _search_top(i, j, img, current_pixel, max_counter, prob_img, changed_img, search_width) if n8: # check top right _search_top_right(i, j, img, current_pixel, max_counter, prob_img, changed_img, search_width) # check top left _search_top_left(i, j, img, current_pixel, max_counter, prob_img, changed_img, search_width) if interactive: plt.imshow(changed_img) plt.figure() plt.imshow(img_copy) plt.show() if changed: print "Saving image with change pixels - {}".format(changed) utils.save_image(changed_img, changed) if binary: print "Saving potentials - binary format - {}".format(output + ".bin") utils.save_prob_map_bin(prob_img, output + ".bin", True) if textual: print "Saving potentials - textual format - {}".format(output + ".txt") utils.save_prob_map_txt(prob_img, output + ".txt", True)
def write_grid(input_images, gt_projs, pred_projs, global_step, input_voxels, output_voxels): """Native python function to call for writing images to files.""" grid = _build_image_grid( input_images, gt_projs, pred_projs, input_voxels=input_voxels, output_voxels=output_voxels) if global_step % summary_freq == 0: img_path = os.path.join(log_dir, '%s.jpg' % str(global_step)) utils.save_image(grid, img_path) return grid
def train(model, generated_image, initial_image): """ Train your model. Don't forget to create folders for checkpoints and outputs. """ skip_step = 1 with tf.Session() as sess: saver = tf.train.Saver() ############################### ## TO DO: ## 1. initialize your variables ## 2. create writer to write your graph saver = tf.train.Saver() sess.run(tf.global_variables_initializer()) writer = tf.summary.FileWriter(EXP + '/graphs', sess.graph) ############################### sess.run(generated_image.assign(initial_image)) ckpt = tf.train.get_checkpoint_state(os.path.dirname('checkpoints/checkpoint')) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) initial_step = model['global_step'].eval() start_time = time.time() for index in range(initial_step, ITERS): if index >= 5 and index < 20: skip_step = 10 elif index >= 20: skip_step = 20 sess.run(model['optimizer']) if (index + 1) % skip_step == 0: ############################### ## TO DO: obtain generated image and loss gen_image, total_loss, summary = sess.run([generated_image, model['total_loss'], model['summary_op']]) ############################### gen_image = gen_image + MEAN_PIXELS writer.add_summary(summary, global_step=index) print('Step {}\n Sum: {:5.1f}'.format(index + 1, np.sum(gen_image))) print(' Loss: {:5.1f}'.format(total_loss)) print(' Time: {}'.format(time.time() - start_time)) start_time = time.time() filename = 'outputs/%d.png' % (index) utils.save_image(filename, gen_image) if (index + 1) % 20 == 0: saver.save(sess, 'checkpoints/style_transfer', index)
def main(): args = utils.get_args() print("Prepare dataset...") mnist = input_data.read_data_sets("mnist/", one_hot = True) with tf.Graph().as_default(), tf.Session() as session: autoencoder = Autoencoder( 784, args.hid_shape, args.lat_shape, optimizer = tf.train.AdagradOptimizer(args.lr), batch_size = args.batch_size, dropout = args.dropout) session.run(tf.initialize_all_variables()) if args.save_model or args.load_model: saver = tf.train.Saver() if args.load_model: try: saver.restore(session, utils.SAVER_FILE) except ValueError: print("Cant find model file") sys.exit(1) if args.make_imgs: index = 0 print("Prepare images directory...") utils.prepare_image_folder() example = utils.get_example(args.digit, mnist.test) print("Start training...") for epoch in range(args.epoches): for i, batch in enumerate(utils.gen_data(args.batch_size, mnist.train.images)): autoencoder.fit_on_batch(session, batch) if (i+1) % args.log_after == 0: test_cost = autoencoder.evaluate(session, mnist.test.images) print("Test error = {0:.4f} on {1} batch in {2} epoch".format(test_cost, i+1, epoch+1)) if args.make_imgs: path = os.path.join(utils.IMG_FOLDER, "{0:03}.png".format(index)) autoencoded = autoencoder.encode_decode(session, example.reshape(1, 784)) utils.save_image(autoencoded.reshape((28, 28)), path) index += 1 if args.save_model: saver.save(session, utils.SAVER_FILE) print("Model saved")
def generate(self, x_fixed, x_target_fixed, pose_target_fixed, root_path=None, path=None, idx=None, save=True): G = self.sess.run(self.G, {self.x: x_fixed, self.pose_target: pose_target_fixed}) ssim_G_x_list = [] # x_0_255 = utils_wgan.unprocess_image(x_target_fixed, 127.5, 127.5) for i in xrange(G.shape[0]): # G_gray = rgb2gray((G[i,:]/127.5-1).clip(min=-1,max=1)) # x_target_gray = rgb2gray((x_target_fixed[i,:]).clip(min=-1,max=1)) G_gray = rgb2gray((G[i,:]).clip(min=0,max=255).astype(np.uint8)) x_target_gray = rgb2gray(((x_target_fixed[i,:]+1)*127.5).clip(min=0,max=255).astype(np.uint8)) ssim_G_x_list.append(ssim(G_gray, x_target_gray, data_range=x_target_gray.max() - x_target_gray.min(), multichannel=False)) ssim_G_x_mean = np.mean(ssim_G_x_list) if path is None and save: path = os.path.join(root_path, '{}_G_ssim{}.png'.format(idx,ssim_G_x_mean)) save_image(G, path) print("[*] Samples saved: {}".format(path)) return G
def train(self, epoches=300): skip_step = 1 with tf.Session() as sess: sess.run(tf.global_variables_initializer()) writer = tf.summary.FileWriter("graphs/style_transfer", sess.graph) sess.run(self.input_img.assign(self.initial_img)) saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(os.path.dirname("checkpoints/%s_%s_style_transfer/checkpoint" % (self.content_name, self.style_name))) if ckpt and ckpt.model_checkpoint_path: print("You have pre-trained model, if you do not want to use this, please delete the existing one.") saver.restore(sess, ckpt.model_checkpoint_path) initial_step = self.gstep.eval() for epoch in range(initial_step, epoches): # 前面几轮每隔10个epoch生成一张图片 if epoch >= 5 and epoch < 20: skip_step = 10 # 后面每隔20个epoch生成一张图片 elif epoch >= 20: skip_step = 20 sess.run(self.optimizer) if (epoch + 1) % skip_step == 0: gen_image, total_loss, summary = sess.run([self.input_img, self.total_loss, self.summary_op]) # 对生成的图片逆向mean-center,即在每个channel上加上mean gen_image = gen_image + self.vgg.mean_pixels writer.add_summary(summary, global_step=epoch) print("Step {}\n Sum: {:5.1f}".format(epoch + 1, np.sum(gen_image))) print(" Loss: {:5.1f}".format(total_loss)) filename = "outputs/%s_%s/epoch_%d.png" % (self.content_name, self.style_name, epoch) utils.save_image(filename, gen_image) # 存储模型 if (epoch + 1) % 20 == 0: saver.save(sess, "checkpoints/%s_%s_style_transfer/style_transfer" % (self.content_name, self.style_name), epoch)
def write_grid(input_images, gt_projs, pred_projs, pred_voxels, global_step): """Native python function to call for writing images to files.""" grid = _build_image_grid(input_images, gt_projs, pred_projs, pred_voxels) if global_step % summary_freq == 0: img_path = os.path.join(log_dir, '%s.jpg' % str(global_step)) utils.save_image(grid, img_path) with open( os.path.join(log_dir, 'pred_voxels_%s' % str(global_step)), 'w') as fout: np.save(fout, pred_voxels) with open( os.path.join(log_dir, 'input_images_%s' % str(global_step)), 'w') as fout: np.save(fout, input_images) return grid
def stylize(args): content_image = utils.load_image(args.content_image, scale=args.content_scale) content_transform = transforms.Compose([ transforms.ToTensor(), transforms.Lambda(lambda x: x.mul(255)) ]) content_image = content_transform(content_image) content_image = content_image.unsqueeze(0) if args.cuda: content_image = content_image.cuda() content_image = Variable(content_image, volatile=True) style_model = TransformerNet() style_model.load_state_dict(torch.load(args.model)) if args.cuda: style_model.cuda() output = style_model(content_image) if args.cuda: output = output.cpu() output_data = output.data[0] utils.save_image(args.output_image, output_data)
def motion_detection(camera, folder, until): """ Uses 3 frames to look for motion, can't remember where I found it but it gives better result than my first try with comparing 2 frames. """ utils.clear_directory(folder) # Need to get 2 images to start with previous_image = cv2.cvtColor(camera.read()[1], cv2.cv.CV_RGB2GRAY) current_image = cv2.cvtColor(camera.read()[1], cv2.cv.CV_RGB2GRAY) purple = (140, 25, 71) while True: now = datetime.datetime.now() _, image = camera.read() gray_image = cv2.cvtColor(image, cv2.cv.CV_RGB2GRAY) difference1 = cv2.absdiff(previous_image, gray_image) difference2 = cv2.absdiff(current_image, gray_image) result = cv2.bitwise_and(difference1, difference2) # Basic threshold, turn the bitwise_and into a black or white (haha) # result, white (255) being a motion _, result = cv2.threshold(result, 40, 255, cv2.THRESH_BINARY) # Let's show a square around the detected motion in the original pic low_point, high_point = utils.find_motion_boundaries(result.tolist()) if low_point is not None and high_point is not None: cv2.rectangle(image, low_point, high_point, purple, 3) print 'Motion detected ! Taking picture' utils.save_image(image, folder, now) previous_image = current_image current_image = gray_image if utils.time_over(until, now): break del(camera)
def start_camera(camera, folder, interval, until=None): """ Start taking pictures every interval. If until is specified, it will take pictures until that time is reached (24h format). Needs to be of the following format: HH:MM """ utils.clear_directory(folder) number = 0 while True: _, image = camera.read() now = datetime.datetime.now() number += 1 print 'Taking picture number %d at %s' % (number, now.isoformat()) utils.save_image(image, folder, now) if utils.time_over(until, now): break time.sleep(interval) del(camera)
def main(config): prepare_dirs_and_logger(config) batch_manager = BatchManager(config) preprocess_path('data/kanji/train/0f9a8.svg_pre', 64, 64, batch_manager.rng) preprocess_overlap('data/kanji/train/0f9a8.svg_pre', 64, 64, batch_manager.rng) # thread test sess_config = tf.ConfigProto() sess_config.gpu_options.allow_growth = True sess_config.allow_soft_placement = True sess_config.log_device_placement = False sess = tf.Session(config=sess_config) batch_manager.start_thread(sess) x, y = batch_manager.batch() if config.data_format == 'NCHW': x = nhwc_to_nchw(x) x_, y_ = sess.run([x, y]) batch_manager.stop_thread() if config.data_format == 'NCHW': x_ = x_.transpose([0, 2, 3, 1]) if config.archi == 'path': b_ch = np.zeros([config.batch_size,config.height,config.width,1]) x_ = np.concatenate((x_*255, b_ch), axis=-1) else: x_ = x_*255 y_ = y_*255 save_image(x_, '{}/x_fixed.png'.format(config.model_dir)) save_image(y_, '{}/y_fixed.png'.format(config.model_dir)) # random pick from parameter space x_samples, x_gt, y_gt, sample_list = batch_manager.random_list(8) save_image(x_gt, '{}/x_gt.png'.format(config.model_dir)) save_image(y_gt, '{}/y_gt.png'.format(config.model_dir)) with open('{}/sample_list.txt'.format(config.model_dir), 'w') as f: for sample in sample_list: f.write(sample+'\n') print('batch manager test done')
def get_image(): """Gets an image file via POST request, feeds the image to the FaceNet model then saves both the original image and its resulting embedding from the FaceNet model in their designated folders. 'uploads' folder: for image files 'embeddings' folder: for embedding numpy files. """ if request.method == 'POST': if 'file' not in request.files: return render_template(template_name_or_list="warning.html", status="No 'file' field in POST request!") file = request.files['file'] filename = file.filename if filename == "": return render_template(template_name_or_list="warning.html", status="No selected file!") if file and allowed_file(filename=filename, allowed_set=allowed_set): filename = secure_filename(filename=filename) # Read image file as numpy array of RGB dimension img = imread(name=file, mode='RGB') # Detect and crop a 160 x 160 image containing a human face in the image file img = get_face(img=img, pnet=pnet, rnet=rnet, onet=onet, image_size=image_size) # If a human face is detected if img is not None: embedding = forward_pass( img=img, session=facenet_persistent_session, images_placeholder=images_placeholder, embeddings=embeddings, phase_train_placeholder=phase_train_placeholder, image_size=image_size) # Save cropped face image to 'uploads/' folder save_image(img=img, filename=filename, uploads_path=uploads_path) # Remove file extension from image filename for numpy file storage being based on image filename filename = remove_file_extension(filename=filename) # Save embedding to 'embeddings/' folder save_embedding(embedding=embedding, filename=filename, embeddings_path=embeddings_path) return render_template( template_name_or_list="upload_result.html", status="Image uploaded and embedded successfully!") else: return render_template( template_name_or_list="upload_result.html", status= "Image upload was unsuccessful! No human face was detected!" ) else: return render_template(template_name_or_list="warning.html", status="POST HTTP method required!")
def produce_art(content_image_path, style_image_path, model_path, model_type, width, alpha, beta, num_iters): # The actual calculation print "Read images..." content_image = read_image(content_image_path, width) style_image = read_image(style_image_path, width) g = tf.Graph() with g.device(device), g.as_default(), tf.Session(graph=g, config=tf.ConfigProto(allow_soft_placement=True)) as sess: print "Load content values..." image = tf.constant(content_image) model = getModel(image, model_path, model_type) content_image_y_val = [sess.run(y_l) for y_l in model.y()] # sess.run(y_l) is a constant numpy array print "Load style values..." image = tf.constant(style_image) model = getModel(image, model_path, model_type) y = model.y() style_image_st_val = [] for l in range(len(y)): num_filters = content_image_y_val[l].shape[3] st_shape = [-1, num_filters] st_ = tf.reshape(y[l], st_shape) st = tf.matmul(tf.transpose(st_), st_) style_image_st_val.append(sess.run(st)) # sess.run(st) is a constant numpy array print "Construct graph..." # Start from white noise # gen_image = tf.Variable(tf.truncated_normal(content_image.shape, stddev=20), trainable=True, name='gen_image') # Start from the original image gen_image = tf.Variable(tf.constant(np.array(content_image, dtype=np.float32)), trainable=True, name='gen_image') model = getModel(gen_image, model_path, model_type) y = model.y() L_content = 0.0 L_style = 0.0 for l in range(len(y)): # Content loss L_content += model.alpha[l] * tf.nn.l2_loss(y[l] - content_image_y_val[l]) # Style loss num_filters = content_image_y_val[l].shape[3] st_shape = [-1, num_filters] st_ = tf.reshape(y[l], st_shape) st = tf.matmul(tf.transpose(st_), st_) N = np.prod(content_image_y_val[l].shape).astype(np.float32) L_style += model.beta[l] * tf.nn.l2_loss(st - style_image_st_val[l]) / N ** 2 / len(y) # The loss L = alpha * L_content + beta * L_style # The optimizer global_step = tf.Variable(0, trainable=False) learning_rate = tf.train.exponential_decay(learning_rate=2.0, global_step=global_step, decay_steps=100, decay_rate=0.94, staircase=True) train_step = tf.train.AdamOptimizer(learning_rate).minimize(L, global_step=global_step) # A more simple optimizer # train_step = tf.train.AdamOptimizer(learning_rate=2.0).minimize(L) # Set up the summary writer (saving summaries is optional) # (do `tensorboard --logdir=/tmp/na-logs` to view it) tf.scalar_summary("L_content", L_content) tf.scalar_summary("L_style", L_style) gen_image_addmean = tf.Variable(tf.constant(np.array(content_image, dtype=np.float32)), trainable=False) tf.image_summary("Generated image (TODO: add mean)", gen_image_addmean) summary_op = tf.merge_all_summaries() summary_writer = tf.train.SummaryWriter('/tmp/na-logs', graph_def=sess.graph_def) print "Start calculation..." # The optimizer has variables that require initialization as well sess.run(tf.initialize_all_variables()) for i in range(num_iters): if i % 10 == 0: gen_image_val = sess.run(gen_image) save_image(gen_image_val, i, out_dir) print "L_content, L_style:", sess.run(L_content), sess.run(L_style) # Increment summary sess.run(tf.assign(gen_image_addmean, add_mean(gen_image_val))) summary_str = sess.run(summary_op) summary_writer.add_summary(summary_str, i) print "Iter:", i sess.run(train_step)
def main(): # make dirs target_dirs = [args.logdir, args.test_result_dir] #., args.test_LR_result_dir] create_dirs(target_dirs) # set logger logflag = set_logger(args) log(logflag, 'Test script start', 'info') NLR_data = tf.placeholder(tf.float32, shape=[None, None, None, args.channel], name='NLR_input') CLR_data = tf.placeholder(tf.float32, shape=[None, None, None, args.channel], name='CLR_input') NHR_data = tf.placeholder(tf.float32, shape=[None, None, None, args.channel], name='NHR_input') CHR_data = tf.placeholder(tf.float32, shape=[None, None, None, args.channel], name='CHR_input') # build Generator and Discriminator network = Network(args, NLR_data=NLR_data, CLR_data=CLR_data, NHR_data=NHR_data, CHR_data=CHR_data, is_test=True) CLR_C1, NLR_C1, CLR_C2, CHR_C3, NLR_C3, CHR_C4, CLR_I1, CHR_I1, CHR_I2 = network.train_generator( ) # define optimizers global_iter = tf.Variable(0, trainable=False) gc.collect() config = tf.ConfigProto(gpu_options=tf.GPUOptions( allow_growth=True, visible_device_list=args.gpu_dev_num)) # Start Session with tf.Session(config=config) as sess: log(logflag, 'Start Session', 'info') sess.run(tf.global_variables_initializer()) sess.run(global_iter.initializer) saver = tf.train.Saver(max_to_keep=10) saver.restore(sess, tf.train.latest_checkpoint(args.checkpoint_dir)) validpathLR = np.sort( np.asarray(glob(os.path.join(args.data_dir, '*.png')))) validpathHR = np.sort( np.asarray(glob(os.path.join(args.data_dir, '*.png')))) import time avgtime = 0 for valid_i in range(100): validLR, validHR = generate_testset(validpathLR[valid_i], validpathHR[valid_i], args) name = validpathLR[valid_i].split('/')[-1] validLR = np.transpose(validLR[:, :, :, np.newaxis], (3, 0, 1, 2)) validHR = np.transpose(validHR[:, :, :, np.newaxis], (3, 0, 1, 2)) starttime = time.time() valid_out, valid_out_LR = sess.run( [CHR_C3, CLR_C1], feed_dict={ NLR_data: validLR, NHR_data: validHR, CLR_data: validLR, CHR_data: validHR }) validLR = np.rot90(validLR, 1, axes=(1, 2)) validHR = np.rot90(validHR, 1, axes=(1, 2)) valid_out90, valid_out_LR90 = sess.run( [CHR_C3, CLR_C1], feed_dict={ NLR_data: validLR, NHR_data: validHR, CLR_data: validLR, CHR_data: validHR }) valid_out += np.rot90(valid_out90, 3, axes=(1, 2)) valid_out_LR += np.rot90(valid_out_LR90, 3, axes=(1, 2)) validLR = np.rot90(validLR, 1, axes=(1, 2)) validHR = np.rot90(validHR, 1, axes=(1, 2)) valid_out90, valid_out_LR90 = sess.run( [CHR_C3, CLR_C1], feed_dict={ NLR_data: validLR, NHR_data: validHR, CLR_data: validLR, CHR_data: validHR }) valid_out += np.rot90(valid_out90, 2, axes=(1, 2)) valid_out_LR += np.rot90(valid_out_LR90, 2, axes=(1, 2)) validLR = np.rot90(validLR, 1, axes=(1, 2)) validHR = np.rot90(validHR, 1, axes=(1, 2)) valid_out90, valid_out_LR90 = sess.run( [CHR_C3, CLR_C1], feed_dict={ NLR_data: validLR, NHR_data: validHR, CLR_data: validLR, CHR_data: validHR }) valid_out += np.rot90(valid_out90, 1, axes=(1, 2)) valid_out_LR += np.rot90(valid_out_LR90, 1, axes=(1, 2)) validLR = np.rot90(validLR, 1, axes=(1, 2)) validHR = np.rot90(validHR, 1, axes=(1, 2)) validLR = validLR[:, ::-1, :, :] validHR = validHR[:, ::-1, :, :] valid_out90, valid_out_LR90 = sess.run( [CHR_C3, CLR_C1], feed_dict={ NLR_data: validLR, NHR_data: validHR, CLR_data: validLR, CHR_data: validHR }) valid_out += valid_out90[:, ::-1, :, :] valid_out_LR += valid_out_LR90[:, ::-1, :, :] validLR = np.rot90(validLR, 1, axes=(1, 2)) validHR = np.rot90(validHR, 1, axes=(1, 2)) valid_out90, valid_out_LR90 = sess.run( [CHR_C3, CLR_C1], feed_dict={ NLR_data: validLR, NHR_data: validHR, CLR_data: validLR, CHR_data: validHR }) valid_out90 = np.rot90(valid_out90, 3, axes=(1, 2)) valid_out_LR90 = np.rot90(valid_out_LR90, 3, axes=(1, 2)) valid_out += valid_out90[:, ::-1, :, :] valid_out_LR += valid_out_LR90[:, ::-1, :, :] validLR = np.rot90(validLR, 1, axes=(1, 2)) validHR = np.rot90(validHR, 1, axes=(1, 2)) valid_out90, valid_out_LR90 = sess.run( [CHR_C3, CLR_C1], feed_dict={ NLR_data: validLR, NHR_data: validHR, CLR_data: validLR, CHR_data: validHR }) valid_out90 = np.rot90(valid_out90, 2, axes=(1, 2)) valid_out_LR90 = np.rot90(valid_out_LR90, 2, axes=(1, 2)) valid_out += valid_out90[:, ::-1, :, :] valid_out_LR += valid_out_LR90[:, ::-1, :, :] validLR = np.rot90(validLR, 1, axes=(1, 2)) validHR = np.rot90(validHR, 1, axes=(1, 2)) valid_out90, valid_out_LR90 = sess.run( [CHR_C3, CLR_C1], feed_dict={ NLR_data: validLR, NHR_data: validHR, CLR_data: validLR, CHR_data: validHR }) valid_out90 = np.rot90(valid_out90, 1, axes=(1, 2)) valid_out_LR90 = np.rot90(valid_out_LR90, 1, axes=(1, 2)) valid_out += valid_out90[:, ::-1, :, :] valid_out_LR += valid_out_LR90[:, ::-1, :, :] valid_out /= 8. valid_out_LR /= 8. currtime = time.time() - starttime print("time : %fs" % (currtime)) avgtime += currtime / 100 save_image(args, valid_out, 'test', name, save_max_num=1) #save_image(args, valid_out_LR, 'test_LR', valid_i, save_max_num=5) print("avg. time : %f" % avgtime)
def test(self): x_fixed = self.get_image_from_loader() save_image(x_fixed, '{}/x_fixed_test.png'.format(self.model_dir)) self.autoencode(x_fixed, self.model_dir, idx="test", x_fake=None)
def test(model, data_loader, num_train_batches, epoch, writer): """ Evaluate model on validation set Args: model: The CapsuleNet model. data_loader: An interator over the dataset. It combines a dataset and a sampler. """ print('===> Evaluate mode') # Switch to evaluate mode model.eval() if args.cuda: # When we wrap a Module in DataParallel for multi-GPUs model = model.module loss = 0 margin_loss = 0 recon_loss = 0 correct = 0 num_batches = len(data_loader) global_step = epoch * num_train_batches + num_train_batches step['step'] = global_step for data, target in data_loader: batch_size = data.size(0) target_indices = target target_one_hot = utils.one_hot_encode(target_indices, length=args.num_classes) assert target_one_hot.size() == torch.Size([batch_size, 10]) data, target = Variable(data, volatile=True), Variable(target_one_hot) if args.cuda: data = data.cuda() target = target.cuda() # Output predictions output = model(data) # output from DigitCaps (out_digit_caps) # Sum up batch loss t_loss, m_loss, r_loss = model.loss(data, output, target, size_average=False) loss += t_loss.data[0] margin_loss += m_loss.data[0] recon_loss += r_loss.data[0] # Count number of correct predictions # v_magnitude shape: [128, 10, 1, 1] v_magnitude = torch.sqrt((output**2).sum(dim=2, keepdim=True)) # pred shape: [128, 1, 1, 1] pred = v_magnitude.data.max(1, keepdim=True)[1].cpu() correct += pred.eq(target_indices.view_as(pred)).sum() # Get the reconstructed images of the last batch if args.use_reconstruction_loss: reconstruction = model.decoder(output, target) # Input image size and number of channel. # By default, for MNIST, the image width and height is 28x28 and 1 channel for black/white. image_width = args.input_width image_height = args.input_height image_channel = args.num_conv_in_channel recon_img = reconstruction.view(-1, image_channel, image_width, image_height) assert recon_img.size() == torch.Size( [batch_size, image_channel, image_width, image_height]) # Save the image into file system utils.save_image( recon_img, 'results/recons_image_test_{}_{}.png'.format(epoch, global_step)) utils.save_image( data, 'results/original_image_test_{}_{}.png'.format(epoch, global_step)) # Add and visualize the image in TensorBoard recon_img = vutils.make_grid(recon_img.data, normalize=True, scale_each=True) original_img = vutils.make_grid(data.data, normalize=True, scale_each=True) writer.add_image('test/recons-image-{}-{}'.format(epoch, global_step), recon_img, global_step) writer.add_image( 'test/original-image-{}-{}'.format(epoch, global_step), original_img, global_step) # Log test losses loss /= num_batches margin_loss /= num_batches recon_loss /= num_batches # Log test accuracies num_test_data = len(data_loader.dataset) accuracy = correct / num_test_data accuracy_percentage = 100. * accuracy # TensorBoard logging # 1) Log the scalar values writer.add_scalar('test/total_loss', loss, global_step) writer.add_scalar('test/margin_loss', margin_loss, global_step) if args.use_reconstruction_loss: writer.add_scalar('test/reconstruction_loss', recon_loss, global_step) writer.add_scalar('test/accuracy', accuracy, global_step) # Print test losses and accuracy print('Test: [Loss: {:.6f},' \ '\tMargin loss: {:.6f},' \ '\tReconstruction loss: {:.6f}]'.format( loss, margin_loss, recon_loss if args.use_reconstruction_loss else 0)) print('Test Accuracy: {}/{} ({:.0f}%)\n'.format(correct, num_test_data, accuracy_percentage))
optimizer.zero_grad() model(target_image) style_score = 0 content_score = 0 for s1 in style_losses: style_score += s1.loss for c1 in content_losses: content_score += c1.loss style_score *= STYLE_WEIGHT content_score *= CONTENT_WEIGHT loss = style_score + content_score loss.backward() run[0] += 1 if run[0] % 10 == 0: print('Run: {}'.format(run)) print('Style Loss: {:4f} Content Loss: {:4f}'.format( style_score.item(), content_score.item())) print() return style_score + content_score optimizer.step(closure) target_image.data.clamp_(0, 1) utils.save_image(target_image, OUTPUT_PATH)
def main(): global args, best_result, output_directory, train_csv, test_csv # evaluation mode start_epoch = 0 if args.evaluate: assert os.path.isfile(args.evaluate), \ "=> no best model found at '{}'".format(args.evaluate) print("=> loading best model '{}'".format(args.evaluate)) checkpoint = torch.load(args.evaluate) output_directory = os.path.dirname(args.evaluate) args = checkpoint['args'] start_epoch = checkpoint['epoch'] + 1 best_result = checkpoint['best_result'] model = checkpoint['model'] print("=> loaded best model (epoch {})".format(checkpoint['epoch'])) _, val_loader = create_data_loaders(args) args.evaluate = True validate(val_loader, model, checkpoint['epoch'], write_to_file=False) return # optionally resume from a checkpoint elif args.resume: chkpt_path = args.resume assert os.path.isfile(chkpt_path), \ "=> no checkpoint found at '{}'".format(chkpt_path) print("=> loading checkpoint '{}'".format(chkpt_path)) checkpoint = torch.load(chkpt_path) args = checkpoint['args'] start_epoch = checkpoint['epoch'] + 1 best_result = checkpoint['best_result'] model = checkpoint['model'] optimizer = checkpoint['optimizer'] output_directory = os.path.dirname(os.path.abspath(chkpt_path)) print("=> loaded checkpoint (epoch {})".format(checkpoint['epoch'])) train_loader, val_loader = create_data_loaders(args) args.resume = True # create new model else: train_loader, val_loader = create_data_loaders(args) print("=> creating Model ({}-{}) ...".format(args.arch, args.decoder)) in_channels = len(args.modality) if args.arch == 'resnet50': model = ResNet(layers=50, decoder=args.decoder, output_size=train_loader.dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet18': model = ResNet(layers=18, decoder=args.decoder, output_size=train_loader.dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) print("=> model created.") optimizer = torch.optim.SGD(model.parameters(), args.lr, \ momentum=args.momentum, weight_decay=args.weight_decay) # model = torch.nn.DataParallel(model).cuda() # for multi-gpu training model = model.cuda() # define loss function (criterion) and optimizer if args.criterion == 'l2': criterion = criteria.MaskedMSELoss().cuda() elif args.criterion == 'l1': criterion = criteria.MaskedL1Loss().cuda() # create results folder, if not already exists output_directory = utils.get_output_directory(args) if not os.path.exists(output_directory): os.makedirs(output_directory) train_csv = os.path.join(output_directory, 'train.csv') test_csv = os.path.join(output_directory, 'test.csv') best_txt = os.path.join(output_directory, 'best.txt') # create new csv files with only header if not args.resume: with open(train_csv, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() with open(test_csv, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() for epoch in range(start_epoch, args.epochs): utils.adjust_learning_rate(optimizer, epoch, args.lr) train(train_loader, model, criterion, optimizer, epoch) # train for one epoch result, img_merge = validate(val_loader, model, epoch) # evaluate on validation set # remember best rmse and save checkpoint is_best = result.rmse < best_result.rmse if is_best: best_result = result with open(best_txt, 'w') as txtfile: txtfile.write( "epoch={}\nmse={:.3f}\nrmse={:.3f}\nabsrel={:.3f}\nlg10={:.3f}\nmae={:.3f}\ndelta1={:.3f}\nt_gpu={:.4f}\n" .format(epoch, result.mse, result.rmse, result.absrel, result.lg10, result.mae, result.delta1, result.gpu_time)) if img_merge is not None: img_filename = output_directory + '/comparison_best.png' utils.save_image(img_merge, img_filename) utils.save_checkpoint( { 'args': args, 'epoch': epoch, 'arch': args.arch, 'model': model, 'best_result': best_result, 'optimizer': optimizer, }, is_best, epoch, output_directory)
def validate(val_loader, model, epoch, write_to_file=True): average_meter = AverageMeter() model.eval() # switch to evaluate mode end = time.time() for i, (input, target) in enumerate(val_loader): if configuration_file.GPU == True: input, target = input.cuda(), target.cuda() # torch.cuda.synchronize() data_time = time.time() - end # compute output end = time.time() with torch.no_grad(): pred = model(input) # torch.cuda.synchronize() gpu_time = time.time() - end # measure accuracy and record loss result = Result() result.evaluate(pred.data, target.data) average_meter.update(result, gpu_time, data_time, input.size(0)) end = time.time() # save 8 images for visualization skip = 50 if args.modality == 'rgb': rgb = input if i == 0: img_merge = utils.merge_into_row(rgb, target, pred) elif (i < 8 * skip) and (i % skip == 0): row = utils.merge_into_row(rgb, target, pred) img_merge = utils.add_row(img_merge, row) elif i == 8 * skip: filename = output_directory + '/comparison_' + str(epoch) + '.png' utils.save_image(img_merge, filename) if (i + 1) % args.print_freq == 0: print('Test: [{0}/{1}]\t' 't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t' 'RMSE={result.rmse:.2f}({average.rmse:.2f}) ' 'MAE={result.mae:.2f}({average.mae:.2f}) ' 'Delta1={result.delta1:.3f}({average.delta1:.3f}) ' 'REL={result.absrel:.3f}({average.absrel:.3f}) ' 'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format( i + 1, len(val_loader), gpu_time=gpu_time, result=result, average=average_meter.average())) avg = average_meter.average() print('\n*\n' 'RMSE={average.rmse:.3f}\n' 'MAE={average.mae:.3f}\n' 'Delta1={average.delta1:.3f}\n' 'REL={average.absrel:.3f}\n' 'Lg10={average.lg10:.3f}\n' 't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time)) if write_to_file: with open(test_csv, 'a') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writerow({ 'mse': avg.mse, 'rmse': avg.rmse, 'absrel': avg.absrel, 'lg10': avg.lg10, 'mae': avg.mae, 'delta1': avg.delta1, 'delta2': avg.delta2, 'delta3': avg.delta3, 'data_time': avg.data_time, 'gpu_time': avg.gpu_time }) return avg, img_merge
from MIDIUtil.Melody import Melody from MIDIUtil.Note import Note from MIDIUtil.MIDIReader import MIDIReader from utils import save_image import numpy as np reader = MIDIReader() melody = reader.read_file( '/home/miguel/src/GeneticComposition/data/training_test/mary_had_a_little_lamb.mid' )[0][0] image = np.zeros((len(melody.notes), 129), dtype=np.int32) for i, note in enumerate(melody.notes): image[i, note.getMIDIIndex()] = 1 image = image.T save_image( image, '/home/miguel/src/GeneticComposition/data/training_test/mary_had_a_little_lamb.png' )
def run_one_order(padded_input, order, adj, loc, all_patches, patch_size, model, context, final_outdir, style_name, disp_name, binary_thresh): styledResults = {} full_destyled = (patch_size - context) * (patch_size - context) nump = len(all_patches) rf_score = np.zeros((nump, 1)) h, w, c = padded_input.shape res = copy.deepcopy(padded_input) mask = np.ones([h, w]) transform = get_transform(opt) #, grayscale=True) all_can = [] all_can.append(copy.deepcopy(res)) adj += adj.transpose() visited = np.zeros((adj.shape[0]), dtype=int) margin = [] for idx, o in enumerate(order): if idx == 0: first = o visited[o] = 1 tmp = adj[o, :] nidx = [x for x in range(len(tmp)) if tmp[x] > 0] noverlap = tmp[nidx] vnidx = visited[nidx] vnidx2 = [ix for ix, x in enumerate(vnidx) if x == 1] o_dep = np.asarray([nidx[index] for index in vnidx2]) o_dep_overlap = np.asarray([noverlap[index] for index in vnidx2]) curPatch = all_patches[o] cloc = loc[o] c = cloc hybrid = res[c[0]:c[0] + patch_size, c[1]:c[1] + patch_size, :] cmask = mask[c[0]:c[0] + patch_size, c[1]:c[1] + patch_size] msk = 255 - np.sum(hybrid, axis=2) / 3 msk = msk.astype(int) unstyled = msk[cmask == 1] nz = np.count_nonzero(unstyled) if nz == 0: margin = margin + [o] mask[c[0]:c[0] + patch_size, c[1]:c[1] + patch_size] = -1 continue cres = translate_patch(hybrid, cmask, o, c, model) x1, x2, y1, y2 = 0, 0, 0, 0 buf = round(context / 2) buf2 = round(buf / 2) for dp in o_dep: if dp == o - 1: y1 = buf olreg1 = res[c[0]:c[0] + patch_size, c[1]:c[1] + context, :] olreg2 = cres[:, 0:context, :] blended = blend_overlap(olreg1, olreg2) cres[:, 0:context, :] = blended if dp == o + 1: y2 = -buf olreg2 = res[c[0]:c[0] + patch_size, c[1] + patch_size - context:c[1] + patch_size, :] olreg1 = cres[:, patch_size - context:patch_size, :] blended = blend_overlap(olreg1, olreg2) cres[:, patch_size - context:patch_size, :] = blended if dp < o - 1: x1 = buf olreg1 = res[c[0]:c[0] + context, c[1]:c[1] + patch_size, :] olreg2 = cres[0:context, :, :] blended = blend_overlap(olreg1, olreg2) cres[0:context, :, :] = blended if dp > o + 1: x2 = -buf olreg2 = res[c[0] + patch_size - context:c[0] + patch_size, c[1]:c[1] + patch_size, :] olreg1 = cres[patch_size - context:patch_size, :, :] blended = blend_overlap(olreg1, olreg2) cres[patch_size - context:patch_size, :, :] = blended res[c[0]:c[0] + patch_size, c[1]:c[1] + patch_size] = cres mask[c[0]:c[0] + patch_size, c[1]:c[1] + patch_size] = -1 all_can.append(copy.deepcopy(res)) name_str = '%s_%s_%d' % (style_name, disp_name, first) file_name = os.path.join(final_outdir, name_str + '.png') samename = glob.glob(file_name) currnum = str(len(samename) + 1) sname = os.path.join(final_outdir, name_str + '_' + currnum + '.png') print(sname) utils.save_image(res, sname)
def train(self): print('\n{}\n'.format(self.config)) # Save some fixed images once. z_fixed = np.random.normal(0, 1, size=(self.batch_size, self.z_dim)) x_fixed = self.get_images_from_loader() save_image(x_fixed, '{}/x_fixed.png'.format(self.model_dir)) # Use tensorflow tutorial set for conveniently labeled mnist. # NOTE: Data originally on [0,1], but immediately converted to [0, 255]. self.mnist = input_data.read_data_sets('MNIST_data', one_hot=True) self.images_train = self.prep_data(split='train', n=8000) # Train generator. for step in trange(self.start_step, self.max_step): # For MMDGAN training, use data with predicted weights. batch_train = self.get_n_images(self.batch_size, self.images_train) # Let training focus on ae_loss in case it is high. ae_loss_out = self.sess.run(self.ae_loss, feed_dict={ self.x: batch_train, self.lambda_mmd: self.lambda_mmd_setting, self.lambda_ae: self.lambda_ae_setting}) # Set up basket of items to be run. Occasionally fetch items # useful for logging and saving. if ae_loss_out > 0.5: fetch_dict = { 'd_optim': self.d_optim, } else: fetch_dict = { 'd_optim': self.d_optim, 'g_optim': self.g_optim, } if step % self.log_step == 0: fetch_dict.update({ 'summary': self.summary_op, 'ae_loss_real': self.ae_loss_real, 'ae_loss_fake': self.ae_loss_fake, 'mmd': self.mmd, 'ncmd_k': self.ncmd_k, 'x_enc': self.x_enc, 'g_enc': self.g_enc, }) # Run full training step on pre-fetched data and simulations. result = self.sess.run(fetch_dict, feed_dict={ self.x: batch_train, self.lambda_mmd: self.lambda_mmd_setting, self.lambda_ae: self.lambda_ae_setting}) if step % self.lr_update_step == self.lr_update_step - 1: self.sess.run([self.g_lr_update, self.d_lr_update]) # Log and save as needed. if step % self.log_step == 0: self.summary_writer.add_summary(result['summary'], step) self.summary_writer.flush() ae_loss_real = result['ae_loss_real'] ae_loss_fake = result['ae_loss_fake'] mmd = result['mmd'] ncmd_k = result['ncmd_k'] print(('[{}/{}] LOSSES: ae_real/fake: {:.3f}, {:.3f} ' 'mmd: {:.3f}, ncmd_k: {:.3f}').format( step, self.max_step, ae_loss_real, ae_loss_fake, mmd, ncmd_k)) print(np.round(np.percentile(result['x_enc'], [0, 20, 50, 80, 100]), 2)) print(np.round(np.percentile(result['g_enc'], [0, 20, 50, 80, 100]), 2)) if step % (self.save_step) == 0: # First save a sample. if step == 0: x_samp = self.get_n_images(1, self.images_train) save_image(x_samp, '{}/x_samp.png'.format(self.model_dir)) # Save images for fixed and random z. gen_fixed = self.generate( z_fixed, root_path=self.model_dir, step='fix'+str(step), save=True) z = np.random.normal(0, 1, size=(self.batch_size, self.z_dim)) gen_rand = self.generate( z, root_path=self.model_dir, step='rand'+str(step), save=True) # Save image of interpolation of z. self.interpolate_z(step)
def validate(val_loader, model, epoch, write_to_file=True): average_meter = AverageMeter() # switch to evaluate mode model.eval() end = time.time() for i, (input, target) in enumerate(val_loader): input, target = input.to(device), target.to(device) torch.cuda.synchronize() data_time = time.time() - end # compute output end = time.time() depth_pred = model(input) torch.cuda.synchronize() gpu_time = time.time() - end # measure accuracy and record loss result = Result() output1 = torch.index_select(depth_pred, 1, torch.LongTensor([0]).to(device)) result.evaluate(output1, target) average_meter.update(result, gpu_time, data_time, input.size(0)) end = time.time() # save 8 images for visualization skip = 50 if args.modality == 'd': img_merge = None else: if args.modality == 'rgb': rgb = input elif args.modality == 'rgbd': rgb = input[:, :3, :, :] depth = input[:, 3:, :, :] if i == 0: if args.modality == 'rgbd': img_merge = utils.merge_into_row_with_gt( rgb, depth, target, depth_pred) else: img_merge = utils.merge_into_row(rgb, target, depth_pred) elif (i < 8 * skip) and (i % skip == 0): if args.modality == 'rgbd': row = utils.merge_into_row_with_gt(rgb, depth, target, depth_pred) else: row = utils.merge_into_row(rgb, target, depth_pred) img_merge = utils.add_row(img_merge, row) elif i == 8 * skip: filename = output_directory + '/comparison_' + str( epoch) + '.png' utils.save_image(img_merge, filename) if (i + 1) % args.print_freq == 0: print('Test: [{0}/{1}]\t' 't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\t' 'RMSE={result.rmse:.2f}({average.rmse:.2f}) ' 'MAE={result.mae:.2f}({average.mae:.2f}) ' 'Delta1={result.delta1:.3f}({average.delta1:.3f}) ' 'REL={result.absrel:.3f}({average.absrel:.3f}) ' 'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format( i + 1, len(val_loader), gpu_time=gpu_time, result=result, average=average_meter.average())) avg = average_meter.average() print('\n*\n' 'RMSE={average.rmse:.3f}\n' 'MAE={average.mae:.3f}\n' 'Delta1={average.delta1:.3f}\n' 'REL={average.absrel:.3f}\n' 'Lg10={average.lg10:.3f}\n' 't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time)) if write_to_file: with open(test_csv, 'a') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writerow({ 'mse': avg.mse.item(), 'rmse': avg.rmse.item(), 'absrel': avg.absrel.item(), 'lg10': avg.lg10.item(), 'mae': avg.mae.item(), 'delta1': avg.delta1.item(), 'delta2': avg.delta2.item(), 'delta3': avg.delta3.item(), 'gpu_time': avg.gpu_time, 'data_time': avg.data_time }) return avg, img_merge
def main(): global args, best_result, output_directory, train_csv, test_csv, device device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") args = parser.parse_args() if args.modality == 'rgb' and args.num_samples != 0: print("number of samples is forced to be 0 when input modality is rgb") args.num_samples = 0 if args.modality == 'rgb' and args.max_depth != 0.0: print("max depth is forced to be 0.0 when input modality is rgb/rgbd") args.max_depth = 0.0 sparsifier = None max_depth = args.max_depth if args.max_depth >= 0.0 else np.inf if args.sparsifier == UniformSampling.name: sparsifier = UniformSampling(num_samples=args.num_samples, max_depth=max_depth) elif args.sparsifier == SimulatedStereo.name: sparsifier = SimulatedStereo(num_samples=args.num_samples, max_depth=max_depth) # create results folder, if not already exists output_directory = os.path.join( 'results', '{}.sparsifier={}.modality={}.arch={}.decoder={}.criterion={}.lr={}.bs={}' .format(args.data, sparsifier, args.modality, args.arch, args.decoder, args.criterion, args.lr, args.batch_size)) if not os.path.exists(output_directory): os.makedirs(output_directory) train_csv = os.path.join(output_directory, 'train.csv') test_csv = os.path.join(output_directory, 'test.csv') best_txt = os.path.join(output_directory, 'best.txt') # define loss function (criterion) and optimizer if args.criterion == 'l2': criterion = criteria.MaskedMSELoss().cuda() elif args.criterion == 'l1': criterion = criteria.MaskedL1Loss().cuda() out_channels = 1 # Data loading code print("=> creating data loaders ...") traindir = os.path.join('data', args.data, 'train') valdir = os.path.join('data', args.data, 'val') train_dataset = NYUDataset(traindir, type='train', modality=args.modality, sparsifier=sparsifier) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True, sampler=None) # set batch size to be 1 for validation val_dataset = NYUDataset(valdir, type='val', modality=args.modality, sparsifier=sparsifier) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=1, shuffle=False, num_workers=args.workers, pin_memory=True) print("=> data loaders created.") # evaluation mode if args.evaluate: best_model_filename = os.path.join(output_directory, 'model_best.pth.tar') if os.path.isfile(best_model_filename): print("=> loading best model '{}'".format(best_model_filename)) checkpoint = torch.load(best_model_filename) args.start_epoch = checkpoint['epoch'] best_result = checkpoint['best_result'] model = checkpoint['model'] print("=> loaded best model (epoch {})".format( checkpoint['epoch'])) else: print("=> no best model found at '{}'".format(best_model_filename)) validate(val_loader, model, checkpoint['epoch'], write_to_file=False) return # optionally resume from a checkpoint elif args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] + 1 best_result = checkpoint['best_result'] model = checkpoint['model'] optimizer = checkpoint['optimizer'] print("=> loaded checkpoint (epoch {})".format( checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(args.resume)) return # create new model else: # define model print("=> creating Model ({}-{}) ...".format(args.arch, args.decoder)) in_channels = len(args.modality) if args.arch == 'resnet50': model = ResNet(layers=50, decoder=args.decoder, in_channels=in_channels, out_channels=out_channels, pretrained=args.pretrained) elif args.arch == 'resnet18': model = ResNet(layers=18, decoder=args.decoder, in_channels=in_channels, out_channels=out_channels, pretrained=args.pretrained) print("=> model created.") optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # create new csv files with only header with open(train_csv, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() with open(test_csv, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() # model = torch.nn.DataParallel(model).cuda() model = model.to(device) print(model) print("=> model transferred to GPU.") for epoch in range(args.start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) # train for one epoch train(train_loader, model, criterion, optimizer, epoch) # evaluate on validation set result, img_merge = validate(val_loader, model, epoch) # remember best rmse and save checkpoint is_best = result.rmse < best_result.rmse if is_best: best_result = result with open(best_txt, 'w') as txtfile: txtfile.write( "epoch={}\nmse={:.3f}\nrmse={:.3f}\nabsrel={:.3f}\nlg10={:.3f}\nmae={:.3f}\ndelta1={:.3f}\nt_gpu={:.4f}\n" .format(epoch, result.mse, result.rmse, result.absrel, result.lg10, result.mae, result.delta1, result.gpu_time)) if img_merge is not None: img_filename = output_directory + '/comparison_best.png' utils.save_image(img_merge, img_filename) save_checkpoint( { 'epoch': epoch, 'arch': args.arch, 'model': model, 'best_result': best_result, 'optimizer': optimizer, }, is_best, epoch)
def train(args): # GPU enabling if (args.gpu != None): use_cuda = True dtype = torch.cuda.FloatTensor torch.cuda.set_device(args.gpu) print("Current device: %d" %torch.cuda.current_device()) # visualization of training controlled by flag visualize = (args.visualize != None) if (visualize): img_transform_512 = transforms.Compose([ transforms.Scale(512), # scale shortest side to image_size transforms.CenterCrop(512), # crop center image_size out transforms.ToTensor(), # turn image from [0-255] to [0-1] utils.normalize_tensor_transform() # normalize with ImageNet values ]) testImage_amber = utils.load_image("content_imgs/amber.jpg") testImage_amber = img_transform_512(testImage_amber) testImage_amber = Variable(testImage_amber.repeat(1, 1, 1, 1), requires_grad=False).type(dtype) testImage_dan = utils.load_image("content_imgs/dan.jpg") testImage_dan = img_transform_512(testImage_dan) testImage_dan = Variable(testImage_dan.repeat(1, 1, 1, 1), requires_grad=False).type(dtype) testImage_maine = utils.load_image("content_imgs/maine.jpg") testImage_maine = img_transform_512(testImage_maine) testImage_maine = Variable(testImage_maine.repeat(1, 1, 1, 1), requires_grad=False).type(dtype) # define network image_transformer = ImageTransformNet().type(dtype) optimizer = Adam(image_transformer.parameters(), LEARNING_RATE) loss_mse = torch.nn.MSELoss() # load vgg network vgg = Vgg16().type(dtype) # get training dataset dataset_transform = transforms.Compose([ transforms.Scale(IMAGE_SIZE), # scale shortest side to image_size transforms.CenterCrop(IMAGE_SIZE), # crop center image_size out transforms.ToTensor(), # turn image from [0-255] to [0-1] utils.normalize_tensor_transform() # normalize with ImageNet values ]) train_dataset = datasets.ImageFolder(args.dataset, dataset_transform) train_loader = DataLoader(train_dataset, batch_size = BATCH_SIZE) # style image style_transform = transforms.Compose([ transforms.ToTensor(), # turn image from [0-255] to [0-1] utils.normalize_tensor_transform() # normalize with ImageNet values ]) style = utils.load_image(args.style_image) style = style_transform(style) style = Variable(style.repeat(BATCH_SIZE, 1, 1, 1)).type(dtype) style_name = os.path.split(args.style_image)[-1].split('.')[0] # calculate gram matrices for style feature layer maps we care about style_features = vgg(style) style_gram = [utils.gram(fmap) for fmap in style_features] for e in range(EPOCHS): # track values for... img_count = 0 aggregate_style_loss = 0.0 aggregate_content_loss = 0.0 aggregate_tv_loss = 0.0 # train network image_transformer.train() for batch_num, (x, label) in enumerate(train_loader): img_batch_read = len(x) img_count += img_batch_read # zero out gradients optimizer.zero_grad() # input batch to transformer network x = Variable(x).type(dtype) y_hat = image_transformer(x) # get vgg features y_c_features = vgg(x) y_hat_features = vgg(y_hat) # calculate style loss y_hat_gram = [utils.gram(fmap) for fmap in y_hat_features] style_loss = 0.0 for j in range(4): style_loss += loss_mse(y_hat_gram[j], style_gram[j][:img_batch_read]) style_loss = STYLE_WEIGHT*style_loss aggregate_style_loss += style_loss.data.item() # calculate content loss (h_relu_2_2) recon = y_c_features[1] recon_hat = y_hat_features[1] content_loss = CONTENT_WEIGHT*loss_mse(recon_hat, recon) aggregate_content_loss += content_loss.data.item() # calculate total variation regularization (anisotropic version) # https://www.wikiwand.com/en/Total_variation_denoising diff_i = torch.sum(torch.abs(y_hat[:, :, :, 1:] - y_hat[:, :, :, :-1])) diff_j = torch.sum(torch.abs(y_hat[:, :, 1:, :] - y_hat[:, :, :-1, :])) tv_loss = TV_WEIGHT*(diff_i + diff_j) aggregate_tv_loss += tv_loss.data.item() # total loss total_loss = style_loss + content_loss + tv_loss # backprop total_loss.backward() optimizer.step() # print out status message if ((batch_num + 1) % 100 == 0): status = "{} Epoch {}: [{}/{}] Batch:[{}] agg_style: {:.6f} agg_content: {:.6f} agg_tv: {:.6f} style: {:.6f} content: {:.6f} tv: {:.6f} ".format( time.ctime(), e + 1, img_count, len(train_dataset), batch_num+1, aggregate_style_loss/(batch_num+1.0), aggregate_content_loss/(batch_num+1.0), aggregate_tv_loss/(batch_num+1.0), style_loss.item(), content_loss.item(), tv_loss.item() ) print(status) if ((batch_num + 1) % 1000 == 0) and (visualize): image_transformer.eval() if not os.path.exists("visualization"): os.makedirs("visualization") if not os.path.exists("visualization/%s" %style_name): os.makedirs("visualization/%s" %style_name) outputTestImage_amber = image_transformer(testImage_amber).cpu() amber_path = "visualization/%s/amber_%d_%05d.jpg" %(style_name, e+1, batch_num+1) utils.save_image(amber_path, outputTestImage_amber.data[0]) outputTestImage_dan = image_transformer(testImage_dan).cpu() dan_path = "visualization/%s/dan_%d_%05d.jpg" %(style_name, e+1, batch_num+1) utils.save_image(dan_path, outputTestImage_dan.data[0]) outputTestImage_maine = image_transformer(testImage_maine).cpu() maine_path = "visualization/%s/maine_%d_%05d.jpg" %(style_name, e+1, batch_num+1) utils.save_image(maine_path, outputTestImage_maine.data[0]) print("images saved") image_transformer.train() # save model image_transformer.eval() if use_cuda: image_transformer.cpu() if not os.path.exists("models"): os.makedirs("models") filename = "models/" + str(style_name) + "_" + str(time.ctime()).replace(' ', '_') + ".model" torch.save(image_transformer.state_dict(), filename) if use_cuda: image_transformer.cuda()
def build_graph(content_feat_map, style_grams, content_image, max_steps, output_dir, output_image_name, option_weights, scope): print "Make graph for new image..." # initial = tf.random_normal(content_image.shape) * 0.256 initial = tf.truncated_normal(content_image.shape, stddev=20) image = tf.Variable(initial, trainable=True) image_vars_set = set(tf.all_variables()) net = VGG19({'data': image}, scope=scope, trainable=False) with tf.Session() as sess: # Load the data net.load(neural_config.model_path, sess) # Forward pass feature_maps = [net.layers[tensor] for tensor in neural_config.new_image_layers] img_content_feat_map = feature_maps[0] utils.activation_summary(img_content_feat_map, "gen_image_content_map") img_style_grams = [] for index, layer in enumerate(neural_config.style_layers): feat_map = feature_maps[index + 1] layer_shape = feat_map.get_shape().dims size = layer_shape[1].value * layer_shape[1].value * layer_shape[3].value features = tf.reshape(feat_map, (-1, layer_shape[3].value)) gram = tf.matmul(tf.transpose(features), features) / size img_style_grams.append(gram) utils.activation_summary(gram, "gen_image_style_gram%d" % index) # content loss content_loss = option_weights[0] * (2 * tf.nn.l2_loss( img_content_feat_map - content_feat_map) / content_feat_map.size) tf.scalar_summary("content_loss", content_loss) # style loss style_loss = 0 style_losses = [] for index, style_layer in enumerate(neural_config.style_layers): style_losses.append(2 * tf.nn.l2_loss( img_style_grams[index] - style_grams[index]) / style_grams[index].size) style_loss += option_weights[1] * reduce(tf.add, style_losses) tf.scalar_summary("style_loss", style_loss) # total variation denoising tv_y_size = utils.getTensorSize(image[:,1:,:,:]) tv_x_size = utils.getTensorSize(image[:,:,1:,:]) tv_loss = option_weights[2] * 2 * ( (tf.nn.l2_loss(image[:,1:,:,:] - image[:,:content_image.shape[1]-1,:,:]) / tv_y_size) + (tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:content_image.shape[2]-1,:]) / tv_x_size)) tf.scalar_summary("tv_loss", tv_loss) # overall loss loss = content_loss + style_loss + tv_loss tf.scalar_summary("total_loss", loss) temp = set(tf.all_variables()) # optimizer setup # opt = tf.train.AdamOptimizer(neural_config.learning_rate).minimize(loss) global_step = tf.Variable(0, trainable=False) learning_rate = tf.train.exponential_decay(learning_rate=neural_config.learning_rate, global_step=global_step, decay_steps=neural_config.decay_steps, decay_rate=neural_config.decay_rate, staircase=True) opt = tf.train.AdamOptimizer(learning_rate).minimize(loss, global_step=global_step) vars = set(tf.all_variables()) - temp vars = vars.union(image_vars_set) # vars.add(image) init = tf.initialize_variables(vars) sess.run(init) # optimization best_loss = float('inf') for step in xrange(max_steps): _, content_loss_value, style_loss_value, loss_value = sess.run([opt, content_loss, style_loss, loss]) format_str = ('step %d, content_loss_value = %.2f, style_loss_value = %.2f, loss = %.2f') print (format_str % (step, content_loss_value, style_loss_value, loss_value)) last_step = step == neural_config.max_iter - 1 if (neural_config.checkpoint_steps and step % neural_config.checkpoint_steps == 0) or last_step: if loss_value < best_loss: best_loss = loss_value best = sess.run(image) # best_image = utils.save_image(best, step, output_dir) utils.add_mean(best) tf.image_summary(("images/step%d" % step), best, name="gen_image") summary_op = tf.merge_all_summaries() summary_writer = tf.train.SummaryWriter(neural_config.train_dir) summary_str = sess.run(summary_op) summary_writer.add_summary(summary_str, step) # if last_step: utils.save_image(best, step, output_dir, output_image_name)
def main(): global args, best_result, output_directory, train_csv, test_csv print(args) start = 0 # evaluation mode if args.evaluate: datasets = configuration_file.datasets_path valdir = os.path.join(datasets, args.data, 'val') print("Validation directory ", valdir) if args.data == 'nyudepthv2': from dataloaders.nyu import NYUDataset val_dataset = NYUDataset(valdir, split='val', modality=args.modality) else: raise RuntimeError('Dataset not found.') #set batch size to be 1 for validation val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=1, shuffle=False, num_workers=args.workers, pin_memory=True) print("=> validation loaders created.") assert os.path.isfile(args.evaluate), \ "=> no model found at '{}'".format(args.evaluate) print("=> loading model '{}'".format(args.evaluate)) checkpoint = torch.load(args.evaluate) if type(checkpoint) is dict: args.start_epoch = checkpoint['epoch'] best_result = checkpoint['best_result'] model = checkpoint['model'] print("=> loaded best model (epoch {})".format( checkpoint['epoch'])) else: model = checkpoint args.start_epoch = 0 output_directory = os.path.dirname(args.evaluate) validate(val_loader, model, args.start_epoch, write_to_file=False) return # resume from a particular check point elif args.resume: chkpt_path = args.resume assert os.path.isfile( chkpt_path), "=> no checkpoint found at '{}'".format(chkpt_path) print("=> loading checkpoint '{}'".format(chkpt_path)) checkpoint = torch.load(chkpt_path) args = checkpoint['args'] start_epoch = checkpoint['epoch'] + 1 # load epoch number start = start_epoch # resume from the checkpoint epoch best_result = checkpoint['best_result'] # load best result model = checkpoint['model'] # load model optimizer = checkpoint['optimizer'] # load optimizer output_directory = os.path.dirname(os.path.abspath(chkpt_path)) print("=> loaded checkpoint (epoch {})".format(checkpoint['epoch'])) train_loader, val_loader = create_data_loaders( args) # create data loader args.resume = True # create new model if checkpoint does not exist elif args.train: train_loader, val_loader = create_data_loaders( args) # load train and validation data print("=> creating Model ({}-{}) ...".format(args.arch, args.decoder)) in_channels = len(args.modality) if args.arch == 'MobileNet': # if encoder is MobileNet model = models.MobileNetSkipAdd( output_size=train_loader.dataset.output_size ) # MobileNet model is created else: model = models.MobileNetSkipAdd( output_size=train_loader.dataset.output_size ) # by default MobileNet print("=> model created.") optimizer = torch.optim.SGD(model.parameters(), args.lr, \ momentum=args.momentum, weight_decay=args.weight_decay) # configure optimizer if configuration_file.GPU == True: if configuration_file.MULTI_GPU == True: # training on multiple GPU model = torch.nn.DataParallel(model).cuda() else: # training on single GPU model = model.cuda() else: pass # define loss function and optimizer if args.criterion == 'l2': if configuration_file.GPU == True: criterion = MaskedMSELoss().cuda() else: criterion = MaskedMSELoss() elif args.criterion == 'l1': if configuration_file.GPU == True: criterion = MaskedL1Loss().cuda() else: criterion = MaskedL1Loss() # create results folder, if not already exists output_directory = utils.get_output_directory(args) if not os.path.exists(output_directory): # create new directory os.makedirs(output_directory) train_csv = os.path.join(output_directory, 'train.csv') # store training result test_csv = os.path.join(output_directory, 'test.csv') # store test result best_txt = os.path.join(output_directory, 'best.txt') # store best result # create new csv files with only header if not args.resume: with open(train_csv, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() with open(test_csv, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() # training is strarted from here for epoch in range(start, args.epochs): utils.adjust_learning_rate(optimizer, epoch, args.lr) train(train_loader, model, criterion, optimizer, epoch) # train for one epoch result, img_merge = validate(val_loader, model, epoch) # evaluate on validation set # remember best rmse and save checkpoint is_best = result.rmse < best_result.rmse # compare result of the current epoch and best result if is_best: best_result = result with open(best_txt, 'w') as txtfile: txtfile.write( "epoch={}\nmse={:.3f}\nrmse={:.3f}\nabsrel={:.3f}\nlg10={:.3f}\nmae={:.3f}\ndelta1={:.3f}\nt_gpu={:.4f}\n" .format(epoch, result.mse, result.rmse, result.absrel, result.lg10, result.mae, result.delta1, result.gpu_time)) if img_merge is not None: img_filename = output_directory + '/comparison_best.png' utils.save_image(img_merge, img_filename) utils.save_checkpoint( { 'args': args, 'epoch': epoch, 'arch': args.arch, 'model': model, 'best_result': best_result, 'optimizer': optimizer, }, is_best, epoch, output_directory)
def run(self): self.slotsStartWatch.start() self.slotsEndWatch.start() while not self.stopped_thread(): self.check_timeouts() new_timeout = self.get_next_timeout() readable, writable, exceptional = select.select( self.inputs, [], self.inputs, new_timeout) for s in readable: if s is self.sock_tcp: (connection, address) = self.sock_tcp.accept() connection.setblocking(False) self.logger.info('TCP CON (%s, %d)' % (address[0], address[1])) self.inputs.append(connection) else: buffer = utils.read_data_from_socket(s) if buffer: self.logger.debug( 'Req_data = %s\t client = (%s, %d)' % (buffer, s.getpeername()[0], s.getpeername()[1])) action, data = utils.segment_packet(buffer) if action == utils.GATEWAY_NODES_FLASH: image_name = self.get_image_name(s) result = [] for node in data[db_utils.NODES]: if node['status'] == utils.FLASHED: gateway_id = self.dbConnector.find_gateway_by_addr( s.getpeername()) node_uid = self.dbConnector.get_node_uid_by_gateway_id_and_node_id( gateway_id, node['node_id']) self.dbConnector.node_update_flash_info( node_uid, 'FINISHED', image_name) _node = { '_id': node_uid, 'status': node['status'] } else: # node['status'] = utils.ERROR gateway_id = self.dbConnector.find_gateway_by_addr( s.getpeername()) node_uid = self.dbConnector.get_node_uid_by_gateway_id_and_node_id( gateway_id, node['node_id']) _node = { '_id': node_uid, 'status': node['status'] } result.append(_node) self.handle_gateway_request(s, result) self.inputs.remove(s) s.close() elif action == utils.GATEWAY_NODES_ERASE: result = [] for node in data[db_utils.NODES]: if node['status'] == utils.ERASED: gateway_id = self.dbConnector.find_gateway_by_addr( s.getpeername()) node_uid = self.dbConnector.get_node_uid_by_gateway_id_and_node_id( gateway_id, node['node_id']) self.dbConnector.node_update_flash_info( node_uid, 'NOT_STARTED', None) _node = { '_id': node_uid, 'status': node['status'] } else: # node['status'] = utils.ERROR gateway_id = self.dbConnector.find_gateway_by_addr( s.getpeername()) node_uid = self.dbConnector.get_node_uid_by_gateway_id_and_node_id( gateway_id, node['node_id']) _node = { '_id': node_uid, 'status': node['status'] } result.append(_node) self.handle_gateway_request(s, result) self.inputs.remove(s) s.close() elif action == utils.GATEWAY_NODES_RESET: result = [] for node in data[db_utils.NODES]: gateway_id = self.dbConnector.find_gateway_by_addr( s.getpeername()) node_uid = self.dbConnector.get_node_uid_by_gateway_id_and_node_id( gateway_id, node['node_id']) _node = { '_id': node_uid, 'status': node['status'] } result.append(_node) self.handle_gateway_request(s, result) self.inputs.remove(s) s.close() elif action == utils.IMAGES_GET: # { token } token = data[db_utils.TOKEN] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) else: nodetypes_images = utils.get_nodetypes_images( decoded_token[db_utils.USER]) pck = utils.create_response_packet( json.dumps({ 'data': nodetypes_images, 'status': 200 }).encode()) # OnSuccess: { data, status: 200 } s.sendall(pck) elif action == utils.IMAGE_SAVE: # { token, image_name, image_data, nodetype_id } token = data[db_utils.TOKEN] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) else: utils.save_image(decoded_token[db_utils.USER], data[db_utils.NODETYPE_ID], data[db_utils.IMAGE_NAME], data[db_utils.IMAGE_DATA]) pck = utils.create_response_packet( json.dumps({ 'status': 200 }).encode()) # OnSuccess: { status: 200 } s.sendall(pck) elif action == utils.IMAGE_DELETE: # { token, image_name } token = data[db_utils.TOKEN] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) else: nodetype_id = utils.get_nodetype_by_user_and_image_name( decoded_token[db_utils.USER], data[db_utils.IMAGE_NAME]) res = utils.delete_image( decoded_token[db_utils.USER], data[db_utils.IMAGE_NAME], nodetype_id) pck = utils.create_response_packet( json.dumps(res).encode()) # OnSuccess: { status: 204 } # OnError : { status: 404 } s.sendall(pck) elif action == utils.NODES_GET: # { token } token = data[db_utils.TOKEN] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) else: nodes = self.dbConnector.get_nodes() pck = utils.create_response_packet( json.dumps({ 'nodes': nodes, 'status': 200 }).encode()) # OnSuccess: { nodes, status: 200 } s.sendall(pck) elif action == utils.NODES_FLASH: # { token, slot_id, image_name, node_uids} token = data[db_utils.TOKEN] slot_id = data[db_utils.TIMESLOT_ID] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) elif not self.dbConnector.get_slot_by_id(slot_id): utils.send_invalid_slot_error(s) else: gateway_socks = self.send_flash_request( decoded_token[db_utils.USER], data[db_utils.IMAGE_NAME], data[db_utils.NODE_UIDS]) self.inputs.extend(gateway_socks) self.gateway_sockets.append(gateway_socks) self.gateway_sockets_info.append({ 'web_socket': s, 'data': [], db_utils.IMAGE_NAME: data[db_utils.IMAGE_NAME] }) elif action == utils.NODES_ERASE: # { token, slot_id, node_uids } token = data[db_utils.TOKEN] slot_id = data[db_utils.TIMESLOT_ID] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) elif not self.dbConnector.get_slot_by_id(slot_id): utils.send_invalid_slot_error(s) else: gateway_socks = self.send_erase_request( data[db_utils.NODE_UIDS]) self.inputs.extend(gateway_socks) self.gateway_sockets.append(gateway_socks) self.gateway_sockets_info.append({ 'web_socket': s, 'data': [], db_utils.IMAGE_NAME: '' }) elif action == utils.NODES_RESET: # { token, slot_id, node_uids } token = data[db_utils.TOKEN] slot_id = data[db_utils.TIMESLOT_ID] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) elif not self.dbConnector.get_slot_by_id(slot_id): utils.send_invalid_slot_error(s) else: gateway_socks = self.send_reset_request( data[db_utils.NODE_UIDS]) self.inputs.extend(gateway_socks) self.gateway_sockets.append(gateway_socks) self.gateway_sockets_info.append({ 'web_socket': s, 'data': [], db_utils.IMAGE_NAME: '' }) elif action == utils.TIMESLOTS_SAVE: # { token, slots: [{start, end}] } token = data[db_utils.TOKEN] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) else: slots = db_utils.convert_isoformat_to_datetime( data[db_utils.SLOTS]) slots_saved = self.dbConnector.save_timeslots( decoded_token[db_utils.USER], slots) slots = db_utils.convert_datetime_to_isoformat( slots_saved) pck = utils.create_response_packet( json.dumps({ 'slots': slots, 'status': 200 }).encode()) # OnSuccess: { slots: [{start, end}], status: 200 } s.sendall(pck) elif action == utils.TIMESLOTS_GET_DAYSLOTS: # { token, date } token = data[db_utils.TOKEN] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) else: slots_day = self.dbConnector.get_day_slots( data[db_utils.DATE]) slots = db_utils.convert_datetime_to_isoformat( slots_day) pck = utils.create_response_packet( json.dumps({ 'slots': slots, 'status': 200 }).encode()) # OnSuccess: { slots: [{start, end, user_id}], status: 200 } s.sendall(pck) elif action == utils.TIMESLOTS_GET_USERSLOTS: # { token } token = data[db_utils.TOKEN] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) else: user_slots = self.dbConnector.get_user_slots( decoded_token[db_utils.USER]) slots = db_utils.convert_datetime_to_isoformat( user_slots) pck = utils.create_response_packet( json.dumps({ 'slots': slots, 'status': 200 }).encode()) # OnSuccess: { slots: [{slot_id, start, end}], status: 200 } s.sendall(pck) elif action == utils.NODETYPES_GET: # { token } token = data[db_utils.TOKEN] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) else: nodetypes = self.dbConnector.get_nodetypes() pck = utils.create_response_packet( json.dumps({ 'nodetypes': nodetypes, 'status': 200 }).encode()) # OnSuccess: { nodetypes, status: 201 } s.sendall(pck) elif action == utils.USERS_SIGNUP: # { email, username, password } res = self.dbConnector.create_user(data) pck = utils.create_response_packet( json.dumps(res).encode()) # OnSuccess: { status: 201 } # OnError : { message, status: 403 } s.sendall(pck) elif action == utils.USERS_LOGIN: # { email, username } res = self.dbConnector.login_user(data) pck = utils.create_response_packet( json.dumps(res).encode()) # OnSuccess: { token, status: 200} # OnError : { message, status: 401 } s.sendall(pck) elif action == utils.DEBUG_START: # { token, slot_id } token = data[db_utils.TOKEN] slot_id = data[db_utils.TIMESLOT_ID] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) elif not self.dbConnector.get_slot_by_id(slot_id): utils.send_invalid_slot_error(s) else: if self.sock_debug: utils.remove_from_list( self.inputs, self.sock_debug) self.sock_debug.close() log_data = [ '=== DEBUG CHANNEL START ===\n===========================\n' ] pck = utils.create_response_packet( json.dumps({ 'data': log_data }).encode()) self.sock_debug = s self.sock_debug.sendall(pck) elif action == utils.DEBUG_END: # { token, slot_id } token = data[db_utils.TOKEN] slot_id = data[db_utils.TIMESLOT_ID] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) elif not self.dbConnector.get_slot_by_id(slot_id): utils.send_invalid_slot_error(s) else: log_data = [ '=== DEBUG CHANNEL END ===\n=========================\n' ] if self.sock_debug: self.experiment_info = {} pck = utils.create_response_packet( json.dumps({ 'data': log_data, 'message': 'STOP DEBUG' }).encode()) self.sock_debug.sendall(pck) utils.remove_from_list( self.inputs, self.sock_debug) self.sock_debug.close() self.sock_debug = None # { status: 204 } pck = utils.create_response_packet( json.dumps({ 'status': 204 }).encode()) s.sendall(pck) elif action == utils.DEBUG_CLEAR_LOG: # { token, slot_id } token = data[db_utils.TOKEN] slot_id = data[db_utils.TIMESLOT_ID] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) elif not self.dbConnector.get_slot_by_id(slot_id): utils.send_invalid_slot_error(s) else: self.clear_debug_log( decoded_token[db_utils.USER], slot_id) pck = utils.create_response_packet( json.dumps({ 'status': 204 }).encode()) # OnSuccess: { status: 204 } s.sendall(pck) elif action == utils.DEBUG_GET_LOG: # { token, slot_id } token = data[db_utils.TOKEN] slot_id = data[db_utils.TIMESLOT_ID] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) elif not self.dbConnector.get_slot_by_id(slot_id): utils.send_invalid_slot_error(s) else: self.send_debug_log( s, decoded_token[db_utils.USER], slot_id) elif action == utils.DEBUG_GATEWAY: # [ TIMESTAMP, NODE_ID, DATA ] print(data[0], '|', data[1], '|', data[2]) if self.experiment_info: self.write_debug_log(data) if self.sock_debug: pck = utils.create_response_packet( json.dumps({ 'data': data }).encode()) self.sock_debug.sendall(pck) else: self.logger.info( 'TCP DISCON (%s, %d)' % (s.getpeername()[0], s.getpeername()[1])) self.inputs.remove(s) s.close() for s in exceptional: self.inputs.remove(s) s.close() for sock in self.inputs: self.logger.debug('Exiting. . . Closing [%s]' % sock) sock.close() self.log_timer.cancel() self.sock_tcp.close() sys.exit('Exiting. . .')
def train(args, model, device): """ train the model :model: image enhancer model :device: cuda or cpu """ extractor = get_feature_extractor(device) true_labels = torch.ones(config.batch_size, dtype=torch.long).to(device) false_labels = torch.zeros(config.batch_size, dtype=torch.long).to(device) settime = str(int(time.time())) logs = open('logs/' + settime + '.txt', "w+") logs.close() for idx in range(args.resume_iter, config.train_iters): train_original, train_style = load_train_dataset( config.data_path, config.batch_size, config.height * config.width * config.channels) x = torch.from_numpy(train_original).float() y_real = torch.from_numpy(train_style).float() x = x.view(-1, config.height, config.width, config.channels).permute(0, 3, 1, 2).to(device) y_real = y_real.view(-1, config.height, config.width, config.channels).permute(0, 3, 1, 2).to(device) # -------------------------------------------------------------------------------------------------------------- # Train generators # -------------------------------------------------------------------------------------------------------------- y_fake = model.gen_g(x) x_rec = model.gen_f(y_fake) # content loss feat_x = get_feature(extractor, x, config.feature_id, device) feat_x_rec = get_feature(extractor, x_rec, config.feature_id, device) loss_content = torch.pow(feat_x.detach() - feat_x_rec, 2).mean() # color loss # gaussian blur image for discriminator_c fake_blur = gaussian_blur(y_fake, config.kernel_size, config.sigma, config.channels, device) logits_fake_blur = model.dis_c(fake_blur) if args.model_type == 'DCGAN': loss_c = model.criterion(logits_fake_blur, true_labels) elif args.model_type == 'WGAN': loss_c = model.criterion(logits_fake_blur) # texture loss # gray-scale image for discriminator_t fake_gray = gray_scale(y_fake) logits_fake_gray = model.dis_t(fake_gray) if args.model_type == 'DCGAN': loss_t = model.criterion(logits_fake_gray, true_labels) elif args.model_type == 'WGAN': loss_t = model.criterion(logits_fake_gray) # total variation loss height_tv = torch.pow( y_fake[:, :, 1:, :] - y_fake[:, :, :config.height - 1, :], 2).mean() width_tv = torch.pow( y_fake[:, :, :, 1:] - y_fake[:, :, :, :config.width - 1], 2).mean() loss_tv = height_tv + width_tv # total generator loss gen_loss = loss_content + config.lambda_c * loss_c + config.lambda_t * loss_t + config.lambda_tv * loss_tv model.g_optimizer.zero_grad() model.f_optimizer.zero_grad() gen_loss.backward() model.g_optimizer.step() model.f_optimizer.step() # -------------------------------------------------------------------------------------------------------------- # Train discriminators # -------------------------------------------------------------------------------------------------------------- y_fake = model.gen_g(x) # color loss fake_blur = gaussian_blur(y_fake, config.kernel_size, config.sigma, config.channels, device) real_blur = gaussian_blur(y_real, config.kernel_size, config.sigma, config.channels, device) logits_fake_blur = model.dis_c(fake_blur.detach()) logits_real_blur = model.dis_c(real_blur.detach()) if args.model_type == 'DCGAN': loss_dc = model.criterion(logits_real_blur, true_labels) + model.criterion( logits_fake_blur, false_labels) elif args.model_type == 'WGAN': loss_dc = model.criterion(logits_real_blur) - model.criterion( logits_fake_blur) # texture loss fake_gray = gray_scale(y_fake) real_gray = gray_scale(y_real) logits_fake_gray = model.dis_t(fake_gray.detach()) logits_real_gray = model.dis_t(real_gray.detach()) if args.model_type == 'DCGAN': loss_dt = model.criterion(logits_real_gray, true_labels) + model.criterion( logits_fake_gray, false_labels) elif args.model_type == 'WGAN': loss_dt = model.criterion(logits_real_gray) - model.criterion( logits_fake_gray) # total discriminator loss dis_loss = config.lambda_c * loss_dc + config.lambda_t * loss_dt model.c_optimizer.zero_grad() model.t_optimizer.zero_grad() dis_loss.backward() model.c_optimizer.step() model.t_optimizer.step() # Add weight clamping for WGAN if args.model_type == 'WGAN': for param in model.dis_c.parameters(): param.data.clamp_(-config.clamp, config.clamp) for param in model.dis_t.parameters(): param.data.clamp_(-config.clamp, config.clamp) print('Iteration : {}/{}, Gen_loss : {:.4f}, Dis_loss : {:.4f}'.format( idx + 1, config.train_iters, gen_loss.data, dis_loss.data)) print( 'Loss_content : {:.4f}, Loss_c : {:.4f}, Loss_t : {:.4f}, Loss_tv: {:.4f}' .format(loss_content.data, loss_c.data, loss_t.data, loss_tv.data)) print('Loss_dc : {:.4f}, Loss_dt : {:.4f}'.format( loss_dc.data, loss_dt.data)) if (idx + 1) % 50 == 0: log1 = 'Iteration : {}/{}, Gen_loss : {:.4f}, Dis_loss : {:.4f}'.format( idx + 1, config.train_iters, gen_loss.data, dis_loss.data) log2 = 'Loss_content : {:.4f}, Loss_c : {:.4f}, Loss_t : {:.4f}, Loss_tv: {:.4f}'.format( loss_content.data, loss_c.data, loss_t.data, loss_tv.data) log3 = 'Loss_dc : {:.4f}, Loss_dt : {:.4f}'.format( loss_dc.data, loss_dt.data) logs = open('logs/' + settime + '.txt', "a") logs.write(log1) logs.write('\n') logs.write(log2) logs.write('\n') logs.write(log3) logs.write('\n') logs.close() if (idx + 1) % 1000 == 0: sample_path = os.path.join(config.sample_path, args.model_type) checkpoint_path = os.path.join(config.checkpoint_path, args.model_type) utils.save_image( x, os.path.join(sample_path, '{}-x.jpg'.format(idx + 1))) utils.save_image( x_rec, os.path.join(sample_path, '{}-x_rec.jpg'.format(idx + 1))) utils.save_image( y_fake, os.path.join(sample_path, '{}-y_fake.jpg'.format(idx + 1))) utils.save_image( y_real, os.path.join(sample_path, '{}-y_real.jpg'.format(idx + 1))) utils.save_image( fake_blur, os.path.join(sample_path, '{}-fake_blur.jpg'.format(idx + 1))) utils.save_image( real_blur, os.path.join(sample_path, '{}-real_blur.jpg'.format(idx + 1))) utils.save_image( fake_gray, os.path.join(sample_path, '{}-fake_gray.jpg'.format(idx + 1))) utils.save_image( real_gray, os.path.join(sample_path, '{}-real_gray.jpg'.format(idx + 1))) torch.save( model.gen_g.state_dict(), os.path.join(checkpoint_path, '{}-Gen_g.ckpt'.format(idx + 1))) torch.save( model.gen_f.state_dict(), os.path.join(checkpoint_path, '{}-Gen_f.ckpt'.format(idx + 1))) torch.save( model.dis_c.state_dict(), os.path.join(checkpoint_path, '{}-Dis_c.ckpt'.format(idx + 1))) torch.save( model.dis_t.state_dict(), os.path.join(checkpoint_path, '{}-Dis_t.ckpt'.format(idx + 1))) print('Saved intermediate images and model checkpoints.')
# The optimizer global_step = tf.Variable(0, trainable=False) learning_rate = tf.train.exponential_decay(learning_rate=2.0, global_step=global_step, decay_steps=100, decay_rate=0.94, staircase=True) train_step = tf.train.AdamOptimizer(learning_rate).minimize(L, global_step=global_step) # A more simple optimizer # train_step = tf.train.AdamOptimizer(learning_rate=2.0).minimize(L) # Set up the summary writer (saving summaries is optional) # (do `tensorboard --logdir=/tmp/na-logs` to view it) tf.scalar_summary("L_content", L_content) tf.scalar_summary("L_style", L_style) gen_image_addmean = tf.Variable(tf.constant(np.array(content_image, dtype=np.float32)), trainable=False) tf.image_summary("Generated image (TODO: add mean)", gen_image_addmean) summary_op = tf.merge_all_summaries() summary_writer = tf.train.SummaryWriter('/tmp/na-logs', graph_def=sess.graph_def) print "Start calculation..." # The optimizer has variables that require initialization as well sess.run(tf.initialize_all_variables()) for i in range(num_iters): if i % 10 == 0: gen_image_val = sess.run(gen_image) save_image(gen_image_val, i) print "L_content, L_style:", sess.run(L_content), sess.run(L_style) # Increment summary sess.run(tf.assign(gen_image_addmean, add_mean(gen_image_val))) summary_str = sess.run(summary_op) summary_writer.add_summary(summary_str, i) print "Iter:", i sess.run(train_step)
def test_full(args, model, device): """ test the trained model with full images :model: image enhancer model :device: cuda or cpu """ test_path = '../data/full/original/' generate_path = '../data/full/generate/' test_image_num = len([ name for name in os.listdir(test_path) if os.path.isfile(os.path.join(test_path, name)) ]) score_psnr, score_ssim_skimage, score_ssim_minstar, score_msssim_minstar = 0.0, 0.0, 0.0, 0.0 ind = 0 for name in os.listdir(test_path): if os.path.isfile(os.path.join(test_path, name)): ind += 1 test_original, test_style, image_height, image_width = load_test_dataset( name) x = torch.from_numpy(test_original).float() y_real = torch.from_numpy(test_style).float() x = x.view(image_height, image_width, config.channels).permute(2, 0, 1).to(device) y_real = y_real.view(image_height, image_width, config.channels).permute(2, 0, 1).to(device) y_fake = model.gen_g( x.view(-1, config.channels, image_height, image_width)) y_fake = y_fake.view(config.channels, image_height, image_width) # Calculate PSNR & SSIM scores score_psnr += psnr_full(y_fake, y_real) y_fake_np = y_fake.detach().cpu().numpy().transpose(1, 2, 0) y_real_np = y_real.cpu().numpy().transpose(1, 2, 0) temp_ssim, _ = compare_ssim(y_fake_np, y_real_np, multichannel=True, gaussian_weights=True, full=True) score_ssim_skimage += temp_ssim temp_ssim, _ = ssim(y_fake, y_real, kernel_size=11, kernel_sigma=1.5) score_ssim_minstar += temp_ssim score_msssim_minstar += multi_scale_ssim(y_fake, y_real, kernel_size=11, kernel_sigma=1.5) print( 'PSNR & SSIM scores of {} images are calculated.'.format(ind)) utils.save_image( y_fake, os.path.join(generate_path, '{}-x.jpg'.format(name[:5] + args.model_type))) score_psnr /= test_image_num score_ssim_skimage /= test_image_num score_ssim_minstar /= test_image_num score_msssim_minstar /= test_image_num print( 'PSNR : {:.4f}, SSIM_skimage : {:.4f}, SSIM_minstar : {:.4f}, SSIM_msssim: {:.4f}' .format(score_psnr, score_ssim_skimage, score_ssim_minstar, score_msssim_minstar))
def generate(self, inputs, path, idx=None): path = '{}/{}_G.png'.format(path, idx) x = self.sess.run(self.G, {self.z: inputs}) save_image(x, path) print("[*] Samples saved: {}".format(path)) return x
return Variable(x) # End of Helper routines # Load synthetic dataset #syn_image1, syn_image2, syn_label = dataLoading.load_synthetic_ldan_data(synthetic_image_dataset_path) #real_image, sirfs_normal, sirfs_SH, sirfs_shading, real_image_val, sirfs_sh_val, sirfs_normal_val, sirfs_shading_val = dataLoading.load_real_images_celebA(real_image_dataset_path, validation = True) #real_image_mask = dataLoading.getMask(real_image_mask, global_batch_size) #real_image, sirfs_normal, sirfs_SH, sirfs_shading, tNormal, real_image_mask, tSH, real_image_val, sirfs_sh_val, sirfs_normal_val, sirfs_shading_val, true_normal_val, mask_val, true_lighting_val = dataLoading.load_SfSNet_data(sfs_net_path, validation = True, twoLevel = True) syn_image, syn_normal, syn_sh, syn_shading, syn_mask, syn_sirfs_shading, syn_sirfs_normal, syn_sirfs_sh, syn_image_val, syn_normal_val, syn_lighting_val, syn_shading_val, syn_mask_val, syn_sirfs_shading_val, syn_sirfs_normal_val, syn_sirfs_sh_val = dataLoading.load_SfSNet_data(synthetic_data_path, twoLevel = True) real_image, real_normal, real_sh, real_shading, real_mask, real_sirfs_shading, real_sirfs_normal, real_sirfs_sh, real_image_val, real_normal_val, real_lighting_val, real_shading_val, real_mask_val, real_sirfs_shading_val, real_sirfs_normal_val, real_sirfs_sh_val = dataLoading.load_SfSNet_data(real_data_path, validation = True, twoLevel = True) # Transforms being used # if SHOW_IMAGES: ''' syn_image_mask_test = next(iter(mask_val)) utils.save_image(torchvision.utils.make_grid(syn_image_mask_test*255, padding=1), output_path+'images/MASK_TEST.png') tmp = next(iter(syn_image1)) utils.save_image(torchvision.utils.make_grid(tmp, padding=1), output_path+'images/test_synthetic_img.png') tmp = var(next(iter(syn_image_val))) tmp = denorm(tmp) print(tmp.data.shape) tmp = applyMask(tmp, syn_image_mask_test) utils.save_image(torchvision.utils.make_grid(tmp.data, padding=1), output_path+'images/test_syn_image.png') tmp = var(next(iter(syn_normal_val))) tmp = denorm(tmp) tmp = applyMask(tmp, syn_image_mask_test)
def create_model_and_train(self): inputs = self.input targets = self.target def create_discriminator(discrim_inputs, discrim_targets): n_layers = 3 layers = [] # 2x [batch, height, width, in_channels] => [batch, height, width, in_channels * 2] input = tf.concat([discrim_inputs, discrim_targets], axis=3) with tf.variable_scope("layer_1"): convolved = self.discrim_conv(input, a.ndf, stride=2) rectified = self.lrelu(convolved, 0.2) layers.append(rectified) for i in range(n_layers): with tf.variable_scope("layer_%d" % (len(layers) + 1)): out_channels = a.ndf * min(2**(i + 1), 8) stride = 1 if i == n_layers - 1 else 2 # last layer here has stride 1 convolved = self.discrim_conv(layers[-1], out_channels, stride=stride) normalized = self.batchnorm(convolved) rectified = self.lrelu(normalized, 0.2) layers.append(rectified) with tf.variable_scope("layer_%d" % (len(layers) + 1)): convolved = self.discrim_conv(rectified, out_channels=1, stride=1) output = tf.sigmoid(convolved) layers.append(output) return layers[-1] with tf.variable_scope("generator"): out_channels = int(targets.get_shape()[-1]) outputs = self.create_generator(inputs, out_channels) with tf.name_scope("real_discriminator"): with tf.variable_scope("discriminator"): predict_real = create_discriminator(inputs, targets) with tf.name_scope("fake_discriminator"): with tf.variable_scope("discriminator", reuse=True): predict_fake = create_discriminator(inputs, outputs) with tf.name_scope("discriminator_loss"): # minimizing -tf.log will try to get inputs to 1 # predict_real => 1 # predict_fake => 0 discrim_loss = tf.reduce_mean(-(tf.log(predict_real + EPS) + tf.log(1 - predict_fake + EPS))) with tf.name_scope("generator_loss"): # predict_fake => 1 # abs(targets - outputs) => 0 gen_loss_GAN = tf.reduce_mean(-tf.log(predict_fake + EPS)) if a.is_L2: gen_loss_L1 = tf.reduce_mean(tf.square(targets - outputs)) else: gen_loss_L1 = tf.reduce_mean(tf.abs(targets - outputs)) if a.gan_weight == 0: gen_loss = gen_loss_L1 else: gen_loss = gen_loss_GAN * a.gan_weight + gen_loss_L1 * a.l1_weight with tf.name_scope("discriminator_train"): discrim_tvars = [ var for var in tf.trainable_variables() if var.name.startswith("discriminator") ] discrim_optim = tf.train.AdamOptimizer(a.lr, a.beta1) optimizer_d = discrim_optim.minimize(discrim_loss, var_list=discrim_tvars) with tf.name_scope("generator_train"): gen_tvars = [ var for var in tf.trainable_variables() if var.name.startswith("generator") ] gen_optim = tf.train.AdamOptimizer(a.lr, a.beta1) optimizer_g = gen_optim.minimize(gen_loss, var_list=gen_tvars) self.sess.run(tf.global_variables_initializer()) with tf.name_scope("parameter_count"): parameter_count = tf.reduce_sum([ tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables() ]) print("parameter_count =", self.sess.run(parameter_count)) ckpt = tf.train.latest_checkpoint(a.checkpoint_dir) saver = tf.train.Saver() if ckpt: print('syccessfully loaded model from: ' + ckpt) saver.restore(self.sess, ckpt) image_input, image_gt = utils.read_image(a.image_path) if a.is_test: images_path = utils.get_all_image_path(a.test_image_path) for image_path in images_path: image = utils.imread(image_path, half_size=64, image_size=128) image = image[np.newaxis, :, :, np.newaxis] output_image = self.sess.run(outputs, feed_dict={self.input: image}) utils.save_image_output(output_image, image_path, 'train') for i in range(a.max_epochs): print('Train epoch {}:-----'.format(i + 1)) start_time = time.time() batch = 1 for batch_input, batch_ouput in utils.batch_iter(image_input, image_gt, a.batch_size, shuffle=True): if np.random.randint(2, size=1)[0] == 1: # random flip batch_input = np.flip(batch_input, axis=1) batch_ouput = np.flip(batch_ouput, axis=1) if np.random.randint(2, size=1)[0] == 1: # random flip batch_input = np.flip(batch_input, axis=1) batch_ouput = np.flip(batch_ouput, axis=1) train_dict = { self.input: batch_input, self.target: batch_ouput } if a.gan_weight == 0: gen_loss_L1_value, _ = self.sess.run( [gen_loss_L1, optimizer_g], feed_dict=train_dict) if batch % 10 == 0: print('Train batch {}:-----'.format(batch)) print('gen_loss_L1_value {0:.6}'.format( gen_loss_L1_value)) else: discrim_loss_value, gen_loss_GAN_value, gen_loss_L1_value, _, _ = self.sess.run( [ discrim_loss, gen_loss_GAN, gen_loss_L1, optimizer_d, optimizer_g ], feed_dict=train_dict) if batch % 4 == 0: print('Train batch {}:-----'.format(batch)) print( 'discrim_loss {0:.6} gen_loss_GAN_value {1:.6} gen_loss_L1_value {2:.6}' .format(discrim_loss_value, gen_loss_GAN_value, gen_loss_L1_value)) batch += 1 end_time = time.time() time_dif = end_time - start_time print("Time usage: " + str(timedelta(seconds=int(round(time_dif))))) if i % 100 == 0: output_image = self.sess.run( outputs, feed_dict={self.input: batch_input}) utils.save_image(batch_input, output_image, i) saver.save(self.sess, os.path.join(a.checkpoint_dir, 'step_' + str(i)))
def predict(self, file_path): # convert svg to raster image img, num_paths, path_list = self.batch_manager.read_svg(file_path) file_name = os.path.splitext(os.path.basename(file_path))[0] input_img_path = os.path.join(self.model_dir, '%s_0_input.png' % file_name) save_image((1-img[np.newaxis,:,:,np.newaxis])*255, input_img_path, padding=0) # # debug # print(num_paths) # plt.imshow(img, cmap=plt.cm.gray) # plt.show() pm = Param() # predict paths through pathnet start_time = time.time() paths, path_pixels = self.extract_path(img) num_path_pixels = len(path_pixels[0]) pids = self.rng.randint(num_path_pixels, size=8) path_img_path = os.path.join(self.model_dir, '%s_1_path.png' % file_name) save_image((1 - paths[pids,:,:,:])*255, path_img_path, padding=0) # # debug # plt.imshow(paths[0,:,:,0], cmap=plt.cm.gray) # plt.show() duration = time.time() - start_time print('%s: %s, predict paths (#pixels:%d) through pathnet (%.3f sec)' % (datetime.now(), file_name, num_path_pixels, duration)) pm.duration_pred = duration pm.duration = duration dup_dict = {} dup_rev_dict = {} dup_id = num_path_pixels # start id of duplicated pixels if self.find_overlap: # predict overlap using overlap net start_time = time.time() ov = self.overlap(img) overlap_img_path = os.path.join(self.model_dir, '%s_2_overlap.png' % file_name) ov_img = ov[np.newaxis,:,:,np.newaxis] save_image((1-ov_img)*255, overlap_img_path, padding=0) # # debug # plt.imshow(ov, cmap=plt.cm.gray) # plt.show() for i in range(num_path_pixels): if ov[path_pixels[0][i], path_pixels[1][i]]: dup_dict[i] = dup_id dup_rev_dict[dup_id] = i dup_id += 1 # debug # print(dup_dict) # print(dup_rev_dict) duration = time.time() - start_time print('%s: %s, predict overlap (#:%d) through ovnet (%.3f sec)' % (datetime.now(), file_name, dup_id-num_path_pixels, duration)) pm.duration_ov = duration pm.duration += duration else: pm.duration_ov = 0 # write config file for graphcut start_time = time.time() tmp_dir = os.path.join(self.model_dir, 'tmp') if not os.path.exists(tmp_dir): os.makedirs(tmp_dir) pred_file_path = os.path.join(tmp_dir, file_name+'.pred') f = open(pred_file_path, 'w') # info f.write(pred_file_path + '\n') f.write(self.data_path + '\n') f.write('%d\n' % self.max_label) f.write('%d\n' % self.label_cost) f.write('%f\n' % self.sigma_neighbor) f.write('%f\n' % self.sigma_predict) # f.write('%d\n' % num_path_pixels) f.write('%d\n' % dup_id) # support only symmetric edge weight radius = self.sigma_neighbor*2 nb = sklearn.neighbors.NearestNeighbors(radius=radius) nb.fit(np.array(path_pixels).transpose()) high_spatial = 100000 for i in range(num_path_pixels-1): p1 = np.array([path_pixels[0][i], path_pixels[1][i]]) pred_p1 = np.reshape(paths[i,:,:,:], [self.height, self.width]) # see close neighbors and some far neighbors (stochastic sampling) rng = nb.radius_neighbors([p1]) num_close = len(rng[1][0]) far = np.setdiff1d(range(i+1,num_path_pixels),rng[1][0]) num_far = len(far) num_far = int(num_far * self.neighbor_sample) if num_far > 0: far_ids = self.rng.choice(far, size=num_far) nb_ids = np.concatenate((rng[1][0],far_ids)) else: nb_ids = rng[1][0] for rj, j in enumerate(nb_ids): # ids if j <= i: continue p2 = np.array([path_pixels[0][j], path_pixels[1][j]]) if rj < num_close: d12 = rng[0][0][rj] else: d12 = np.linalg.norm(p1-p2, 2) # for j in range(i+1, num_path_pixels): # see entire neighbors # p2 = np.array([path_pixels[0][j], path_pixels[1][j]]) # d12 = np.linalg.norm(p1-p2, 2) pred_p2 = np.reshape(paths[j,:,:,:], [self.height, self.width]) pred = (pred_p1[p2[0],p2[1]] + pred_p2[p1[0],p1[1]]) * 0.5 pred = np.exp(-0.5 * (1.0-pred)**2 / self.sigma_predict**2) spatial = np.exp(-0.5 * d12**2 / self.sigma_neighbor**2) f.write('%d %d %f %f\n' % (i, j, pred, spatial)) dup_i = dup_dict.get(i) if dup_i is not None: f.write('%d %d %f %f\n' % (j, dup_i, pred, spatial)) # as dup is always smaller than normal id f.write('%d %d %f %f\n' % (i, dup_i, 0, high_spatial)) # shouldn't be labeled together dup_j = dup_dict.get(j) if dup_j is not None: f.write('%d %d %f %f\n' % (i, dup_j, pred, spatial)) # as dup is always smaller than normal id f.write('%d %d %f %f\n' % (j, dup_j, 0, high_spatial)) # shouldn't be labeled together if dup_i is not None and dup_j is not None: f.write('%d %d %f %f\n' % (dup_i, dup_j, pred, spatial)) # dup_i < dup_j f.close() duration = time.time() - start_time print('%s: %s, prediction computed (%.3f sec)' % (datetime.now(), file_name, duration)) pm.duration_map = duration pm.duration += duration pm.num_paths = num_paths pm.path_list = path_list pm.path_pixels = path_pixels pm.dup_dict = dup_dict pm.dup_rev_dict = dup_rev_dict pm.img = img pm.file_path = file_path pm.model_dir = self.model_dir pm.height = self.height pm.width = self.width pm.max_label = self.max_label pm.sigma_neighbor = self.sigma_neighbor pm.sigma_predict = self.sigma_predict return pm
def transfer(num_iterations=200): # Reset the graph tf.reset_default_graph() # Start interactive session sess = tf.InteractiveSession() # Initialize a noisy image by adding random noise to the content_image generated_image = generate_noise_image(content_image) # load VGG19 model model = load_vgg_model("imagenet-vgg-verydeep-19.mat") # Assign the content image to be the input of the VGG model. sess.run(model['input'].assign(content_image)) # Select the output tensor of layer conv4_2 out = model['conv4_2'] # Set a_C to be the hidden layer activation from the layer we have selected a_C = sess.run(out) # Set a_G to be the hidden layer activation from same layer. Here, a_G references model['conv4_2'] # and isn't evaluated yet. Later in the code, we'll assign the image G as the model input, so that # when we run the session, this will be the activations drawn from the appropriate layer, with G as input. a_G = out # Compute the content cost J_content = compute_content_cost(a_C, a_G) # Assign the input of the model to be the "style" image sess.run(model['input'].assign(style_image)) # Compute the style cost J_style = compute_style_cost(sess, model) J = total_cost(J_content, J_style, alpha, beta) # 10,40 # define optimizer (1 line) optimizer = tf.train.AdamOptimizer(2.0) # define train_step (1 line) train_step = optimizer.minimize(J) # Initialize global variables (you need to run the session on the initializer) sess.run(tf.global_variables_initializer()) # Run the noisy input image (initial generated image) through the model. Use assign(). sess.run(model["input"].assign(generated_image)) for i in range(num_iterations): # Run the session on the train_step to minimize the total cost sess.run(train_step) # Compute the generated image by running the session on the current model['input'] generated_image = sess.run(model["input"]) # Print every 20 iteration. if i % 20 == 0: Jt, Jc, Js = sess.run([J, J_content, J_style]) print("Iteration " + str(i) + " :") print("total cost = " + str(Jt)) print("content cost = " + str(Jc)) print("style cost = " + str(Js)) # save current generated image in the "/output" directory save_image("images/" + str(i) + ".png", generated_image) # save last generated image save_image("images/output.png", generated_image) return generated_image
def train(self): x_list, xs, ys, sample_list = self.batch_manager.random_list(self.b_num) save_image(xs, '{}/x_gt.png'.format(self.model_dir)) save_image(ys, '{}/y_gt.png'.format(self.model_dir)) with open('{}/gt.txt'.format(self.model_dir), 'w') as f: for sample in sample_list: f.write(sample + '\n') # call once summary_once = self.sess.run(self.summary_once) self.summary_writer.add_summary(summary_once, 0) self.summary_writer.flush() for step in trange(self.start_step, self.max_step): fetch_dict = { "optim": self.optim, "loss": self.loss, } if step % self.log_step == 0 or step == self.max_step-1: fetch_dict.update({ "summary": self.summary_op, }) if step % self.test_step == self.test_step-1 or step == self.max_step-1: l1, l2, iou, nb = 0, 0, 0, 0 for x, y in self.batch_manager.test_batch(): if self.data_format == 'NCHW': x = to_nchw_numpy(x) y = to_nchw_numpy(y) tl1, tl2, y_ = self.sess.run([self.tl1, self.tl2, self.yt_], {self.xt: x, self.yt: y}) l1 += tl1 l2 += tl2 nb += 1 # iou y_I = np.logical_and(y>0, y_>0) y_I_sum = np.sum(y_I, axis=(1, 2, 3)) y_U = np.logical_or(y>0, y_>0) y_U_sum = np.sum(y_U, axis=(1, 2, 3)) # print(y_I_sum, y_U_sum) nonzero_id = np.where(y_U_sum != 0)[0] if nonzero_id.shape[0] == 0: acc = 1.0 else: acc = np.average(y_I_sum[nonzero_id] / y_U_sum[nonzero_id]) iou += acc if nb > 500: break l1 /= float(nb) l2 /= float(nb) iou /= float(nb) summary_test = self.sess.run(self.summary_test, {self.test_acc_l1: l1, self.test_acc_l2: l2, self.test_acc_iou: iou}) self.summary_writer.add_summary(summary_test, step) self.summary_writer.flush() result = self.sess.run(fetch_dict) if step % self.log_step == 0 or step == self.max_step-1: self.summary_writer.add_summary(result['summary'], step) self.summary_writer.flush() loss = result['loss'] assert not np.isnan(loss), 'Model diverged with loss = NaN' print("\n[{}/{}] Loss: {:.6f}".format(step, self.max_step, loss)) if step % (self.log_step * 10) == 0 or step == self.max_step-1: self.generate(x_list, self.model_dir, idx=step) if step % self.lr_update_step == self.lr_update_step - 1: self.sess.run(self.lr_update) # save last checkpoint.. save_path = os.path.join(self.model_dir, 'model.ckpt') self.saver.save(self.sess, save_path, global_step=self.step) self.batch_manager.stop_thread()
print("current batch size:", len(filenames)) adversarial_samples = sess.run(gen_adversarial_sample, feed_dict={inceptionV3.x_input: images})[:len(filenames)] predictions = sess.run([inceptionV3.predicts, sm], feed_dict={inceptionV3.x_input: images}) adversarial_predictions = sess.run( inceptionV3.predicts, feed_dict={inceptionV3.x_input: adversarial_samples})[:len(filenames)] imageIds += list(map(lambda x: x[:-4], filenames)) predictedIds += list(predictions[0][:len(filenames)]) adversarialPredictedIds += list(adversarial_predictions) for adv, fn in zip(adversarial_samples, filenames): save_image("output/Attack_" + fn, adv) # generate saliency maps # sms = sess.run(sm, feed_dict={inceptionV3.x_input: images})[:len(filenames)] # for sm_image,fn in zip(sms,filenames): # save_image("output/sm_"+fn,(sm_image-1.0)*2.) summary_frame = pd.DataFrame({"ImageId":imageIds,"predictedLabel":predictedIds,"adversarialLabel":adversarialPredictedIds})\ .merge(image_lc,how='left',on='ImageId') summary_frame = summary_frame.merge(Id2name, how='left', left_on='predictedLabel', right_on='CategoryId', suffixes=['_t', '_p']).drop('CategoryId',
if optimize_simply: # A more simple optimizer train_step = tf.train.AdamOptimizer(learning_rate=2.0).minimize(L) else: # Complicated optimizer train_step = tf.train.AdamOptimizer(learning_rate).minimize(L, global_step=global_step) # Set up the summary writer (saving summaries is optional) # (do `tensorboard --logdir=/tmp/na-logs` to view it) tf.scalar_summary("L_content", L_content) tf.scalar_summary("L_style", L_style) gen_image_addmean = tf.Variable(tf.constant(np.array(content_image, dtype=np.float32)), trainable=False) tf.image_summary("Generated image (TODO: add mean)", gen_image_addmean) summary_op = tf.merge_all_summaries() summary_writer = tf.train.SummaryWriter('/tmp/na-logs', graph_def=sess.graph_def) print "Start calculation..." # The optimizer has variables that require initialization as well sess.run(tf.initialize_all_variables()) for i in range(num_iters): if i % 10 == 0: gen_image_val = sess.run(gen_image) save_image(gen_image_val, i, args.out_dir) print "L_content, L_style:", sess.run(L_content), sess.run(L_style) # Increment summary sess.run(tf.assign(gen_image_addmean, add_mean(gen_image_val))) summary_str = sess.run(summary_op) summary_writer.add_summary(summary_str, i) print "Iter:", i sess.run(train_step)
def train(self, n_iters): skip_step = 1 with tf.Session() as sess: ############################### # TO DO: # 1. initialize your variables # 2. create writer to write your graph ############################### sess.run(tf.global_variables_initializer()) writer = tf.summary.FileWriter('graphs', sess.graph) sess.run(self.input_img.assign(self.initial_img)) ############################### # TO DO: # 1. create a saver object # 2. check if a checkpoint exists, restore the variables ############################## saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state( os.path.dirname('checkpoints/checkpoint')) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) initial_step = self.gstep.eval() start_time = time.time() for index in range(initial_step, n_iters): if 5 <= index < 20: skip_step = 10 elif index >= 20: skip_step = 20 sess.run(self.opt) if (index + 1) % skip_step == 0: ############################### # TO DO: obtain generated image, loss, and summary ############################### gen_image, total_loss, summary = sess.run( [self.input_img, self.total_loss, self.summary_op]) ############################### # add back the mean pixels we subtracted before gen_image = gen_image + self.vgg.mean_pixels writer.add_summary(summary, global_step=index) print('Step {}\n Sum: {:5.1f}'.format( index + 1, np.sum(gen_image))) print(' Loss: {:5.1f}'.format(total_loss)) print(' Took: {} seconds'.format(time.time() - start_time)) start_time = time.time() filename = './outputs/%d.png' % index utils.save_image(filename, gen_image) if (index + 1) % 20 == 0: ############################### # TO DO: save the variables into a checkpoint ############################### saver.save(sess, './checkpoints/style_transfer', index)
(x % self.overrect_lava.width, y % self.overrect_lava.height) ) over_rock = self.overimage_rocks.get_at( (x % self.overrect_rocks.width, y % self.overrect_rocks.height) ) # Processed colours should be between 1 and 250 (if it's 0,0,0 it'll be keyed) new_colour.r = max(1, min(new_colour.r + over_lava.r * 2 - over_rock.r / 2, 250)) new_colour.g = max(1, min(new_colour.g - over_lava.r - over_rock.r / 2, 250)) new_colour.b = max(1, min(new_colour.b - over_lava.r - over_rock.r / 2, 250)) """ this draws a ratio*ratio grid in the level, useful for some debugging if x % self.ratio == 0 or y % self.ratio == 0: new_colour = (0,0,255) """ self.set_at((x, y), new_colour) utils.save_image("{0}_processed.png".format(stage_file), self) level_md5 = utils.file_md5("{0}.gif".format(stage_file)) print "MD5 of stage: {0}".format(level_md5) utils.set_option("{0}_hash".format(stage_file), level_md5) def calculate_limits(self): # TODO: cache limits, it takes a while to process them for x in range(self.rect.width - 1): # will store top and bottom limits and append it to self.limits # to control the ship not getting over this x_limits = [-1, -1] for y in range(self.rect.height): # Calculate top and bottom limits if x_limits[0] == -1 and self.level_data.get_at((x, y)) != self.colors["grass"]: x_limits[0] = y if x_limits[0] != -1 and x_limits[1] == -1 and self.level_data.get_at((x, y)) == self.colors["grass"]:
def train(self): print("训练") x_fixed, x_target_fixed, pose_fixed, pose_target_fixed, mask_fixed, mask_target_fixed = \ self.get_image_from_loader() # 以下是灰度图 # print("x_fixed:", x_fixed.shape) # print("x_target_fixed:", x_target_fixed.shape) # print("mask_fixed:", mask_fixed.shape) # print("mask_target_fixed:", mask_target_fixed.shape) # print("pose_fixed[0][40][40]:", np.array(pose_fixed[0][40][40])) # print("pose_target_fixed:", pose_target_fixed.shape) save_image(x_fixed, '{}/.png'.format(self.model_dir)) save_image(x_target_fixed, '{}/x_target_fixed.png'.format(self.model_dir)) save_image((np.amax(pose_fixed, axis=-1, keepdims=True)+1.0)*127.5, '{}/pose_fixed.png'.format(self.model_dir)) save_image((np.amax(pose_target_fixed, axis=-1, keepdims=True)+1.0)*127.5, '{}/pose_target_fixed.png'.format(self.model_dir)) save_image(mask_fixed, '{}/mask_fixed.png'.format(self.model_dir)) save_image(mask_target_fixed, '{}/mask_target_fixed.png'.format(self.model_dir)) for step in trange(self.start_step, self.max_step): if step < 22000: self.sess.run(self.g_optim1) else: # Train generator if step > 0: self.sess.run(self.g_optim2) # Train critic if (self.wgan_gp.MODE == 'dcgan') or (self.wgan_gp.MODE == 'lsgan'): disc_ITERS = 1 else: disc_ITERS = self.wgan_gp.CRITIC_ITERS for i in xrange(disc_ITERS): self.sess.run(self.d_optim) if self.wgan_gp.MODE == 'wgan': self.sess.run(self.clip_disc_weights) fetch_dict = {} if step % self.log_step == self.log_step-1: fetch_dict.update({"summary": self.summary_op}) # "k_t": self.k_t, result = self.sess.run(fetch_dict) if step % self.log_step == self.log_step-1: self.summary_writer.add_summary(result['summary'], step) self.summary_writer.flush() if step % (self.log_step * 3) == (self.log_step * 3)-1: # if self.data_format == 'NCHW': # x = x_fixed.transpose([0, 3, 1, 2]) # else: # x = x_fixed x = utils_wgan.process_image(x_fixed, 127.5, 127.5) x_target = utils_wgan.process_image(x_target_fixed, 127.5, 127.5) self.generate(x, x_target, pose_target_fixed, self.model_dir, idx=step) if step % self.lr_update_step == self.lr_update_step - 1: self.sess.run([self.g_lr_update, self.d_lr_update]) if step % (self.log_step * 30) == (self.log_step * 30)-1: self.saver.save(self.sess, os.path.join(self.model_dir, 'model.ckpt'), global_step=step)
def write_grid(grid, global_step): """Native python function to call for writing images to files.""" if global_step % summary_freq == 0: img_path = os.path.join(log_dir, '%s.jpg' % str(global_step)) utils.save_image(grid, img_path) return 0
index_record = defaultdict(int) for index, row in labels_df.iterrows(): image_name = row['Frame'] full_path_image_name = os.path.join(data_path, image_name) if os.path.isfile(full_path_image_name): image_name = image_name.split('.')[0] image = utils.load_image(full_path_image_name) xmin = row['xmin'] #if row['xmin'] < row['xmax'] else row['xmax'] xmax = row['xmax'] #if row['xmin'] < row['xmax'] else row['xmin'] ymin = row['ymin'] #if row['ymin'] < row['ymax'] else row['ymax'] ymax = row['ymax'] #if row['ymin'] < row['ymax'] else row['ymin'] cropped_image = image[xmax:ymax, xmin:ymin, :] label = row['Label'] key = '{}_{}'.format(image_name, label) obj_index = index_record[key] obj_index += 1 save_image_full_path = '{path}{image_name}_{obj_index}_{label}.jpg'.format( path=save_path, image_name=image_name, obj_index=obj_index, label=label) index_record[key] = obj_index try: resized_image = utils.resize_image(cropped_image, image_size) utils.save_image(resized_image, save_image_full_path) except Exception as e: print 'failed image {}'.format(image_name)
import cv2 import numpy as np import matplotlib.pyplot as plt from utils import vis_hybrid_image, load_image, save_image, im_range from student_code import my_imfilter, create_hybrid_image image1 = load_image('../data/dog.bmp') image2 = load_image('../data/cat.bmp') cutoff_frequency = 7 filter = cv2.getGaussianKernel(ksize=cutoff_frequency * 4 + 1, sigma=cutoff_frequency) filter = np.dot(filter, filter.T) low_frequencies, high_frequencies, hybrid_image = create_hybrid_image( image1, image2, filter) vis = vis_hybrid_image(hybrid_image) save_image('../results/hybrid_image.jpg', hybrid_image) save_image('../results/hybrid_image_scales.jpg', vis)
# input_real, input_cropped = input_real.cuda(),input_cropped.cuda() # criterionMSE.cuda() # real_center = real_center.cuda() input_real = Variable(input_real) input_cropped = Variable(input_cropped) real_center = Variable(real_center) input_real.data.resize_(image.size()).copy_(image) input_cropped.data.resize_(image.size()).copy_(image) real_center_cpu = image[:,:,opt.imageSize/4:opt.imageSize/4+opt.imageSize/2,opt.imageSize/4:opt.imageSize/4+opt.imageSize/2] real_center.data.resize_(real_center_cpu.size()).copy_(real_center_cpu) input_cropped.data[:,0,opt.imageSize/4+opt.overlapPred:opt.imageSize/4+opt.imageSize/2-opt.overlapPred,opt.imageSize/4+opt.overlapPred:opt.imageSize/4+opt.imageSize/2-opt.overlapPred] = 2*117.0/255.0 - 1.0 input_cropped.data[:,1,opt.imageSize/4+opt.overlapPred:opt.imageSize/4+opt.imageSize/2-opt.overlapPred,opt.imageSize/4+opt.overlapPred:opt.imageSize/4+opt.imageSize/2-opt.overlapPred] = 2*104.0/255.0 - 1.0 input_cropped.data[:,2,opt.imageSize/4+opt.overlapPred:opt.imageSize/4+opt.imageSize/2-opt.overlapPred,opt.imageSize/4+opt.overlapPred:opt.imageSize/4+opt.imageSize/2-opt.overlapPred] = 2*123.0/255.0 - 1.0 fake = netG(input_cropped) errG = criterionMSE(fake,real_center) recon_image = input_cropped.clone() recon_image.data[:,:,opt.imageSize/4:opt.imageSize/4+opt.imageSize/2,opt.imageSize/4:opt.imageSize/4+opt.imageSize/2] = fake.data utils.save_image('val_real_samples.png',image[0]) utils.save_image('val_cropped_samples.png',input_cropped.data[0]) utils.save_image('val_recon_samples.png',recon_image.data[0]) print('%.4f' % errG.data[0])
n = min(data.shape[0], 8) comparison = np.concatenate([data[:n], recon_x.npvalue().T.reshape(batch_size, 1, 28, 28)[:n]]) save_image(comparison, 'results/reconstruction_' + str(epoch) + '.png', nrow=n) test_loss /= len(test_data) print('====> Test set loss: {:.4f}'.format(test_loss)) import time tictocs = [] for epoch in range(1, args.epochs + 1): tic = time.time() train(epoch) test(epoch) sample = dy.inputTensor(np.random.randn(20, 64)) sample = model.decode(sample) save_image(sample.npvalue().T.reshape(64, 1, 28, 28), 'results/sample_' + str(epoch) + '.png') toc = time.time() tictocs.append(toc - tic) print('############\n\n') print('Total Time Cost:', np.sum(tictocs)) print('Epoch Time Cost', np.average(tictocs), '+-', np.std(tictocs) / np.sqrt(len(tictocs))) print('\n\n############')
def main(): # parse arguments args = parse_args() if args is None: exit() # initiate VGG19 model model_file_path = args.model_path + '/' + vgg19.MODEL_FILE_NAME vgg_net = vgg19.VGG19(model_file_path) # load content image and style image content_image = utils.load_image(args.content, max_size=args.max_size) style_image = utils.load_image(args.style, shape=(content_image.shape[1], content_image.shape[0])) ############## (w*h*3)으로 입력이 들어와야 가능 content_image = np.load('cont.npy') content_image = np.transpose(content_image, (1, 2, 0)) # content_image = np.expand_dims(content_image,axis=0) # content_image = np.zeros((256,256,3)) print(content_image.shape) ###########3## # initial guess for output if args.initial_type == 'content': init_image = content_image elif args.initial_type == 'style': init_image = style_image elif args.initial_type == 'random': init_image = np.random.normal(size=content_image.shape, scale=np.std(content_image)) # check input images for style-transfer # utils.plot_images(content_image,style_image, init_image) # create a map for content layers info CONTENT_LAYERS = {} for layer, weight in zip(args.content_layers, args.content_layer_weights): CONTENT_LAYERS[layer] = weight # create a map for style layers info STYLE_LAYERS = {} for layer, weight in zip(args.style_layers, args.style_layer_weights): STYLE_LAYERS[layer] = weight # open session sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # build the graph st = style_transfer.StyleTransfer( session=sess, content_layer_ids=CONTENT_LAYERS, style_layer_ids=STYLE_LAYERS, init_image=add_one_dim(init_image), content_image=add_one_dim(content_image), style_image=add_one_dim(style_image), net=vgg_net, num_iter=args.num_iter, loss_ratio=args.loss_ratio, content_loss_norm_type=args.content_loss_norm_type, ) # launch the graph in a session result_image = st.update() # close session sess.close() # remove batch dimension shape = result_image.shape result_image = np.reshape(result_image, shape[1:]) # save result utils.save_image(result_image, args.output)
metric.update([real_label, ], [output, ]) # train with fake image fake_image = g_net(noise) output = d_net(fake_image.detach()).reshape((-1, 1)) errD_fake = loss(output, fake_label) errD = errD_real + errD_fake errD.backward() metric.update([fake_label, ], [output, ]) d_trainer.step(BATCH_SIZE) # update G with autograd.record(): fake_image = g_net(noise) output = d_net(fake_image).reshape(-1, 1) errG = loss(output, real_label) errG.backward() g_trainer.step(BATCH_SIZE) # print log infomation every 100 batches if i % 100 == 0: name, acc = metric.get() logging.info('discriminator loss = %f, generator loss = %f, \ binary training acc = %f at iter %d epoch %d', nd.mean(errD).asscalar(), nd.mean(errG).asscalar(), acc, i, epoch) if i == 0: save_image(fake_image, epoch, IMAGE_SIZE, BATCH_SIZE, OUTPUT_DIR) metric.reset()
def train(self, n_iters): skip_step = 1 with tf.Session() as sess: ############################### ## TO DO: ## 1. initialize your variables ## 2. create writer to write your grapp sess.run(tf.global_variables_initializer()) writer = tf.summary.FileWriter("graphs", sess.graph) # writer.add_summary(self.summary_op) ############################### sess.run(self.input_img.assign(self.initial_img)) ############################### ## TO DO: ## 1. create a saver object ## 2. check if a checkpoint exists, restore the variables saver = tf.train.Saver() try: saver.restore(sess, "graphs/style_transfer.ckpt") except: pass ############################## initial_step = self.gstep.eval() start_time = time.time() for index in range(initial_step, n_iters): if index >= 5 and index < 20: skip_step = 10 elif index >= 20: skip_step = 20 sess.run(self.opt) if (index + 1) % skip_step == 0: ############################### ## TO DO: obtain generated image, loss, and summary gen_image, total_loss, summary = sess.run( [self.input_img, self.total_loss, self.summary_op]) ############################### # add back the mean pixels we subtracted before gen_image = gen_image + self.vgg.mean_pixels writer.add_summary(summary, global_step=index) print('Step {}\n Sum: {:5.1f}'.format( index + 1, np.sum(gen_image))) print(' Loss: {:5.1f}'.format(total_loss)) print(' Took: {} seconds'.format(time.time() - start_time)) start_time = time.time() filename = 'outputs/%d.png' % (index) utils.save_image(filename, gen_image) if (index + 1) % 20 == 0: ############################### ## TO DO: save the variables into a checkpoint ############################### saver.save(sess, "graph/style_transfer.ckpt")
def stylize(args): device = torch.device("cuda" if args.cuda else "cpu") content_image = utils.load_image(args.content_image, scale=args.content_scale) content_transform = transforms.Compose( [transforms.ToTensor(), transforms.Lambda(lambda x: x.mul(255))]) content_image = content_transform(content_image) content_image = content_image.unsqueeze(0).to(device) if args.model.endswith(".onnx"): output = stylize_onnx_caffe2(content_image, args) else: with torch.no_grad(): if args.distilled: style_model = SmallTransformerNet() else: style_model = TransformerNet() # Load model from checkpoint state_dict = torch.load(args.model) if 'state_dict' in state_dict: state_dict = state_dict['state_dict'] # remove saved deprecated running_* keys in InstanceNorm from the checkpoint for k in list(state_dict.keys()): if re.search(r'in\d+\.running_(mean|var)$', k): del state_dict[k] style_model.load_state_dict(state_dict) style_model.to(device) if args.export_onnx: assert args.export_onnx.endswith( ".onnx"), "Export model file should end with .onnx" output = torch.onnx._export(style_model, content_image, args.export_onnx).cpu() else: output = style_model(content_image).cpu() utils.save_image(args.output_image, output[0]) # If required, print the loss function of the generated image if args.print_loss: with torch.no_grad(): vgg = Vgg16(requires_grad=False).to(device) features_x = vgg(content_image) features_y = vgg(output) # Content loss mse_loss = torch.nn.MSELoss() content_loss = (args.content_weight * mse_loss(features_y.relu2_2, features_x.relu2_2)) # Style loss style_transform = transforms.Compose([ transforms.ToTensor(), transforms.Lambda(lambda x: x.mul(255)) ]) style = utils.load_image(args.style_image, size=args.style_size) style = style_transform(style) style = style.repeat(1, 1, 1, 1).to(device) feature_style = vgg(style) gram_style = utils.gram_matrix(feature_style[0]) gram_y = utils.gram_matrix(features_y[0]) style_loss = mse_loss(gram_y, gram_style) style_loss *= args.style_weight # Print the losses total_loss = content_loss + style_loss print("Style: {:.4} ; Content: {:.4} ; Total: {:.4}".format( style_loss, content_loss, total_loss))
def test(self): print("测试") test_result_dir = os.path.join(self.model_dir, 'test_result') test_result_dir_x = os.path.join(test_result_dir, 'x') test_result_dir_x_target = os.path.join(test_result_dir, 'x_target') test_result_dir_G = os.path.join(test_result_dir, 'G') test_result_dir_pose = os.path.join(test_result_dir, 'pose') test_result_dir_pose_target = os.path.join(test_result_dir, 'pose_target') test_result_dir_mask = os.path.join(test_result_dir, 'mask') test_result_dir_mask_target = os.path.join(test_result_dir, 'mask_target') if not os.path.exists(test_result_dir): os.makedirs(test_result_dir) if not os.path.exists(test_result_dir_x): os.makedirs(test_result_dir_x) if not os.path.exists(test_result_dir_x_target): os.makedirs(test_result_dir_x_target) if not os.path.exists(test_result_dir_G): os.makedirs(test_result_dir_G) if not os.path.exists(test_result_dir_pose): os.makedirs(test_result_dir_pose) if not os.path.exists(test_result_dir_pose_target): os.makedirs(test_result_dir_pose_target) if not os.path.exists(test_result_dir_mask): os.makedirs(test_result_dir_mask) if not os.path.exists(test_result_dir_mask_target): os.makedirs(test_result_dir_mask_target) for i in xrange(400): x_fixed, x_target_fixed, pose_fixed, pose_target_fixed, mask_fixed, mask_target_fixed = self.get_image_from_loader() x = utils_wgan.process_image(x_fixed, 127.5, 127.5) x_target = utils_wgan.process_image(x_target_fixed, 127.5, 127.5) if 0==i: x_fake = self.generate(x, x_target, pose_target_fixed, test_result_dir, idx=self.start_step, save=True) else: x_fake = self.generate(x, x_target, pose_target_fixed, test_result_dir, idx=self.start_step, save=False) p = (np.amax(pose_fixed, axis=-1, keepdims=False)+1.0)*127.5 pt = (np.amax(pose_target_fixed, axis=-1, keepdims=False)+1.0)*127.5 for j in xrange(self.batch_size): idx = i*self.batch_size+j im = Image.fromarray(x_fixed[j,:].astype(np.uint8)) im.save('%s/%05d.png'%(test_result_dir_x, idx)) im = Image.fromarray(x_target_fixed[j,:].astype(np.uint8)) im.save('%s/%05d.png'%(test_result_dir_x_target, idx)) im = Image.fromarray(x_fake[j,:].astype(np.uint8)) im.save('%s/%05d.png'%(test_result_dir_G, idx)) im = Image.fromarray(p[j,:].astype(np.uint8)) im.save('%s/%05d.png'%(test_result_dir_pose, idx)) im = Image.fromarray(pt[j,:].astype(np.uint8)) im.save('%s/%05d.png'%(test_result_dir_pose_target, idx)) im = Image.fromarray(mask_fixed[j,:].squeeze().astype(np.uint8)) im.save('%s/%05d.png'%(test_result_dir_mask, idx)) im = Image.fromarray(mask_target_fixed[j,:].squeeze().astype(np.uint8)) im.save('%s/%05d.png'%(test_result_dir_mask_target, idx)) if 0==i: save_image(x_fixed, '{}/x_fixed.png'.format(test_result_dir)) save_image(x_target_fixed, '{}/x_target_fixed.png'.format(test_result_dir)) save_image(mask_fixed, '{}/mask_fixed.png'.format(test_result_dir)) save_image(mask_target_fixed, '{}/mask_target_fixed.png'.format(test_result_dir)) save_image((np.amax(pose_fixed, axis=-1, keepdims=True)+1.0)*127.5, '{}/pose_fixed.png'.format(test_result_dir)) save_image((np.amax(pose_target_fixed, axis=-1, keepdims=True)+1.0)*127.5, '{}/pose_target_fixed.png'.format(test_result_dir))