def evaluate_inception(self): incep_batch_size = self.cfg.EVAL.INCEP_BATCH_SIZE logits, _ = load_inception_inference( self.sess, self.classes, incep_batch_size, self.cfg.EVAL.INCEP_CHECKPOINT_DIR) pred_op = tf.nn.softmax(logits) z = tf.placeholder(tf.float32, [self.bs, self.model.z_dim], name='z') cond = tf.placeholder(tf.float32, [self.bs] + [self.model.embed_dim], name='cond') eval_gen, _, _ = self.model.generator(z, cond, reuse=False, is_training=False) saver = tf.train.Saver(tf.global_variables('g_net')) could_load, _ = load(saver, self.sess, self.cfg.CHECKPOINT_DIR) if could_load: print(" [*] Load SUCCESS") else: print(" [!] Load failed...") raise RuntimeError( 'Could not load the checkpoints of the generator') print('Generating x...') size = self.cfg.EVAL.SIZE n_batches = size // self.bs w, h, c = self.model.image_dims[0], self.model.image_dims[ 1], self.model.image_dims[2] samples = np.zeros((n_batches * self.bs, w, h, c)) for i in range(n_batches): print("\rGenerating batch %d/%d" % (i + 1, n_batches), end="", flush=True) sample_z = np.random.normal(0, 1, size=(self.bs, self.model.z_dim)) _, _, embed, _, _ = self.dataset.test.next_batch(self.bs, 4, embeddings=True) start = i * self.bs end = start + self.bs gen_batch = self.sess.run(eval_gen, feed_dict={ z: sample_z, cond: embed }) samples[start:end] = denormalize_images(gen_batch) print('\nComputing inception score...') mean, std = inception_score.get_inception_score(samples, self.sess, incep_batch_size, 10, pred_op, verbose=True) print('Inception Score | mean:', "%.2f" % mean, 'std:', "%.2f" % std)
def evaluate_fid(self): incep_batch_size = self.cfg.EVAL.INCEP_BATCH_SIZE _, layers = load_inception_inference(self.sess, 20, incep_batch_size, self.cfg.EVAL.INCEP_CHECKPOINT_DIR) pool3 = layers['PreLogits'] act_op = tf.reshape(pool3, shape=[incep_batch_size, -1]) if not os.path.exists(self.cfg.EVAL.ACT_STAT_PATH): print('Computing activation statistics for real x') fid.compute_and_save_activation_statistics(self.cfg.EVAL.R_IMG_PATH, self.sess, incep_batch_size, act_op, self.cfg.EVAL.ACT_STAT_PATH, verbose=True) print('Loading activation statistics for the real x') stats = np.load(self.cfg.EVAL.ACT_STAT_PATH) mu_real = stats['mu'] sigma_real = stats['sigma'] z = tf.placeholder(tf.float32, [self.bs, self.model.z_dim], name='real_images') cond = tf.placeholder(tf.float32, [self.bs] + [self.model.embed_dim], name='cond') eval_gen, _, _ = self.model.generator(z, cond, reuse=False) saver = tf.train.Saver(tf.global_variables('g_net')) could_load, _ = load(saver, self.sess, self.cfg.CHECKPOINT_DIR) if could_load: print(" [*] Load SUCCESS") else: print(" [!] Load failed...") raise RuntimeError('Could not load the checkpoints of the generator') print('Generating x...') fid_size = self.cfg.EVAL.SIZE n_batches = fid_size // self.bs w, h, c = self.model.image_dims[0], self.model.image_dims[1], self.model.image_dims[2] samples = np.zeros((n_batches * self.bs, w, h, c)) for i in range(n_batches): start = i * self.bs end = start + self.bs sample_z = np.random.normal(0, 1, size=(self.bs, self.model.z_dim)) images, _, embed, _, _ = self.dataset.test.next_batch(self.bs, 4, embeddings=True) samples[start: end] = denormalize_images(self.sess.run(eval_gen, feed_dict={z: sample_z, cond: embed})) print('Computing activation statistics for generated x...') mu_gen, sigma_gen = fid.calculate_activation_statistics(samples, self.sess, incep_batch_size, act_op, verbose=True) print("calculate FID:", end=" ", flush=True) try: FID = fid.calculate_frechet_distance(mu_gen, sigma_gen, mu_real, sigma_real) except Exception as e: print(e) FID = 500 print(FID)
def __init__(self, sess: tf.Session, model: GanCls, dataset: TextDataset, config): self.sess = sess self.model = model self.sampler = model.sampler self.dataset = dataset self.config = config self.saver = tf.train.Saver() could_load, _ = load(self.saver, self.sess, self.config.checkpoint_dir) if could_load: print(" [*] Load SUCCESS") else: print(" [!] Load failed...") raise LookupError('Could not load any checkpoints')
def check_save(): try: save_path = config.SAVE_PATH res = saver.load(save_path) if res == False: # 追加提示 print('未发现存档文件') return False else: if res['slot1'] == 'null' and res['slot2'] == 'null' and res[ 'slot3'] == 'null': return False return True except Exception as err: return False
def load_inception_inference(sess, num_classes, batch_size, checkpoint_dir): """Loads the inception network with the parameters from checkpoint_dir""" # Build a Graph that computes the logits predictions from the inference model. inputs = tf.placeholder(tf.float32, [batch_size, 299, 299, 3], name='inputs') logits, layers = inception_net(inputs, num_classes) inception_vars = tf.global_variables('InceptionV3') saver = tf.train.Saver(inception_vars) print('Restoring Inception model from %s' % checkpoint_dir) could_load, _ = load(saver, sess, checkpoint_dir) if could_load: print(" [*] Load SUCCESS") else: print(" [!] Load failed...") return logits, layers
def dojob(x, y, is_mouse_down, keys): #显示底版 loader.screen.blit(loader.SAVE_BOARD_TITLE, (0, 0)) a0 = (x >= 20 and x < 100 and y >= 20 and y < 60) a1 = (x >= 0 and x <= 860 and y >= 80 and y < 280) a2 = (x >= 0 and x <= 860 and y >= 280 and y < 480) a3 = (x >= 0 and x <= 860 and y >= 480 and y < 680) if a0: loader.screen.blit(loader.SAVE_BOARD_TITLE_BACK2, (20, 20)) if is_mouse_down == True: player_runtime.INFO['loading'] = False else: loader.screen.blit(loader.SAVE_BOARD_TITLE_BACK1, (20, 20)) if player_runtime.INFO['checksover'] == True: loader.screen.blit(loader.SAVE_BOARD_LOT, (0, 80)) loader.screen.blit(loader.SAVE_BOARD_LOT, (0, 280)) loader.screen.blit(loader.SAVE_BOARD_LOT, (0, 480)) s = 90 for sd in player_runtime.SDATA: try: img_url = sd['img_path'] if not img_url == '': img_res = sd['img_res'] cr = sd['round'] cd = sd['date'] loader.screen.blit(img_res, (15, s)) crt = loader.BAIKE_FONT.render(cr, True, color_rgb.WHITE, None) cdt = loader.BAIKE_FONT.render(cd, True, color_rgb.WHITE, None) loader.screen.blit(crt, (450, s + 35)) loader.screen.blit(cdt, (450, s + 90)) except Exception as err: print('save read err') print(err) s = s + 200 #执行读档逻辑 #先提示 loader.screen.blit(loader.CHECK_LOAD, (230, 200)) # 确认按钮 loader.screen.blit(loader.SURE, (300, 370)) # 取消按钮 loader.screen.blit(loader.SURENO, (460, 370)) #点击确认 if x >= 300 and x <= 380 and y >= 370 and y <= 410: loader.screen.blit(loader.SELECT_SAVE, (300, 370)) if is_mouse_down == True: try: cslot = player_runtime.INFO['cslot'] + 1 cslot_path = 'save/slot' + str(cslot) + '.pkl' res = saver.load(cslot_path) if res == False: print('存档异常') else: player_runtime.INFO = res player_runtime.INFO['checksover'] = False player_runtime.INFO['saving'] = False loader.curs_code = game_page.SCODE except Exception as err: print('load error') print(err) elif x >= 460 and x <= 540 and y >= 370 and y <= 410: loader.screen.blit(loader.SELECT_SAVE, (460, 370)) if is_mouse_down == True: #取消的话就返回到界面浏览状态就行了 player_runtime.INFO['checksover'] = False else: #执行选择存档逻辑 if a1: loader.screen.blit(loader.SAVE_BOARD_LOTA, (0, 80)) if is_mouse_down == True and not player_runtime.SDATA[0][ 'img_path'] == '': player_runtime.INFO['checksover'] = True player_runtime.INFO['cslot'] = 0 else: loader.screen.blit(loader.SAVE_BOARD_LOT, (0, 80)) if a2: loader.screen.blit(loader.SAVE_BOARD_LOTA, (0, 280)) if is_mouse_down == True and not player_runtime.SDATA[1][ 'img_path'] == '': player_runtime.INFO['checksover'] = True player_runtime.INFO['cslot'] = 1 else: loader.screen.blit(loader.SAVE_BOARD_LOT, (0, 280)) if a3: loader.screen.blit(loader.SAVE_BOARD_LOTA, (0, 480)) if is_mouse_down == True and not player_runtime.SDATA[2][ 'img_path'] == '': player_runtime.INFO['checksover'] = True player_runtime.INFO['cslot'] = 2 else: loader.screen.blit(loader.SAVE_BOARD_LOT, (0, 480)) #绘制存档截图 s = 90 for sd in player_runtime.SDATA: try: img_url = sd['img_path'] if not img_url == '': img_res = sd['img_res'] cr = sd['round'] cd = sd['date'] loader.screen.blit(img_res, (15, s)) crt = loader.BAIKE_FONT.render(cr, True, color_rgb.WHITE, None) cdt = loader.BAIKE_FONT.render(cd, True, color_rgb.WHITE, None) loader.screen.blit(crt, (450, s + 35)) loader.screen.blit(cdt, (450, s + 90)) except Exception as err: print('save read err') print(err) s = s + 200
def visualize(self): z = tf.placeholder(tf.float32, [self.model.batch_size, self.model.z_dim], name='z') cond = tf.placeholder(tf.float32, [self.model.batch_size] + [self.model.embed_dim], name='cond') gen_stagei, _, _ = self.model.stagei.generator(z, cond, is_training=False) gen, _, _ = self.model.generator(gen_stagei, cond, is_training=False) gen_no_noise, _, _ = self.model.generator(gen_stagei, cond, is_training=False, reuse=True, cond_noise=False) saver = tf.train.Saver(tf.global_variables('g_net')) could_load, _ = load(saver, self.sess, self.model.stagei.cfg.CHECKPOINT_DIR) if could_load: print(" [*] Load SUCCESS") else: print(" [!] Load failed...") raise LookupError('Could not load any checkpoints for stage I') saver = tf.train.Saver(tf.global_variables('stageII_g_net')) could_load, _ = load(saver, self.sess, self.config.CHECKPOINT_DIR) if could_load: print(" [*] Load SUCCESS") else: print(" [!] Load failed...") raise LookupError('Could not load any checkpoints for stage II') dataset_pos = np.random.randint(0, self.dataset.test.num_examples) for idx in range(0): dataset_pos = np.random.randint(0, self.dataset.test.num_examples) dataset_pos2 = np.random.randint(0, self.dataset.test.num_examples) # Interpolation in z space: # --------------------------------------------------------------------------------------------------------- _, cond, _, captions = self.dataset.test.next_batch_test(1, dataset_pos, 1) cond = np.squeeze(cond, axis=0) caption = captions[0][0] samples = gen_noise_interp_img(self.sess, gen_no_noise, cond, self.model.z_dim, self.model.batch_size) save_cap_batch(samples, caption, '{}/{}_visual/z_interp/z_interp{}.png'.format(self.samples_dir, self.dataset.name, idx)) # Interpolation in embedding space: # --------------------------------------------------------------------------------------------------------- _, cond1, _, caps1 = self.dataset.test.next_batch_test(1, dataset_pos, 1) _, cond2, _, caps2 = self.dataset.test.next_batch_test(1, dataset_pos2, 1) cond1 = np.squeeze(cond1, axis=0) cond2 = np.squeeze(cond2, axis=0) cap1, cap2 = caps1[0][0], caps2[0][0] samples = gen_cond_interp_img(self.sess, gen_no_noise, cond1, cond2, self.model.z_dim, self.model.batch_size) save_interp_cap_batch(samples, cap1, cap2, '{}/{}_visual/cond_interp/cond_interp{}.png'.format(self.samples_dir, self.dataset.name, idx)) # make_gif(samples, '{}/{}_visual/cond_interp/gifs/cond_interp{}.gif'.format(self.samples_dir, # self.dataset.name, # idx), duration=4) # Generate captioned image # --------------------------------------------------------------------------------------------------------- _, conditions, _, captions = self.dataset.test.next_batch_test(1, dataset_pos, 1) conditions = np.squeeze(conditions, axis=0) caption = captions[0][0] samples = gen_captioned_img(self.sess, gen, conditions, self.model.z_dim, self.model.batch_size) save_cap_batch(samples, caption, '{}/{}_visual/cap/cap{}.png'.format(self.samples_dir, self.dataset.name, idx)) # Generate Stage I and Stage II images # --------------------------------------------------------------------------------------------------------- _, cond, _, captions = self.dataset.test.next_batch_test(self.model.batch_size, dataset_pos, 1) cond = np.squeeze(cond, axis=0) samples = gen_multiple_stage_img(self.sess, [gen_stagei, gen], cond, self.model.z_dim, self.model.batch_size, size=128) text = "Stage I and Stage II" save_cap_batch(samples, text, '{}/{}_visual/stages/stage{}.png'.format(self.samples_dir, self.dataset.name, idx)) special_flowers = [1126, 908, 398] special_birds = [12, 908, 1005] for idx, special_pos in enumerate(special_birds): print(special_pos) # Generate specific image # --------------------------------------------------------------------------------------------------------- _, conditions, _, captions = self.dataset.test.next_batch_test(1, special_pos, 1) conditions = np.squeeze(conditions, axis=0) caption = captions[0][0] samples = gen_captioned_img(self.sess, gen, conditions, self.model.z_dim, self.model.batch_size) save_cap_batch(samples, caption, '{}/{}_visual/special_cap/cap{}.png'.format(self.samples_dir, self.dataset.name, idx))
def train(self): self.define_model() self.define_summaries() start_time = time.time() self.saver = tf.train.Saver( max_to_keep=self.cfg.TRAIN.CHECKPOINTS_TO_KEEP) if self.cfg.TRAIN.RESTORE_PRETRAIN: pretrain_saver = tf.train.Saver(self.pretrained_to_restore) # Load the pre-trained layer pretrain_saver.restore(self.sess, self.cfg.TRAIN.PRETRAINED_CHECKPOINT_DIR) # Initialise the not restored layers and the optimizer variables self.sess.run( tf.variables_initializer(self.not_to_restore + self.opt_vars)) start_point = 0 else: could_load, checkpoint_counter = load(self.saver, self.sess, self.cfg.CHECKPOINT_DIR) if could_load: start_point = checkpoint_counter print(" [*] Load SUCCESS") else: print(" [!] Load failed...") raise RuntimeError( 'Failed to restore the complete Inception model') sys.stdout.flush() batch_size = self.cfg.TRAIN.BATCH_SIZE for idx in range(start_point + 1, self.cfg.TRAIN.MAX_STEPS): epoch_size = self.dataset.test.num_examples // batch_size epoch = idx // epoch_size images, _, _, _, labels = self.dataset.test.next_batch(batch_size, labels=True) # Bring the labels in a continuous range: [0, num_classes) new_labels = [] for label in labels: new_labels.append(self.class_to_idx[label]) assert (np.min(images) >= -1.) assert (np.max(images) <= 1.) assert (np.min(new_labels) >= 0) assert (np.max(new_labels) < 50) # 20 for flowers, 50 for birds feed_dict = { self.x: images, self.labels: new_labels, } _, err = self.sess.run([self.opt_step, self.loss], feed_dict=feed_dict) summary_period = self.cfg.TRAIN.SUMMARY_PERIOD if np.mod(idx, summary_period) == 0: summary_str, pred = self.sess.run([self.summary_op, self.pred], feed_dict=feed_dict) self.writer.add_summary(summary_str, idx) print("Epoch: [%2d] [%4d] time: %4.4f, loss: %.8f" % (epoch, idx, time.time() - start_time, err)) if np.mod(idx, 200) == 0: save(self.saver, self.sess, self.cfg.CHECKPOINT_DIR, idx) sys.stdout.flush()
def train(self): self.define_losses() self.define_summaries() sample_z = np.random.normal(0, 1, (self.model.sample_num, self.model.z_dim)) _, sample_embed, _, captions = self.dataset.test.next_batch_test( self.model.sample_num, randint(0, self.dataset.test.num_examples), 1) sample_embed = np.squeeze(sample_embed, axis=0) print(sample_embed.shape) # Display the captions of the sampled images print('\nCaptions of the sampled images:') for caption_idx, caption_batch in enumerate(captions): print('{}: {}'.format(caption_idx + 1, caption_batch[0])) print() counter = 1 start_time = time.time() # Try to load the parameters of the stage II networks tf.global_variables_initializer().run() could_load, checkpoint_counter = load(self.stageii_saver, self.sess, self.cfg.CHECKPOINT_DIR) if could_load: counter = checkpoint_counter print(" [*] Load SUCCESS: Stage II networks are loaded.") else: print(" [!] Load failed for stage II networks...") could_load, checkpoint_counter = load(self.stagei_g_saver, self.sess, self.cfg_stage_i.CHECKPOINT_DIR) if could_load: counter = checkpoint_counter print(" [*] Load SUCCESS: Stage I generator is loaded") else: print( " [!] WARNING!!! Failed to load the parameters for stage I generator..." ) for epoch in range(self.cfg.TRAIN.EPOCH): # Updates per epoch are given by the training data size / batch size updates_per_epoch = self.dataset.train.num_examples // self.model.batch_size for idx in range(0, updates_per_epoch): images, wrong_images, embed, _, _ = self.dataset.train.next_batch( self.model.batch_size, 4) batch_z = np.random.normal( 0, 1, (self.model.batch_size, self.model.z_dim)) # Update D network _, err_d_real_match, err_d_real_mismatch, err_d_fake, err_d, summary_str = self.sess.run( [ self.D_optim, self.D_real_match_loss, self.D_real_mismatch_loss, self.D_synthetic_loss, self.D_loss, self.D_merged_summ ], feed_dict={ self.model.inputs: images, self.model.wrong_inputs: wrong_images, self.model.embed_inputs: embed, self.model.z: batch_z }) self.writer.add_summary(summary_str, counter) # Update G network _, err_g, summary_str = self.sess.run( [self.G_optim, self.G_loss, self.G_merged_summ], feed_dict={ self.model.z: batch_z, self.model.embed_inputs: embed }) self.writer.add_summary(summary_str, counter) counter += 1 print( "Epoch: [%2d] [%4d/%4d] time: %4.4f, d_loss: %.8f, g_loss: %.8f" % (epoch, idx, updates_per_epoch, time.time() - start_time, err_d, err_g)) if np.mod(counter, 100) == 0: try: samples = self.sess.run(self.model.sampler, feed_dict={ self.model.z_sample: sample_z, self.model.embed_sample: sample_embed, }) save_images( samples, image_manifold_size(samples.shape[0]), '{}train_{:02d}_{:04d}.png'.format( self.cfg.SAMPLE_DIR, epoch, idx)) print("[Sample] d_loss: %.8f, g_loss: %.8f" % (err_d, err_g)) # Display the captions of the sampled images print('\nCaptions of the sampled images:') for caption_idx, caption_batch in enumerate(captions): print('{}: {}'.format(caption_idx + 1, caption_batch[0])) print() except Exception as e: print("Failed to generate sample image") print(type(e)) print(e.args) print(e) if np.mod(counter, 500) == 2: save(self.stageii_saver, self.sess, self.cfg.CHECKPOINT_DIR, counter)
def train(self): config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: summary_writer = tf.summary.FileWriter(self.log_dir, sess.graph) start_point = 0 if self.stage != 1: if self.trans: could_load, _ = load(self.restore, sess, self.check_dir_read) if not could_load: raise RuntimeError('Could not load previous stage during transition') else: could_load, _ = load(self.saver, sess, self.check_dir_read) if not could_load: raise RuntimeError('Could not load current stage') # variables to init vars_to_init = initialize_uninitialized(sess) sess.run(tf.variables_initializer(vars_to_init)) sample_z = np.random.normal(0, 1, (self.sample_num, self.z_dim)) _, sample_cond, _, captions = self.dataset.test.next_batch_test(self.sample_num, 0, 1) sample_cond = np.squeeze(sample_cond, axis=0) print('Conditionals sampler shape: {}'.format(sample_cond.shape)) save_captions(self.sample_path, captions) start_time = time.time() for idx in range(start_point + 1, self.steps): if self.trans: # Reduce the learning rate during the transition period and slowly increase it p = idx / self.steps self.lr_inp = self.lr # * np.exp(-2 * np.square(1 - p)) epoch_size = self.dataset.train.num_examples // self.batch_size epoch = idx // epoch_size images, wrong_images, embed, _, _ = self.dataset.train.next_batch(self.batch_size, 4, wrong_img=True, embeddings=True) batch_z = np.random.normal(0, 1, (self.batch_size, self.z_dim)) eps = np.random.uniform(0., 1., size=(self.batch_size, 1, 1, 1)) feed_dict = { self.x: images, self.learning_rate: self.lr_inp, self.x_mismatch: wrong_images, self.cond: embed, self.z: batch_z, self.epsilon: eps, self.z_sample: sample_z, self.cond_sample: sample_cond, self.iter: idx, } _, err_d = sess.run([self.D_optim, self.D_loss], feed_dict=feed_dict) _, err_g = sess.run([self.G_optim, self.G_loss], feed_dict=feed_dict) if np.mod(idx, 20) == 0: summary_str = sess.run(self.summary_op, feed_dict=feed_dict) summary_writer.add_summary(summary_str, idx) print("Epoch: [%2d] [%4d] time: %4.4f, d_loss: %.8f, g_loss: %.8f" % (epoch, idx, time.time() - start_time, err_d, err_g)) if np.mod(idx, 2000) == 0: try: samples = sess.run(self.sampler, feed_dict={ self.z_sample: sample_z, self.cond_sample: sample_cond}) samples = np.clip(samples, -1., 1.) if self.out_size > 256: samples = samples[:4] save_images(samples, get_balanced_factorization(samples.shape[0]), '{}train_{:02d}_{:04d}.png'.format(self.sample_path, epoch, idx)) except Exception as e: print("Failed to generate sample image") print(type(e)) print(e.args) print(e) if np.mod(idx, 2000) == 0 or idx == self.steps - 1: save(self.saver, sess, self.check_dir_write, idx) sys.stdout.flush() tf.reset_default_graph()
def train(self): self.define_summaries() self.saver = tf.train.Saver(max_to_keep=self.cfg.TRAIN.CHECKPOINTS_TO_KEEP) sample_z = np.random.normal(0, 1, (self.model.sample_num, self.model.z_dim)) _, sample_cond, _, captions = self.dataset.test.next_batch_test(self.model.sample_num, 0, 1) # _, sample_cond, _, captions = self.dataset.test.next_batch_test(self.model.sample_num, 1, 1) sample_cond = np.squeeze(sample_cond, axis=0) print('Conditionals sampler shape: {}'.format(sample_cond.shape)) save_captions(self.cfg.SAMPLE_DIR, captions) start_time = time.time() tf.global_variables_initializer().run() could_load, checkpoint_counter = load(self.saver, self.sess, self.cfg.CHECKPOINT_DIR) if could_load: start_point = checkpoint_counter print(" [*] Load SUCCESS") else: start_point = 0 print(" [!] Load failed...") sys.stdout.flush() for idx in range(start_point + 1, self.cfg.TRAIN.MAX_STEPS): epoch_size = self.dataset.train.num_examples // self.model.batch_size epoch = idx // epoch_size images, wrong_images, embed, _, _ = self.dataset.train.next_batch(self.model.batch_size, 1, embeddings=True, wrong_img=True) batch_z = np.random.normal(0, 1, (self.model.batch_size, self.model.z_dim)) eps = np.random.uniform(0., 1., size=(self.model.batch_size, 1, 1, 1)) n_critic = self.cfg.TRAIN.N_CRITIC kiter = (idx // n_critic) // 10000 feed_dict = { self.model.learning_rate_d: self.lr_d * (0.95**kiter), self.model.learning_rate_g: self.lr_g * (0.95**kiter), self.model.x: images, self.model.x_mismatch: wrong_images, self.model.cond: embed, self.model.z: batch_z, self.model.epsilon: eps, self.model.z_sample: sample_z, self.model.cond_sample: sample_cond, self.model.iter: idx, } _, _, err_d = self.sess.run([self.model.D_optim, self.model.kt_optim, self.model.D_loss], feed_dict=feed_dict) if idx % n_critic == 0: _, err_g = self.sess.run([self.model.G_optim, self.model.G_loss], feed_dict=feed_dict) summary_period = self.cfg.TRAIN.SUMMARY_PERIOD if np.mod(idx, summary_period) == 0: summary_str = self.sess.run(self.summary_op, feed_dict=feed_dict) self.writer.add_summary(summary_str, idx) if np.mod(idx, self.cfg.TRAIN.SAMPLE_PERIOD) == 0: try: samples = self.sess.run(self.model.sampler, feed_dict={ self.model.z_sample: sample_z, self.model.cond_sample: sample_cond, }) save_images(samples, get_balanced_factorization(samples.shape[0]), '{}train_{:02d}_{:04d}.png'.format(self.cfg.SAMPLE_DIR, epoch, idx)) except Exception as e: print("Failed to generate sample image") print(type(e)) print(e.args) print(e) if np.mod(idx, 500) == 2: save(self.saver, self.sess, self.cfg.CHECKPOINT_DIR, idx) sys.stdout.flush()
sample_path=None, log_dir=None, stage=stage[i], trans=False, build_model=False) cond = tf.placeholder(tf.float32, [None, 1024], name='cond') z = tf.placeholder(tf.float32, [None, z_dim], name='z') gen_op, _, _ = pggan.generator(z, cond, stages=stage[i], t=False) config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: saver = tf.train.Saver(tf.global_variables('g_net')) could_load = load(saver, sess, pggan_checkpoint_dir_read) if not could_load: raise RuntimeError('Could not load stage %d' % stage[i]) samples = sess.run(gen_op, feed_dict={ 'z:0': z_sample, 'cond:0': conditions }) samples = np.clip(samples, -1., 1.) all_samples.append(samples) tf.reset_default_graph() all_samples = gen_pggan_sample(all_samples, 128, interp='nearest')
def evaluate_inception(self): incep_batch_size = self.cfg.EVAL.INCEP_BATCH_SIZE logits, _ = load_inception_inference(self.sess, self.cfg.EVAL.NUM_CLASSES, incep_batch_size, self.cfg.EVAL.INCEP_CHECKPOINT_DIR) pred_op = tf.nn.softmax(logits) z = tf.placeholder(tf.float32, [self.bs, self.model.stagei.z_dim], name='z') cond = tf.placeholder(tf.float32, [self.bs] + [self.model.stagei.embed_dim], name='cond') stagei_gen, _, _ = self.model.stagei.generator(z, cond, reuse=False, is_training=False) eval_gen, _, _ = self.model.generator(stagei_gen, cond, reuse=False, is_training=False) self.Retrieval.eval(self.bs) saver = tf.train.Saver(tf.global_variables('g_net')+tf.global_variables('vf_')+tf.global_variables('sf_')+ tf.global_variables('att')) could_load, _ = load(saver, self.sess, self.model.stagei.cfg.CHECKPOINT_DIR) if could_load: print(" [*] Load SUCCESS") else: print(" [!] Load failed...") raise RuntimeError('Could not load the checkpoints of stage I') saver = tf.train.Saver(tf.global_variables('stageII_g_net')) could_load, _ = load(saver, self.sess, self.cfg.CHECKPOINT_DIR) if could_load: print(" [*] Load SUCCESS") else: print(" [!] Load failed...") raise RuntimeError('Could not load the checkpoints of stage II') print('Generating batches...') size = self.cfg.EVAL.SIZE n_batches = size // self.bs all_preds = [] for i in range(n_batches): print("\rGenerating batch %d/%d" % (i + 1, n_batches), end="", flush=True) sample_z = np.random.normal(0, 1, size=(self.bs, self.model.z_dim)) # _, _, embed, _, _ = self.dataset.test.next_batch(self.bs, 4, embeddings=True) _, _, embed, _, _ = self.dataset.test.next_batch(self.bs, 1, embeddings=True) im_feats, sent_feats, labels = self.test_data_loader.get_batch(i, self.bs, phase = 'incep') # Generate a batch and scale it up for inception sent_emb = self.sess.run(self.Retrieval.sent_embed_tensor, feed_dict={ self.Retrieval.image_placeholder_test: im_feats, self.Retrieval.sent_placeholder_test: sent_feats, }) gen_batch = self.sess.run(eval_gen, feed_dict={z: sample_z, cond: sent_emb}) samples = denormalize_images(gen_batch) incep_samples = np.empty((self.bs, 299, 299, 3)) for sample_idx in range(self.bs): incep_samples[sample_idx] = prep_incep_img(samples[sample_idx]) # Run prediction for current batch pred = self.sess.run(pred_op, feed_dict={'inputs:0': incep_samples}) all_preds.append(pred) # Get rid of the first dimension all_preds = np.concatenate(all_preds, 0) print('\nComputing inception score...') mean, std = inception_score.get_inception_from_predictions(all_preds, 10) print('Inception Score | mean:', "%.2f" % mean, 'std:', "%.2f" % std)
def generate_sequences(fuzzing_requests, checkers, fuzzing_jobs=1): """ Implements core restler algorithm. @param fuzzing_requests: The collection of requests that will be fuzzed @type fuzzing_requests: FuzzingRequestCollection @param checkers: The list of checkers to apply @type checkers: list[Checker] @param fuzzing_jobs: Optional number of fuzzing jobs for parallel fuzzing. Default value passed is one (sequential fuzzing). @type fuzzing_jobs: Int @return: None @rtype : None """ if not fuzzing_requests.size: return logger.create_network_log(logger.LOG_TYPE_TESTING) fuzzing_mode = Settings().fuzzing_mode max_len = Settings().max_sequence_length if fuzzing_mode == 'directed-smoke-test': return generate_sequences_directed_smoketest(fuzzing_requests, checkers) if fuzzing_jobs > 1: render = render_parallel global_lock = multiprocessing.Lock() fuzzing_pool = ThreadPool(fuzzing_jobs) else: global_lock = None fuzzing_pool = None render = render_sequential should_stop = False timeout_reached = False seq_collection_exhausted = False num_total_sequences = 0 while not should_stop: seq_collection = [sequences.Sequence()] # Only for bfs: If any checkpoint file is available, load state of # latest generation. Note that it only makes sense to use checkpoints # for the bfs exploration method, since it is the only systemic and # exhaustive method. min_len = 0 if fuzzing_mode == 'bfs': req_collection = GrammarRequestCollection() monitor = Monitor() req_collection, seq_collection, fuzzing_requests, monitor, min_len =\ saver.load(req_collection, seq_collection, fuzzing_requests, monitor) requests.GlobalRequestCollection.Instance( )._req_collection = req_collection fuzzing_monitor.FuzzingMonitor.__instance = monitor # Repeat external loop only for random walk if fuzzing_mode != 'random-walk': should_stop = True # Initialize fuzzing schedule fuzzing_schedule = {} logger.write_to_main(f"Setting fuzzing schemes: {fuzzing_mode}") for length in range(min_len, max_len): fuzzing_schedule[length] = fuzzing_mode # print(" - {}: {}".format(length + 1, fuzzing_schedule[length])) # print general request-related stats logger.print_req_collection_stats( fuzzing_requests, GrammarRequestCollection().candidate_values_pool) generation = 0 for length in range(min_len, max_len): # we can set this without locking, since noone else writes (main # driver is single-threaded) and every potential worker will just # read-access this value. generation = length + 1 fuzzing_mode = fuzzing_schedule[length] # extend sequences with new request templates seq_collection = extend(seq_collection, fuzzing_requests, global_lock) print(f"{formatting.timestamp()}: Generation: {generation} ") logger.write_to_main( f"{formatting.timestamp()}: Generation: {generation} / " f"Sequences Collection Size: {len(seq_collection)} " f"(After {fuzzing_schedule[length]} Extend)") # render templates try: seq_collection_exhausted = False seq_collection = render(seq_collection, fuzzing_pool, checkers, generation, global_lock) except TimeOutException: logger.write_to_main("Timed out...") timeout_reached = True seq_collection_exhausted = True # Increase fuzzing generation after timeout because the code # that does it would have never been reached. This is done so # the previous generation's test summary is logged correctly. Monitor().current_fuzzing_generation += 1 except ExhaustSeqCollectionException: logger.write_to_main("Exhausted collection...") seq_collection = [] seq_collection_exhausted = True logger.write_to_main( f"{formatting.timestamp()}: Generation: {generation} / " f"Sequences Collection Size: {len(seq_collection)} " f"(After {fuzzing_schedule[length]} Render)") # saving latest state saver.save(GrammarRequestCollection(), seq_collection, fuzzing_requests, Monitor(), generation) # Print stats for iteration of the current generation logger.print_generation_stats(GrammarRequestCollection(), Monitor(), global_lock) num_total_sequences += len(seq_collection) logger.print_request_rendering_stats( GrammarRequestCollection().candidate_values_pool, fuzzing_requests, Monitor(), Monitor().num_fully_rendered_requests( fuzzing_requests.all_requests), generation, global_lock) if timeout_reached or seq_collection_exhausted: if timeout_reached: should_stop = True break logger.write_to_main("--\n") if fuzzing_pool is not None: fuzzing_pool.close() fuzzing_pool.join() return num_total_sequences
def visualize(self): z = tf.compat.v1.placeholder(tf.float32, [None, self.model.z_dim], name='z') cond = tf.compat.v1.placeholder(tf.float32, [None] + [self.model.embed_dim], name='cond') gen = self.model.generator(z, cond, is_training=False) saver = tf.compat.v1.train.Saver( tf.compat.v1.global_variables('g_net')) could_load, _ = load(saver, self.sess, self.config.CHECKPOINT_DIR) if could_load: print(" [*] Load SUCCESS") else: print(" [!] Load failed...") raise LookupError('Could not load any checkpoints') dataset_pos = np.random.randint(0, self.dataset.test.num_examples) for idx in range(0): dataset_pos = np.random.randint(0, self.dataset.test.num_examples) # Interpolation in z space: # --------------------------------------------------------------------------------------------------------- _, cond, _, captions = self.dataset.test.next_batch_test( 1, dataset_pos, 1) cond = np.squeeze(cond, axis=0) caption = captions[0][0] samples = gen_noise_interp_img(self.sess, gen, cond, self.model.z_dim, self.model.batch_size) save_cap_batch( samples, caption, '{}/{}_visual/z_interp/z_interp{}.png'.format( self.samples_dir, self.dataset.name, idx)) # Interpolation in embedding space: # --------------------------------------------------------------------------------------------------------- _, cond, _, caps = self.dataset.test.next_batch_test( 2, dataset_pos, 1) cond = np.squeeze(cond, axis=0) cond1, cond2 = cond[0], cond[1] cap1, cap2 = caps[0][0], caps[1][0] samples = gen_cond_interp_img(self.sess, gen, cond1, cond2, self.model.z_dim, self.model.batch_size) save_interp_cap_batch( samples, cap1, cap2, '{}/{}_visual/cond_interp/cond_interp{}.png'.format( self.samples_dir, self.dataset.name, idx)) # make_gif(samples, '{}/{}_visual/cond_interp/gifs/cond_interp{}.gif'.format(self.samples_dir, # self.dataset.name, # idx), duration=4) # Generate captioned image # --------------------------------------------------------------------------------------------------------- _, conditions, _, captions = self.dataset.test.next_batch_test( 1, dataset_pos, 1) conditions = np.squeeze(conditions, axis=0) caption = captions[0][0] samples = gen_captioned_img(self.sess, gen, conditions, self.model.z_dim, self.model.batch_size) save_cap_batch( samples, caption, '{}/{}_visual/cap/cap{}.png'.format(self.samples_dir, self.dataset.name, idx)) for idx, special_pos in enumerate([1112, 900, 398]): print(special_pos) # Generate specific image # --------------------------------------------------------------------------------------------------------- _, conditions, _, captions = self.dataset.test.next_batch_test( 1, special_pos, 1) conditions = np.squeeze(conditions, axis=0) caption = captions[0][0] samples = gen_captioned_img(self.sess, gen, conditions, self.model.z_dim, self.model.batch_size) save_cap_batch( samples, caption, '{}/{}_visual/special_cap/cap{}.png'.format( self.samples_dir, self.dataset.name, idx)) # Generate some images and their closest neighbours # --------------------------------------------------------------------------------------------------------- _, conditions, _, _ = self.dataset.test.next_batch_test( self.model.batch_size, dataset_pos, 1) conditions = np.squeeze(conditions) samples, neighbours = gen_closest_neighbour_img( self.sess, gen, conditions, self.model.z_dim, self.model.batch_size, self.dataset) batch = np.concatenate([samples, neighbours]) text = 'Generated images (first row) and their closest neighbours (second row)' save_cap_batch( batch, text, '{}/{}_visual/neighb/neighb.png'.format(self.samples_dir, self.dataset.name))
def train(self): self.define_losses() self.define_summaries() sample_z = np.random.normal(0, 1, (self.model.sample_num, self.model.z_dim)) _, sample_embed, _, captions = self.dataset.test.next_batch_test( self.model.sample_num, 0, 1) sample_embed = np.squeeze(sample_embed, axis=0) print(sample_embed.shape) save_captions(self.cfg.SAMPLE_DIR, captions) counter = 1 start_time = time.time() could_load, checkpoint_counter = load(self.saver, self.sess, self.cfg.CHECKPOINT_DIR) if could_load: counter = checkpoint_counter print(" [*] Load SUCCESS") else: print(" [!] Load failed...") initialize_uninitialized(self.sess) # Updates per epoch are given by the training data size / batch size updates_per_epoch = self.dataset.train.num_examples // self.model.batch_size epoch_start = counter // updates_per_epoch for epoch in range(epoch_start, self.cfg.TRAIN.EPOCH): cen_epoch = epoch // 100 for idx in range(0, updates_per_epoch): images, wrong_images, embed, _, _ = self.dataset.train.next_batch( self.model.batch_size, 4, embeddings=True, wrong_img=True) batch_z = np.random.normal( 0, 1, (self.model.batch_size, self.model.z_dim)) feed_dict = { self.learning_rate: self.lr * (0.5**cen_epoch), self.model.inputs: images, self.model.wrong_inputs: wrong_images, self.model.embed_inputs: embed, self.model.z: batch_z, } # Update D network _, err_d, summary_str = self.sess.run( [self.D_optim, self.D_loss, self.D_merged_summ], feed_dict=feed_dict) self.writer.add_summary(summary_str, counter) # Update G network _, err_g, summary_str = self.sess.run( [self.G_optim, self.G_loss, self.G_merged_summ], feed_dict=feed_dict) self.writer.add_summary(summary_str, counter) counter += 1 print( "Epoch: [%2d] [%4d/%4d] time: %4.4f, d_loss: %.8f, g_loss: %.8f" % (epoch, idx, updates_per_epoch, time.time() - start_time, err_d, err_g)) if np.mod(counter, 500) == 0: try: samples = self.sess.run(self.model.sampler, feed_dict={ self.model.z_sample: sample_z, self.model.embed_sample: sample_embed, }) save_images( samples, get_balanced_factorization(samples.shape[0]), '{}train_{:02d}_{:04d}.png'.format( self.cfg.SAMPLE_DIR, epoch, idx)) except Exception as e: print("Failed to generate sample image") print(type(e)) print(e.args) print(e) if np.mod(counter, 500) == 0: save(self.saver, self.sess, self.cfg.CHECKPOINT_DIR, counter)
def train(self): self.define_losses() self.define_summaries() sample_z = np.random.normal(0, 1, (self.model.sample_num, self.model.z_dim)) _, sample_embed, _, captions = self.dataset.test.next_batch_test(self.model.sample_num, 0, 1) im_feats_test, sent_feats_test, labels_test = self.test_data_loader.get_batch(0,self.cfg.RETRIEVAL.SAMPLE_NUM,\ image_aug = self.cfg.RETRIEVAL.IMAGE_AUG, phase='test') sample_embed = np.squeeze(sample_embed, axis=0) print(sample_embed.shape) save_captions(self.cfg.SAMPLE_DIR, captions) counter = 1 start_time = time.time() could_load, checkpoint_counter = load(self.stageii_saver, self.sess, self.cfg.CHECKPOINT_DIR) if could_load: counter = checkpoint_counter print(" [*] Load SUCCESS: Stage II networks are loaded.") else: print(" [!] Load failed for stage II networks...") could_load, checkpoint_counter = load(self.stagei_g_saver, self.sess, self.cfg_stage_i.CHECKPOINT_DIR) if could_load: print(" [*] Load SUCCESS: Stage I generator is loaded") else: print(" [!] WARNING!!! Failed to load the parameters for stage I generator...") initialize_uninitialized(self.sess) # Updates per epoch are given by the training data size / batch size updates_per_epoch = self.dataset.train.num_examples // self.model.batch_size epoch_start = counter // updates_per_epoch for epoch in range(epoch_start, self.cfg.TRAIN.EPOCH): cen_epoch = epoch // 100 for idx in range(0, updates_per_epoch): images, wrong_images, embed, _, _ = self.dataset.train.next_batch(self.model.batch_size, 1, embeddings=True, wrong_img=True) batch_z = np.random.normal(0, 1, (self.model.batch_size, self.model.z_dim)) # Retrieval data loader if idx % updates_per_epoch == 0: self.R_loader.shuffle_inds() im_feats, sent_feats, labels = self.R_loader.get_batch(idx % updates_per_epoch,\ self.cfg.RETRIEVAL.BATCH_SIZE, image_aug = self.cfg.RETRIEVAL.IMAGE_AUG) feed_dict = { self.learning_rate: self.lr * (0.5**cen_epoch), self.model.inputs: images, self.model.wrong_inputs: wrong_images, # self.model.embed_inputs: embed, # self.model.embed_inputs: self.txt_emb, self.model.z: batch_z, self.Retrieval.image_placeholder : im_feats, self.Retrieval.sent_placeholder : sent_feats, self.Retrieval.label_placeholder : labels } # Update D network _, err_d, summary_str = self.sess.run([self.D_optim, self.D_loss, self.D_merged_summ], feed_dict=feed_dict) self.writer.add_summary(summary_str, counter) # Update G network _, err_g, summary_str = self.sess.run([self.G_optim, self.G_loss, self.G_merged_summ], feed_dict=feed_dict) self.writer.add_summary(summary_str, counter) # Update R network _, err_r, summary_str = self.sess.run([self.R_optim, self.R_loss, self.R_loss_summ], feed_dict=feed_dict) self.writer.add_summary(summary_str, counter) counter += 1 print("Epoch: [%2d] [%4d/%4d] time: %4.4f, d_loss: %.8f, g_loss: %.8f, r_loss: %.8f" % (epoch, idx, updates_per_epoch, time.time() - start_time, err_d, err_g, err_r)) if np.mod(counter, 1000) == 0: try: # pdb.set_trace() self.Retrieval.eval() sent_emb = self.sess.run(self.Retrieval.sent_embed_tensor, feed_dict={ self.Retrieval.image_placeholder_test: im_feats_test, self.Retrieval.sent_placeholder_test: sent_feats_test, }) self.model.eval(sent_emb) samples = self.sess.run(self.model.sampler, feed_dict={ self.model.z_sample: sample_z, # self.model.embed_sample: sample_embed, self.model.embed_sample: sent_emb, }) save_images(samples, get_balanced_factorization(samples.shape[0]), '{}train_{:02d}_{:04d}.png'.format(self.cfg.SAMPLE_DIR, epoch, idx)) except Exception as e: print("Failed to generate sample image") print(type(e)) print(e.args) print(e) if np.mod(counter, 500) == 2: save(self.stageii_saver, self.sess, self.cfg.CHECKPOINT_DIR, counter) if np.mod(epoch, 50) == 0 and epoch!=0: self.ret_eval(epoch)