def scoring_function(solution_file, predict_file): np.random.seed(1337) score = 0. solution = np.load(solution_file, allow_pickle=True) predict = np.load(predict_file, allow_pickle=True) EnergyDeposit_sol = solution['EnergyDeposit'] ParticleMomentum_sol = solution['ParticleMomentum'] ParticlePoint_sol = solution['ParticlePoint'] EnergyDeposit_pred = predict['EnergyDeposit'] ParticleMomentum_pred = solution['ParticleMomentum'] ParticlePoint_pred = solution['ParticlePoint'] embedder = load_embedder('./embedder.tp') EnergyDeposit_sol_emb = embedder.get_encoding( torch.tensor(EnergyDeposit_sol).float().view(-1, 1, 30, 30)).detach().numpy() EnergyDeposit_pred_emb = embedder.get_encoding( torch.tensor(EnergyDeposit_sol).float().view(-1, 1, 30, 30)).detach().numpy() precision, recall = compute_prd_from_embedding( EnergyDeposit_sol_emb.reshape(len(EnergyDeposit_sol), -1), EnergyDeposit_pred_emb.reshape(len(EnergyDeposit_sol), -1), num_clusters=100, num_runs=100) auc_img = auc(precision, recall) physical_metrics_sol = get_physical_stats(EnergyDeposit_sol, ParticleMomentum_sol, ParticlePoint_sol) physical_metrics_pred = get_physical_stats(EnergyDeposit_pred, ParticleMomentum_pred, ParticlePoint_pred) precision, recall = compute_prd_from_embedding(physical_metrics_sol, physical_metrics_pred, num_clusters=100, num_runs=100) auc_physical_metrics = auc(precision, recall) return min(auc_img, auc_physical_metrics)
def getPRD(): realdata = getRealData(10000) fakedata = getFakedata(10000) ref_emb = fid_tf.get_inception_activations(realdata) eval_emb = fid_tf.get_inception_activations(fakedata) prd_res = prd.compute_prd_from_embedding(eval_data=eval_emb, ref_data=ref_emb) print("PRD =", prd_res) return prd_res
def plot_prd_train(learner, index, training_batch): sample_num = 32 true_data = training_batch.detach().cpu().numpy() samples = torch.randn(sample_num, 3).cuda() reconst_x = learner.decoder(samples).detach().cpu().numpy() prd_data = prd.compute_prd_from_embedding( reconst_x, true_data.reshape(sample_num, -1)) prd.plot([prd_data], [str(index)], out_path='./result/prd_train_' + str(index) + '.png')
def plot_prd(learner, index, prd_train_tasks): train_task = prd_train_tasks.sample() true_data, label = train_task sample_num = 1000 true_data = true_data[index * sample_num:index * sample_num + sample_num] label = label[index * sample_num:index * sample_num + sample_num] samples = torch.randn(sample_num, 3).cuda() reconst_x = learner.decoder(samples).detach().cpu().numpy() print(label, index) print(reconst_x.shape, true_data.shape) prd_data = prd.compute_prd_from_embedding( reconst_x, true_data.reshape(sample_num, -1)) prd.plot([prd_data], [str(index)], out_path='./result/prd_' + str(index) + '.png')
def scoring_function(solution_file, predict_file): np.random.seed(1337) score = 0. solution = np.load(solution_file, allow_pickle=True) predict = np.load(predict_file, allow_pickle=True) EnergyDeposit_sol = solution['EnergyDeposit'] ParticleMomentum_sol = solution['ParticleMomentum'] ParticlePoint_sol = solution['ParticlePoint'] EnergyDeposit_pred = predict['EnergyDeposit'] precision, recall = compute_prd_from_embedding( EnergyDeposit_sol.reshape(len(EnergyDeposit_sol), -1), EnergyDeposit_pred.reshape(len(EnergyDeposit_sol), -1),, num_clusters=100, num_runs=100) auc_img = auc(precision, recall) physical_metrics_sol = get_physical_stats( EnergyDeposit_sol, ParticleMomentum_sol, ParticlePoint_sol) physical_metrics_pred = get_physical_stats( EnergyDeposit_pred, ParticleMomentum_sol, ParticlePoint_sol) precision, recall = compute_prd_from_embedding( physical_metrics_sol, physical_metrics_pred, num_clusters=100, num_runs=100) auc_physical_metrics = auc(precision, recall) return min(auc_img, auc_physical_metrics)
def cluster_main(reference_dir, eval_dirs): if args.verbose: print('computing inception embeddings for ' + reference_dir) real_embeddings = load_or_generate_inception_embedding( reference_dir, args.cache_dir, args.use_raw_images) prd_data = [] for directory in eval_dirs: if args.verbose: print('computing inception embeddings for ' + directory) eval_embeddings = load_or_generate_inception_embedding( directory, args.cache_dir, args.use_raw_images) if args.verbose: print('computing PRD') prd_data.append(prd.compute_prd_from_embedding( eval_data=eval_embeddings, ref_data=real_embeddings, num_clusters=args.num_clusters, num_angles=args.num_angles, num_runs=args.num_runs)) return prd_data
def compute_prd(reference_dir, eval_dirs, inception_path): real_embeddings = load_or_generate_inception_embedding( reference_dir, '/tmp/prd_cache/', inception_path) prd_data = [] for directory in eval_dirs: print('computing inception embeddings for ' + directory) eval_embeddings = load_or_generate_inception_embedding( directory, '/tmp/prd_cache/', inception_path) print('computing PRD') prd_data.append( prd.compute_prd_from_embedding(eval_data=eval_embeddings, ref_data=real_embeddings, num_clusters=20, num_angles=1001, num_runs=10)) f_beta_data = [ prd.prd_to_max_f_beta_pair(precision, recall, beta=8) for precision, recall in prd_data ] prd.plot(prd_data) print('F_8 F_1/8 model') for directory, f_beta in zip(eval_dirs, f_beta_data): print('%.3f %.3f %s' % (f_beta[0], f_beta[1], directory))
# Transform test samples and generated samples with it ref_np_original = test_dataset.get_subset(len(test_dataset), n_samples) ref_np = transformer.transform(ref_np_original) # Fully trained model model_np = sample_prior(vae_algorithm.model) # Load a chpt for comparisson vae_garbage = getattr(alg, vae_config['algorithm_type'])(vae_config['vae_opt']) chp1_path = 'models/{0}/vae_checkpoint{1}.pth'.format( args_opt.exp_vae, chpnt1) vae_garbage.load_checkpoint(chp1_path, eval=True) chpnt1_np = sample_prior(vae_garbage.model) # Init a new model vae_garbage2 = getattr(alg, vae_config['algorithm_type'])(vae_config['vae_opt']) chp2_path = 'models/{0}/vae_checkpoint{1}.pth'.format( args_opt.exp_vae, chpnt2) vae_garbage2.load_checkpoint(chp2_path, eval=True) chpnt2_np = sample_prior(vae_garbage2.model) # Compute prd prd_data_model = prd.compute_prd_from_embedding(model_np, ref_np) prd_data_chpnt1 = prd.compute_prd_from_embedding(chpnt1_np, ref_np) prd_data_chpnt2 = prd.compute_prd_from_embedding(chpnt2_np, ref_np) prd.plot([prd_data_model, prd_data_chpnt1, prd_data_chpnt2], [args_opt.exp_vae, 'chpt' + str(chpnt1), 'chpt' + str(chpnt2)], out_path='models/{0}/prd.png'.format(args_opt.exp_vae))
if args.verbose: print('computing inception embeddings for ' + reference_dir) real_embeddings = load_or_generate_inception_embedding( reference_dir, args.cache_dir, args.inception_path) prd_data = [] for directory in eval_dirs: if args.verbose: print('computing inception embeddings for ' + directory) eval_embeddings = load_or_generate_inception_embedding( directory, args.cache_dir, args.inception_path) if args.verbose: print('computing PRD') prd_data.append( prd.compute_prd_from_embedding(eval_data=eval_embeddings, ref_data=real_embeddings, num_clusters=args.num_clusters, num_angles=args.num_angles, num_runs=args.num_runs)) if args.verbose: print('plotting results') print() f_beta_data = [ prd.prd_to_max_f_beta_pair(precision, recall, beta=8) for precision, recall in prd_data ] print('F_8 F_1/8 model') for directory, f_beta in zip(eval_dirs, f_beta_data): print('%.3f %.3f %s' % (f_beta[0], f_beta[1], directory)) prd.plot(prd_data, labels=args.eval_labels, out_path=args.plot_path)
model.Dnet.eval() model.Qnet.eval() n_prd_samples = eval_config['n_prd_samples'] # Get the ground truth np array test_dataset = TrajDataset(path_to_data, device) ref_np = test_dataset.get_subset(len(test_dataset), n_prd_samples) # Get the sampled np array with torch.no_grad(): z_noise, con_noise = model.ginput_noise(n_prd_samples) eval_data = model.g_forward(z_noise, con_noise) eval_np = eval_data.cpu().numpy().reshape(n_prd_samples, -1) prd_data = prd.compute_prd_from_embedding(eval_np, ref_np) # Only evaluate the last model if not compute_chpnt_prds: # Compute prd prd.plot([prd_data], [args.config_name], out_path='models/{0}/prd.png'.format(args.config_name)) # Evaluate the intermediate checkponts else: # Load the chpt chpr_prd_list = [] for c in chnpt_list: model_chpt = models.InfoGAN(config_file) model_chpt.load_checkpoint( 'models/{0}/infogan_checkpoint{1}.pth'.format(args.config_name, c))
if args.verbose: print('computing inception embeddings for ' + reference_dir) real_embeddings = load_or_generate_inception_embedding( reference_dir, args.cache_dir, args.inception_path) prd_data = [] for directory in eval_dirs: if args.verbose: print('computing inception embeddings for ' + directory) eval_embeddings = load_or_generate_inception_embedding( directory, args.cache_dir, args.inception_path) if args.verbose: print('computing PRD') prd_data.append(prd.compute_prd_from_embedding( eval_data=eval_embeddings, ref_data=real_embeddings, num_clusters=args.num_clusters, num_angles=args.num_angles, num_runs=args.num_runs)) if args.verbose: print('plotting results') print() f_beta_data = [prd.prd_to_max_f_beta_pair(precision, recall, beta=8) for precision, recall in prd_data] print('F_8 F_1/8 model') for directory, f_beta in zip(eval_dirs, f_beta_data): print('%.3f %.3f %s' % (f_beta[0], f_beta[1], directory)) prd.plot(prd_data, labels=args.eval_labels, out_path=args.plot_path)