def main(): # load config file config_test = run_utils.load_config(TEST_CONFIG_NAME) # Find data_dir data_dir = python_utils.choose_first_existing_path(config_test["data_dir_candidates"]) if data_dir is None: print("ERROR: Data directory not found!") exit() else: print("Using data from {}".format(data_dir)) dataset_raw_dirpath = os.path.join(data_dir, config_test["dataset_raw_partial_dirpath"]) output_dir = config_test["align_dir"] + OUTPUT_DIRNAME_EXTENTION if not os.path.exists(output_dir): os.makedirs(output_dir) for images_info in config_test["images_info_list"]: for number in images_info["numbers"]: image_info = { "city": images_info["city"], "number": number, } test_image(RUNS_DIRPATH, dataset_raw_dirpath, image_info, config_test["disp_maps_dir"], config_test["disp_map_params"]["disp_map_count"], config_test["disp_map_params"]["disp_max_abs_value"], config_test["batch_size"], DS_FAC_LIST, RUN_NAME_LIST, config_test["model_disp_max_abs_value"], config_test["thresholds"], output_dir, config_test["output_shapefiles"])
def generate_test(config, params, split_name="train", seed=None, sobol_generator=None): # Find data_dir data_dirpath = python_utils.choose_first_existing_path( config["data_dir_candidates"]) if data_dirpath is None: print_utils.print_error("ERROR: Data directory not found!") exit() data_dirpath = os.path.expanduser(data_dirpath) print_utils.print_info("Using data from {}".format(data_dirpath)) root_dir = os.path.join(data_dirpath, config["data_root_partial_dirpath"]) alpha, x, density, gt, noise, curvature = generate_data( root_dir, params, split_name=split_name, seed=seed, sobol_generator=sobol_generator) noisy_gt = gt + noise import matplotlib.pyplot as plt f = plt.figure() f.set_tight_layout({"pad": .0}) ax = f.gca() # plt.scatter(alpha, noisy_gt, s=10) ax.plot(alpha, noisy_gt) ax.set_xlabel("alpha") ax.set_ylabel("y") # plt.title("Sinusoid, freq = {}".format(params["f"])) plt.show()
def main(): # load config file config = run_utils.load_config(CONFIG_NAME) config_test = run_utils.load_config(TEST_CONFIG_NAME) # Find data_dir data_dir = python_utils.choose_first_existing_path( config["data_dir_candidates"]) if data_dir is None: print("ERROR: Data directory not found!") exit() else: print("Using data from {}".format(data_dir)) dataset_raw_dirpath = os.path.join( data_dir, config_test["dataset_raw_partial_dirpath"]) if not os.path.exists(config_test["disp_maps_dir"]): os.makedirs(config_test["disp_maps_dir"]) for images_info in config_test["images_info_list"]: for number in images_info["numbers"]: image_info = { "city": images_info["city"], "number": number, } generate_disp_maps(dataset_raw_dirpath, image_info, config_test["disp_map_params"], config_test["thresholds"], config_test["disp_maps_dir"])
def main(): # TODO: pick center pixel when computing gradients # TODO: solve bug.= (look at output) # TODO: display input polygons as well in final plot # TODO: find theta (rotation) that minimizes k(.,.) in closed form # TODO: measure k(., .) with different models trained at different rounds args = get_args() # load overwrite_config file overwrite_config = run_utils.load_config(args.config) if args.runs_dirpath is not None: overwrite_config["runs_dirpath"] = args.runs_dirpath overwrite_config["input_res"] = PATCH_RES overwrite_config["batch_size"] = 1 # Find data_dir if args.data_dirpath is None: data_dirpath = python_utils.choose_first_existing_path( overwrite_config["data_dir_candidates"]) if data_dirpath is None: print("ERROR: Data directory not found!") exit() else: print("Using data from {}".format(data_dirpath)) else: data_dirpath = args.data_dirpath raw_dirpath = os.path.join(data_dirpath, DATASET_NAME, "raw") # Get all tiles tile_info_list_filepath = "{}.tile_info_list.npy".format( args.output_dirname) try: print("Loading tile_info_list from disk...") tile_info_list = np.load(tile_info_list_filepath) except FileNotFoundError: tile_info_list = read.get_tile_info_list(raw_dirpath=raw_dirpath) # Sample patches in each tile pool_size = 4 with Pool(pool_size) as p: params_list = [(raw_dirpath, tile_info, args.ds_fac, PATCH_RES, PATCH_PER_TILE, SEED) for tile_info in tile_info_list] tile_info_list = list( tqdm(p.imap(sample_patches, params_list), total=len(params_list), desc="Sample patches: ")) np.save(tile_info_list_filepath, tile_info_list) # tile_info_list = tile_info_list[-60:-50] # TODO: remove to take all tiles if args.mode == "compute": compute(args, raw_dirpath, overwrite_config, tile_info_list) elif args.mode == "individual": individual(args, raw_dirpath, tile_info_list)
def main(): # --- Process args --- # args = get_args() config = run_utils.load_config(args.config) if config is None: print_utils.print_error( "ERROR: cannot continue without a config file. Exiting now...") exit() if args.batch_size is not None: config["batch_size"] = args.batch_size distribution = "uniform" dataset_params = { "n": args.sample_count, "f": args.frequency, "s": args.noise_std, "d": distribution, } # Find data_dir data_dirpath = python_utils.choose_first_existing_path( config["data_dir_candidates"]) if data_dirpath is None: print_utils.print_error("ERROR: Data directory not found!") exit() data_dirpath = os.path.expanduser(data_dirpath) print_utils.print_info("Using data from {}".format(data_dirpath)) root_dir = os.path.join(data_dirpath, config["data_root_partial_dirpath"]) sobol_generator = rand_utils.SobolGenerator() train_ds = Synthetic1DDataset(root_dir=root_dir, params=dataset_params, split_name="train", sobol_generator=sobol_generator, transform=torchvision.transforms.Compose([ transforms.ToTensor(), transforms.ToDevice(device="cuda") ])) train_dl = DataLoader(train_ds, batch_size=config["batch_size"], shuffle=True, num_workers=4) for i_batch, sample_batched in enumerate(train_dl): print( i_batch, sample_batched['density'].max(), # sample_batched['gt'], # sample_batched['noise'], )
def compute_grads(config, run_params, dataset_params, split_name): # print("# --- Compute grads --- #") working_dir = os.path.dirname(os.path.abspath(__file__)) # Find data_dir data_dirpath = python_utils.choose_first_existing_path( config["data_dir_candidates"]) if data_dirpath is None: print_utils.print_error("ERROR: Data directory not found!") exit() # print_utils.print_info("Using data from {}".format(data_dirpath)) root_dir = os.path.join(data_dirpath, config["data_root_partial_dirpath"]) # setup run directory: runs_dir = os.path.join(working_dir, config["runs_dirpath"]) run_dirpath = None try: run_dirpath = run_utils.setup_run_dir(runs_dir, run_params["run_name"]) except ValueError: print_utils.print_error( "Run name {} was not found. Aborting...".format( run_params["run_name"])) exit() # Choose device dev = torch.device("cuda") if torch.cuda.is_available() else torch.device( "cpu") # Instantiate dataset ds = Synthetic1DDataset(root_dir=root_dir, params=dataset_params, split_name=split_name, transform=torchvision.transforms.Compose([ transforms.ToTensor(), transforms.ToDevice(device=dev) ])) dl = DataLoader(ds, batch_size=1) model = Simple1DInputNet(config, run_params["capacity"]) model.to(dev) analyzer = Analyzer(config, model, run_dirpath) analyzer.compute_and_save_grads(dl)
def main(): # TODO: pick center pixel when computing gradients # TODO: solve bug.= (look at output) # TODO: display input polygons as well in final plot # TODO: find theta (rotation) that minimizes k(.,.) in closed form # TODO: measure k(., .) with different models trained at different rounds args = get_args() # load overwrite_config file overwrite_config = run_utils.load_config(args.config) # Find data_dir data_dir = python_utils.choose_first_existing_path( overwrite_config["data_dir_candidates"]) if data_dir is None: print("ERROR: Data directory not found!") exit() else: print("Using data from {}".format(data_dir)) raw_dirpath = os.path.join(data_dir, DATASET_NAME, "raw") # Get all tiles print("Loading tile_info_list from disk...") tile_info_list_filepath = "{}.tile_info_list.npy".format( args.output_dirname) tile_info_list = np.load(tile_info_list_filepath) # tile_info_list = tile_info_list[-60:-50] # TODO: remove to take all tiles if args.mode == "overall": print("Plot overall histogram of neighbors_soft:") fig_name = args.output_dirname + ".overall_hist" plot_neighbors_soft_hist(raw_dirpath, tile_info_list, args.output_dirname, fig_name) elif args.mode == "individual": plot_similarities(raw_dirpath, tile_info_list, args.output_dirname, args.individual_selection, args.k)
def main(): # load config file config_test = run_utils.load_config(TEST_CONFIG_NAME) # # Handle FLAGS # if FLAGS.batch_size is not None: # batch_size = FLAGS.batch_size # else: # batch_size = config_test["batch_size"] # print("#--- Used params: ---#") # print("batch_size: {}".format(FLAGS.batch_size)) # Find data_dir data_dir = python_utils.choose_first_existing_path( config_test["data_dir_candidates"]) if data_dir is None: print("ERROR: Data directory not found!") exit() else: print("Using data from {}".format(data_dir)) dataset_raw_dirpath = os.path.join( data_dir, config_test["dataset_raw_partial_dirpath"]) output_dir_stem = config_test["align_dir"] for images_info in config_test["images_info_list"]: for number in images_info["numbers"]: image_info = { "city": images_info["city"], "number": number, } measure_image(dataset_raw_dirpath, image_info, PERFECT_GT_POLYGONS_DIRNAME, GT_POLYGONS_DIRNAME_LIST, THRESHOLDS, output_dir_stem)
import os import sys sys.path.append("../../utils") import python_utils PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) # Dataset online processing DATA_DIR = python_utils.choose_first_existing_path([ "/local/shared/epitome-polygon-deep-learning/data", # Try local node first "/home/nigirard/epitome-polygon-deep-learning/data", "/workspace/data", # Try inside Docker image ]) if DATA_DIR is None: print("ERROR: Data directory not found!") exit() else: print("Using data from {}".format(DATA_DIR)) REFERENCE_PIXEL_SIZE = 0.3 # In meters. DS_FAC_LIST = [1, 2, 4, 8] DS_REPEAT_LIST = [ 1, 4, 16, 64 ] # To balance more samples in batches, otherwise there would be too few samples with downsampling_factor=8 IMAGE_DYNAMIC_RANGE = [-1, 1] DISP_MAP_DYNAMIC_RANGE_FAC = 0.5 # Sets disp_map values in [-0.5, 0.5] DISP_MAX_ABS_VALUE = 4 TFRECORDS_DIR_LIST = [ os.path.join(DATA_DIR, "AerialImageDataset/tfrecords.mapalign.multires"), os.path.join(
def train(config, run_params, dataset_params): # print("# --- Starting training --- #") run_name = run_params["run_name"] new_run = run_params["new_run"] init_run_name = run_params["init_run_name"] working_dir = os.path.dirname(os.path.abspath(__file__)) # Find data_dir data_dirpath = python_utils.choose_first_existing_path( config["data_dir_candidates"]) if data_dirpath is None: print_utils.print_error("ERROR: Data directory not found!") exit() # print_utils.print_info("Using data from {}".format(data_dirpath)) root_dir = os.path.join(data_dirpath, config["data_root_partial_dirpath"]) # setup init checkpoints directory path if one is specified: if init_run_name is not None: init_run_dirpath = run_utils.setup_run_dir(config["runs_dirpath"], init_run_name) _, init_checkpoints_dirpath = run_utils.setup_run_subdirs( init_run_dirpath) else: init_checkpoints_dirpath = None # setup run directory: runs_dir = os.path.join(working_dir, config["runs_dirpath"]) run_dirpath = run_utils.setup_run_dir(runs_dir, run_name, new_run) # save config in logs directory run_utils.save_config(config, run_dirpath) # save args args_filepath = os.path.join(run_dirpath, "args.json") python_utils.save_json( args_filepath, { "run_name": run_name, "new_run": new_run, "init_run_name": init_run_name, "batch_size": config["batch_size"], }) # Choose device # dev = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") dev = "cpu" # For small networks and experiments, cpu is much faster # Instantiate dataset # sobol_generator = rand_utils.SobolGenerator() sobol_generator = None train_ds = Synthetic1DDataset(root_dir=root_dir, params=dataset_params, split_name="train", sobol_generator=sobol_generator, transform=torchvision.transforms.Compose([ transforms.ToTensor(), transforms.ToDevice(device=dev) ])) val_ds = Synthetic1DDataset(root_dir=root_dir, params=dataset_params, split_name="val", sobol_generator=sobol_generator, transform=torchvision.transforms.Compose([ transforms.ToTensor(), transforms.ToDevice(device=dev) ])) # print(train_ds.alpha) # print(val_ds.alpha) # exit() # Generate test dataset here because if using Sobel numbers, all datasets should be using the same SobolGenerator # so that they do not generate the same samples. test_ds = Synthetic1DDataset(root_dir=root_dir, params=dataset_params, split_name="test", sobol_generator=sobol_generator, transform=torchvision.transforms.Compose([ transforms.ToTensor(), transforms.ToDevice(device=dev) ])) train_dl = DataLoader(train_ds, batch_size=config["batch_size"], shuffle=True) val_dl = DataLoader(val_ds, batch_size=config["batch_size"]) success = False while not success: try: model = Simple1DInputNet(config) model.to(dev) optimizer = torch.optim.Adam(model.parameters(), lr=config["lr"], weight_decay=config["weight_decay"]) loss_func = measures.l1_loss trainer = Trainer(config, model, optimizer, loss_func, init_checkpoints_dirpath, run_dirpath) trainer.fit(config, train_dl, val_dl) success = True except ValueError: # Catches NaN errors # Try again run_utils.wipe_run_subdirs(run_dirpath) print("\nTry again\n") pass
def main(_): working_dir = os.path.dirname(os.path.abspath(__file__)) config_dir = os.path.dirname(os.path.realpath(__file__)) # print FLAGS print("#--- FLAGS: ---#") print("config: {}".format(FLAGS.config)) print("new_run: {}".format(FLAGS.new_run)) print("init_run_name: {}".format(FLAGS.init_run_name)) print("run_name: {}".format(FLAGS.run_name)) print("batch_size: {}".format(FLAGS.batch_size)) print("ds_fac: {}".format(FLAGS.ds_fac)) # load config file config = run_utils.load_config(FLAGS.config, config_dir) # Check config setting coherences assert len(config["level_loss_coefs_params"]) == config["pool_count"], \ "level_loss_coefs_params ({} elements) must have model_res_levels ({}) elements".format( len(config["level_loss_coefs_params"]), config["pool_count"]) # Find data_dir data_dir = python_utils.choose_first_existing_path( config["data_dir_candidates"]) if data_dir is None: print("ERROR: Data directory not found!") exit() else: print("Using data from {}".format(data_dir)) # Setup dataset dirpaths tfrecords_dirpath_list = [ os.path.join(data_dir, tfrecords_dirpath) for tfrecords_dirpath in config["tfrecords_partial_dirpath_list"] ] # Overwrite config ds_fac if FLAGS specify them if FLAGS.ds_fac is not None: ds_fac_list = [FLAGS.ds_fac] ds_repeat_list = [1] else: ds_fac_list = config["ds_fac_list"] ds_repeat_list = config["ds_repeat_list"] # setup init run directory of one is specified: if FLAGS.init_run_name is not None: init_run_dirpath = run_utils.setup_run_dir(config["runs_dirname"], FLAGS.init_run_name) else: init_run_dirpath = None # setup run directory: runs_dir = os.path.join(working_dir, config["runs_dirname"]) current_run_dirpath = run_utils.setup_run_dir(runs_dir, FLAGS.run_name, FLAGS.new_run) # save config in logs directory run_utils.save_config(config, current_run_dirpath) # save FLAGS FLAGS_filepath = os.path.join(current_run_dirpath, "FLAGS.json") python_utils.save_json( FLAGS_filepath, { "run_name": FLAGS.run_name, "new_run": FLAGS.new_run, "batch_size": FLAGS.batch_size, "ds_fac": FLAGS.ds_fac, }) train(config, tfrecords_dirpath_list, init_run_dirpath, current_run_dirpath, FLAGS.batch_size, ds_fac_list, ds_repeat_list)
def main(): # --- Params --- # seed = 0 data_dir = python_utils.choose_first_existing_path([ "/local/shared/epitome-polygon-deep-learning/data", # Try local node first "/home/nigirard/epitome-polygon-deep-learning/data", "/workspace/data", # Try inside Docker image ]) tfrecords_dir_list = [ # os.path.join(data_dir, "AerialImageDataset/tfrecords.mapalign.multires"), os.path.join( data_dir, "bradbury_buildings_roads_height_dataset/tfrecords.mapalign.multires" ), # os.path.join(data_dir, "mapping_challenge_dataset/tfrecords.mapalign.multires"), ] print("tfrecords_dir_list:") print(tfrecords_dir_list) # downsampling_factors = [1, 2, 4, 8] # resolution_file_repeats = [1, 4, 16, 64] tfrecord_filename_format = "ds_fac_{:02d}.{{:06d}}.tfrecord" downsampling_factors = [1] resolution_file_repeats = [1] dataset_filename_list = create_dataset_filename_list( tfrecords_dir_list, tfrecord_filename_format, downsampling_factors, dataset="train", resolution_file_repeats=resolution_file_repeats) print("Length of dataset_filename_list:") print(len(dataset_filename_list)) patch_outer_res = 220 patch_inner_res = 100 padding = (patch_outer_res - patch_inner_res) // 2 disp_max_abs_value = 4 batch_size = 32 dynamic_range = [-1, 1] keep_poly_prob = 0.1 # Default: 0.1 data_aug = True train = True # --- --- # # Even when reading in multiple threads, share the filename # queue. image, gt_polygons, disp_polygons, gt_polygon_map, disp_field_map, disp_polygon_map = read_and_decode( dataset_filename_list, patch_inner_res, patch_outer_res, batch_size, dynamic_range, keep_poly_prob=keep_poly_prob, data_aug=data_aug, train=train, seed=seed) # The op for initializing the variables. init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) with tf.Session() as sess: sess.run(init_op) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) # Let's read off 3 batches just for example for i in range(30000): print("---- {} ---".format(i)) if train: image_batch, gt_polygon_map_batch, disp_field_map_batch, disp_polygon_map_batch = sess.run( [image, gt_polygon_map, disp_field_map, disp_polygon_map]) else: image_batch, gt_polygons_batch, disp_polygons_batch, gt_polygon_map_batch, disp_field_map_batch, disp_polygon_map_batch = sess.run( [ image, gt_polygons, disp_polygons, gt_polygon_map, disp_field_map, disp_polygon_map ]) print(gt_polygons_batch[0, 0, 0, :]) print(disp_polygons_batch[0, 0, 0, :]) print(gt_polygons_batch.shape) print(disp_polygons_batch.shape) print(image_batch.shape) print(gt_polygon_map_batch.shape) print(disp_field_map_batch.shape) print(disp_polygon_map_batch.shape) # np.set_printoptions(threshold=np.nan) # print(image_batch) # print(gt_polygon_map_batch) # print(disp_field_map_batch) # print(disp_polygon_map_batch) print("image_batch:") print(image_batch.min()) print(image_batch.max()) print("gt_polygon_map_batch:") print(gt_polygon_map_batch.min()) print(gt_polygon_map_batch.max()) try: print(disp_field_map_batch[:, :, :, 0].min()) print(disp_field_map_batch[:, :, :, 0].max()) except IndexError: print( "Skip min and max of disp_field_map_batch because of wrong rank" ) # visualization.plot_field_map("disp_field_map", disp_field_map_batch[0]) print("disp_polygon_map_batch:") print(disp_polygon_map_batch.min()) print(disp_polygon_map_batch.max()) dynamic_range = [-1, 1] image_batch = (image_batch - dynamic_range[0]) / ( dynamic_range[1] - dynamic_range[0]) disp_field_map_batch = disp_field_map_batch * 2 # Within [-1, 1] disp_field_map_batch = disp_field_map_batch * disp_max_abs_value # Within [-disp_max_abs_value, disp_max_abs_value] # gt_polygon_map_batch *= 0 # TODO: Remove # for batch_index in range(batch_size): # if train: # visualization.init_figures(["example"]) # # visualization.plot_example("example", # # image_batch[batch_index], # # gt_polygon_map_batch[batch_index], # # disp_field_map_batch[batch_index], # # disp_polygon_map_batch[batch_index]) # visualization.plot_example("example", # image_batch[batch_index], # disp_polygon_map_batch[batch_index]) # else: # visualization.init_figures(["example", "example polygons"]) # visualization.plot_example("example", # image_batch[batch_index], # gt_polygon_map_batch[batch_index], # disp_field_map_batch[batch_index], # disp_polygon_map_batch[batch_index]) # visualization.plot_example_polygons("example polygons", # image_batch[batch_index], # gt_polygons_batch[batch_index], # disp_polygons_batch[batch_index]) # input("Press <Enter> to continue...") skimage.io.imsave("misaligned_polygon_raster.png", disp_polygon_map_batch[0]) skimage.io.imsave("image.png", image_batch[0]) disp_field_map_image = visualization.flow_to_image( disp_field_map_batch[0]) skimage.io.imsave("displacement_field_map.png", disp_field_map_image) segmentation = gt_polygon_map_batch[0][padding:-padding, padding:-padding, :] skimage.io.imsave("segmentation.png", segmentation) # input("Press <Enter> to continue...") coord.request_stop() coord.join(threads)
def similarity_stats_1d(config, run_name, dataset_params, split_name, stats_params): # print("# --- Similarity Stats --- #") working_dir = os.path.dirname(os.path.abspath(__file__)) # Find data_dir data_dirpath = python_utils.choose_first_existing_path( config["data_dir_candidates"]) if data_dirpath is None: print_utils.print_error("ERROR: Data directory not found!") exit() # print_utils.print_info("Using data from {}".format(data_dirpath)) root_dir = os.path.join(data_dirpath, config["data_root_partial_dirpath"]) # setup run directory: runs_dir = os.path.join(working_dir, config["runs_dirpath"]) run_dirpath = None try: run_dirpath = run_utils.setup_run_dir(runs_dir, run_name) except ValueError: print_utils.print_error( "Run name {} was not found. Aborting...".format(run_name)) exit() # Instantiate dataset # ds = Synthetic1DDataset(root_dir=root_dir, params=dataset_params, split_name="test", # distribution="triangular" # ) ds = Synthetic1DDataset(root_dir=root_dir, params=dataset_params, split_name=split_name, transform=None) sample_count = len(ds) # Load grads and pred grads_dirpath = os.path.join(run_dirpath, "grads") grads_filepath_list = python_utils.get_filepaths(grads_dirpath, endswith_str=".npy", startswith_str="grads.") grads_list = [ np.load(grads_filepath) for grads_filepath in tqdm(grads_filepath_list, desc="Loading grads") ] # print("Grads shape: {}".format(grads_list[0].shape)) pred_filepath_list = python_utils.get_filepaths(grads_dirpath, endswith_str=".npy", startswith_str="pred.") pred_list = [ np.load(pred_filepath) for pred_filepath in tqdm(pred_filepath_list, desc="Loading pred") ] # Create stats dir stats_dirpath = os.path.join(run_dirpath, "stats_1d") os.makedirs(stats_dirpath, exist_ok=True) # import time # t1 = time.clock() neighbor_count, neighbor_count_no_normalization = netsimilarity_utils.compute_soft_neighbor_count( grads_list) neighbors_filepath = os.path.join(stats_dirpath, "neighbors_soft.npy") np.save(neighbors_filepath, neighbor_count) neighbors_filepath = os.path.join(stats_dirpath, "neighbors_soft_no_normalization.npy") np.save(neighbors_filepath, neighbor_count_no_normalization) if not COMPUTE_ONLY_NEIGHBORS_SOFT: # Compute similarity matrix similarity_mat = netsimilarity_utils.compute_similarity_mat_1d( grads_list) # Compute number of neighbors # Hard-thresholding: for t in stats_params["neighbors_t"]: neighbor_count = netsimilarity_utils.compute_neighbor_count( similarity_mat, "hard", t=t) neighbors_filepath = os.path.join( stats_dirpath, "neighbors_hard_t_{}.npy".format(t)) np.save(neighbors_filepath, neighbor_count) # # Soft estimate # neighbor_count = netsimilarity_utils.compute_neighbor_count(similarity_mat, "soft") # neighbors_filepath = os.path.join(stats_dirpath, "neighbors_soft.npy") # np.save(neighbors_filepath, neighbor_count) # Mix for n in stats_params["neighbors_n"]: neighbor_count = netsimilarity_utils.compute_neighbor_count( similarity_mat, "less_soft", n=n) neighbors_filepath = os.path.join( stats_dirpath, "neighbors_less_soft_n_{}.npy".format(n)) np.save(neighbors_filepath, neighbor_count) # print("Time to compute number of neighbors:") # print(time.clock() - t1) # Save inputs for key in ["alpha", "x", "density", "gt", "noise", "curvature"]: filepath = os.path.join(stats_dirpath, "{}.npy".format(key)) values = [sample[key] for sample in ds] np.save(filepath, values) # Save outputs pred_filepath = os.path.join(stats_dirpath, "pred.npy") pred = [pred[0] for pred in pred_list] np.save(pred_filepath, pred) # Error error_filepath = os.path.join(stats_dirpath, "error.npy") error = [ np.abs(sample["gt"] - pred[0]) for sample, pred in zip(ds, pred_list) ] np.save(error_filepath, error) # Losses logs_dirpath = os.path.join(run_dirpath, "logs") final_losses = python_utils.load_json( os.path.join(logs_dirpath, "final_losses.json")) train_loss_filepath = os.path.join(stats_dirpath, "train_loss.npy") np.save(train_loss_filepath, final_losses["train_loss"]) val_loss_filepath = os.path.join(stats_dirpath, "val_loss.npy") np.save(val_loss_filepath, final_losses["val_loss"]) loss_ratio_filepath = os.path.join(stats_dirpath, "loss_ratio.npy") np.save(loss_ratio_filepath, final_losses["val_loss"] / final_losses["train_loss"])