def main():
# load config file
config_test = run_utils.load_config(TEST_CONFIG_NAME)
# Find data_dir
data_dir = python_utils.choose_first_existing_path(config_test["data_dir_candidates"])
if data_dir is None:
print("ERROR: Data directory not found!")
exit()
else:
print("Using data from {}".format(data_dir))
dataset_raw_dirpath = os.path.join(data_dir, config_test["dataset_raw_partial_dirpath"])
output_dir = config_test["align_dir"] + OUTPUT_DIRNAME_EXTENTION
if not os.path.exists(output_dir):
os.makedirs(output_dir)
for images_info in config_test["images_info_list"]:
for number in images_info["numbers"]:
image_info = {
"city": images_info["city"],
"number": number,
}
test_image(RUNS_DIRPATH, dataset_raw_dirpath, image_info, config_test["disp_maps_dir"],
config_test["disp_map_params"]["disp_map_count"],
config_test["disp_map_params"]["disp_max_abs_value"], config_test["batch_size"], DS_FAC_LIST,
RUN_NAME_LIST, config_test["model_disp_max_abs_value"], config_test["thresholds"], output_dir,
config_test["output_shapefiles"])
def generate_test(config,
params,
split_name="train",
seed=None,
sobol_generator=None):
# Find data_dir
data_dirpath = python_utils.choose_first_existing_path(
config["data_dir_candidates"])
if data_dirpath is None:
print_utils.print_error("ERROR: Data directory not found!")
exit()
data_dirpath = os.path.expanduser(data_dirpath)
print_utils.print_info("Using data from {}".format(data_dirpath))
root_dir = os.path.join(data_dirpath, config["data_root_partial_dirpath"])
alpha, x, density, gt, noise, curvature = generate_data(
root_dir,
params,
split_name=split_name,
seed=seed,
sobol_generator=sobol_generator)
noisy_gt = gt + noise
import matplotlib.pyplot as plt
f = plt.figure()
f.set_tight_layout({"pad": .0})
ax = f.gca()
# plt.scatter(alpha, noisy_gt, s=10)
ax.plot(alpha, noisy_gt)
ax.set_xlabel("alpha")
ax.set_ylabel("y")
# plt.title("Sinusoid, freq = {}".format(params["f"]))
plt.show()
def main():
# load config file
config = run_utils.load_config(CONFIG_NAME)
config_test = run_utils.load_config(TEST_CONFIG_NAME)
# Find data_dir
data_dir = python_utils.choose_first_existing_path(
config["data_dir_candidates"])
if data_dir is None:
print("ERROR: Data directory not found!")
exit()
else:
print("Using data from {}".format(data_dir))
dataset_raw_dirpath = os.path.join(
data_dir, config_test["dataset_raw_partial_dirpath"])
if not os.path.exists(config_test["disp_maps_dir"]):
os.makedirs(config_test["disp_maps_dir"])
for images_info in config_test["images_info_list"]:
for number in images_info["numbers"]:
image_info = {
"city": images_info["city"],
"number": number,
}
generate_disp_maps(dataset_raw_dirpath, image_info,
config_test["disp_map_params"],
config_test["thresholds"],
config_test["disp_maps_dir"])
def main():
# TODO: pick center pixel when computing gradients
# TODO: solve bug.= (look at output)
# TODO: display input polygons as well in final plot
# TODO: find theta (rotation) that minimizes k(.,.) in closed form
# TODO: measure k(., .) with different models trained at different rounds
args = get_args()
# load overwrite_config file
overwrite_config = run_utils.load_config(args.config)
if args.runs_dirpath is not None:
overwrite_config["runs_dirpath"] = args.runs_dirpath
overwrite_config["input_res"] = PATCH_RES
overwrite_config["batch_size"] = 1
# Find data_dir
if args.data_dirpath is None:
data_dirpath = python_utils.choose_first_existing_path(
overwrite_config["data_dir_candidates"])
if data_dirpath is None:
print("ERROR: Data directory not found!")
exit()
else:
print("Using data from {}".format(data_dirpath))
else:
data_dirpath = args.data_dirpath
raw_dirpath = os.path.join(data_dirpath, DATASET_NAME, "raw")
# Get all tiles
tile_info_list_filepath = "{}.tile_info_list.npy".format(
args.output_dirname)
try:
print("Loading tile_info_list from disk...")
tile_info_list = np.load(tile_info_list_filepath)
except FileNotFoundError:
tile_info_list = read.get_tile_info_list(raw_dirpath=raw_dirpath)
# Sample patches in each tile
pool_size = 4
with Pool(pool_size) as p:
params_list = [(raw_dirpath, tile_info, args.ds_fac, PATCH_RES,
PATCH_PER_TILE, SEED)
for tile_info in tile_info_list]
tile_info_list = list(
tqdm(p.imap(sample_patches, params_list),
total=len(params_list),
desc="Sample patches: "))
np.save(tile_info_list_filepath, tile_info_list)
# tile_info_list = tile_info_list[-60:-50] # TODO: remove to take all tiles
if args.mode == "compute":
compute(args, raw_dirpath, overwrite_config, tile_info_list)
elif args.mode == "individual":
individual(args, raw_dirpath, tile_info_list)
def main():
# --- Process args --- #
args = get_args()
config = run_utils.load_config(args.config)
if config is None:
print_utils.print_error(
"ERROR: cannot continue without a config file. Exiting now...")
exit()
if args.batch_size is not None:
config["batch_size"] = args.batch_size
distribution = "uniform"
dataset_params = {
"n": args.sample_count,
"f": args.frequency,
"s": args.noise_std,
"d": distribution,
}
# Find data_dir
data_dirpath = python_utils.choose_first_existing_path(
config["data_dir_candidates"])
if data_dirpath is None:
print_utils.print_error("ERROR: Data directory not found!")
exit()
data_dirpath = os.path.expanduser(data_dirpath)
print_utils.print_info("Using data from {}".format(data_dirpath))
root_dir = os.path.join(data_dirpath, config["data_root_partial_dirpath"])
sobol_generator = rand_utils.SobolGenerator()
train_ds = Synthetic1DDataset(root_dir=root_dir,
params=dataset_params,
split_name="train",
sobol_generator=sobol_generator,
transform=torchvision.transforms.Compose([
transforms.ToTensor(),
transforms.ToDevice(device="cuda")
]))
train_dl = DataLoader(train_ds,
batch_size=config["batch_size"],
shuffle=True,
num_workers=4)
for i_batch, sample_batched in enumerate(train_dl):
print(
i_batch,
sample_batched['density'].max(),
# sample_batched['gt'],
# sample_batched['noise'],
)
def compute_grads(config, run_params, dataset_params, split_name):
# print("# --- Compute grads --- #")
working_dir = os.path.dirname(os.path.abspath(__file__))
# Find data_dir
data_dirpath = python_utils.choose_first_existing_path(
config["data_dir_candidates"])
if data_dirpath is None:
print_utils.print_error("ERROR: Data directory not found!")
exit()
# print_utils.print_info("Using data from {}".format(data_dirpath))
root_dir = os.path.join(data_dirpath, config["data_root_partial_dirpath"])
# setup run directory:
runs_dir = os.path.join(working_dir, config["runs_dirpath"])
run_dirpath = None
try:
run_dirpath = run_utils.setup_run_dir(runs_dir, run_params["run_name"])
except ValueError:
print_utils.print_error(
"Run name {} was not found. Aborting...".format(
run_params["run_name"]))
exit()
# Choose device
dev = torch.device("cuda") if torch.cuda.is_available() else torch.device(
"cpu")
# Instantiate dataset
ds = Synthetic1DDataset(root_dir=root_dir,
params=dataset_params,
split_name=split_name,
transform=torchvision.transforms.Compose([
transforms.ToTensor(),
transforms.ToDevice(device=dev)
]))
dl = DataLoader(ds, batch_size=1)
model = Simple1DInputNet(config, run_params["capacity"])
model.to(dev)
analyzer = Analyzer(config, model, run_dirpath)
analyzer.compute_and_save_grads(dl)
def main():
# TODO: pick center pixel when computing gradients
# TODO: solve bug.= (look at output)
# TODO: display input polygons as well in final plot
# TODO: find theta (rotation) that minimizes k(.,.) in closed form
# TODO: measure k(., .) with different models trained at different rounds
args = get_args()
# load overwrite_config file
overwrite_config = run_utils.load_config(args.config)
# Find data_dir
data_dir = python_utils.choose_first_existing_path(
overwrite_config["data_dir_candidates"])
if data_dir is None:
print("ERROR: Data directory not found!")
exit()
else:
print("Using data from {}".format(data_dir))
raw_dirpath = os.path.join(data_dir, DATASET_NAME, "raw")
# Get all tiles
print("Loading tile_info_list from disk...")
tile_info_list_filepath = "{}.tile_info_list.npy".format(
args.output_dirname)
tile_info_list = np.load(tile_info_list_filepath)
# tile_info_list = tile_info_list[-60:-50] # TODO: remove to take all tiles
if args.mode == "overall":
print("Plot overall histogram of neighbors_soft:")
fig_name = args.output_dirname + ".overall_hist"
plot_neighbors_soft_hist(raw_dirpath, tile_info_list,
args.output_dirname, fig_name)
elif args.mode == "individual":
plot_similarities(raw_dirpath, tile_info_list, args.output_dirname,
args.individual_selection, args.k)
def main():
# load config file
config_test = run_utils.load_config(TEST_CONFIG_NAME)
# # Handle FLAGS
# if FLAGS.batch_size is not None:
# batch_size = FLAGS.batch_size
# else:
# batch_size = config_test["batch_size"]
# print("#--- Used params: ---#")
# print("batch_size: {}".format(FLAGS.batch_size))
# Find data_dir
data_dir = python_utils.choose_first_existing_path(
config_test["data_dir_candidates"])
if data_dir is None:
print("ERROR: Data directory not found!")
exit()
else:
print("Using data from {}".format(data_dir))
dataset_raw_dirpath = os.path.join(
data_dir, config_test["dataset_raw_partial_dirpath"])
output_dir_stem = config_test["align_dir"]
for images_info in config_test["images_info_list"]:
for number in images_info["numbers"]:
image_info = {
"city": images_info["city"],
"number": number,
}
measure_image(dataset_raw_dirpath, image_info,
PERFECT_GT_POLYGONS_DIRNAME,
GT_POLYGONS_DIRNAME_LIST, THRESHOLDS,
output_dir_stem)
import os
import sys
sys.path.append("../../utils")
import python_utils
PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
# Dataset online processing
DATA_DIR = python_utils.choose_first_existing_path([
"/local/shared/epitome-polygon-deep-learning/data", # Try local node first
"/home/nigirard/epitome-polygon-deep-learning/data",
"/workspace/data", # Try inside Docker image
])
if DATA_DIR is None:
print("ERROR: Data directory not found!")
exit()
else:
print("Using data from {}".format(DATA_DIR))
REFERENCE_PIXEL_SIZE = 0.3 # In meters.
DS_FAC_LIST = [1, 2, 4, 8]
DS_REPEAT_LIST = [
1, 4, 16, 64
] # To balance more samples in batches, otherwise there would be too few samples with downsampling_factor=8
IMAGE_DYNAMIC_RANGE = [-1, 1]
DISP_MAP_DYNAMIC_RANGE_FAC = 0.5 # Sets disp_map values in [-0.5, 0.5]
DISP_MAX_ABS_VALUE = 4
TFRECORDS_DIR_LIST = [
os.path.join(DATA_DIR, "AerialImageDataset/tfrecords.mapalign.multires"),
os.path.join(
def train(config, run_params, dataset_params):
# print("# --- Starting training --- #")
run_name = run_params["run_name"]
new_run = run_params["new_run"]
init_run_name = run_params["init_run_name"]
working_dir = os.path.dirname(os.path.abspath(__file__))
# Find data_dir
data_dirpath = python_utils.choose_first_existing_path(
config["data_dir_candidates"])
if data_dirpath is None:
print_utils.print_error("ERROR: Data directory not found!")
exit()
# print_utils.print_info("Using data from {}".format(data_dirpath))
root_dir = os.path.join(data_dirpath, config["data_root_partial_dirpath"])
# setup init checkpoints directory path if one is specified:
if init_run_name is not None:
init_run_dirpath = run_utils.setup_run_dir(config["runs_dirpath"],
init_run_name)
_, init_checkpoints_dirpath = run_utils.setup_run_subdirs(
init_run_dirpath)
else:
init_checkpoints_dirpath = None
# setup run directory:
runs_dir = os.path.join(working_dir, config["runs_dirpath"])
run_dirpath = run_utils.setup_run_dir(runs_dir, run_name, new_run)
# save config in logs directory
run_utils.save_config(config, run_dirpath)
# save args
args_filepath = os.path.join(run_dirpath, "args.json")
python_utils.save_json(
args_filepath, {
"run_name": run_name,
"new_run": new_run,
"init_run_name": init_run_name,
"batch_size": config["batch_size"],
})
# Choose device
# dev = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
dev = "cpu" # For small networks and experiments, cpu is much faster
# Instantiate dataset
# sobol_generator = rand_utils.SobolGenerator()
sobol_generator = None
train_ds = Synthetic1DDataset(root_dir=root_dir,
params=dataset_params,
split_name="train",
sobol_generator=sobol_generator,
transform=torchvision.transforms.Compose([
transforms.ToTensor(),
transforms.ToDevice(device=dev)
]))
val_ds = Synthetic1DDataset(root_dir=root_dir,
params=dataset_params,
split_name="val",
sobol_generator=sobol_generator,
transform=torchvision.transforms.Compose([
transforms.ToTensor(),
transforms.ToDevice(device=dev)
]))
# print(train_ds.alpha)
# print(val_ds.alpha)
# exit()
# Generate test dataset here because if using Sobel numbers, all datasets should be using the same SobolGenerator
# so that they do not generate the same samples.
test_ds = Synthetic1DDataset(root_dir=root_dir,
params=dataset_params,
split_name="test",
sobol_generator=sobol_generator,
transform=torchvision.transforms.Compose([
transforms.ToTensor(),
transforms.ToDevice(device=dev)
]))
train_dl = DataLoader(train_ds,
batch_size=config["batch_size"],
shuffle=True)
val_dl = DataLoader(val_ds, batch_size=config["batch_size"])
success = False
while not success:
try:
model = Simple1DInputNet(config)
model.to(dev)
optimizer = torch.optim.Adam(model.parameters(),
lr=config["lr"],
weight_decay=config["weight_decay"])
loss_func = measures.l1_loss
trainer = Trainer(config, model, optimizer, loss_func,
init_checkpoints_dirpath, run_dirpath)
trainer.fit(config, train_dl, val_dl)
success = True
except ValueError: # Catches NaN errors
# Try again
run_utils.wipe_run_subdirs(run_dirpath)
print("\nTry again\n")
pass
def main(_):
working_dir = os.path.dirname(os.path.abspath(__file__))
config_dir = os.path.dirname(os.path.realpath(__file__))
# print FLAGS
print("#--- FLAGS: ---#")
print("config: {}".format(FLAGS.config))
print("new_run: {}".format(FLAGS.new_run))
print("init_run_name: {}".format(FLAGS.init_run_name))
print("run_name: {}".format(FLAGS.run_name))
print("batch_size: {}".format(FLAGS.batch_size))
print("ds_fac: {}".format(FLAGS.ds_fac))
# load config file
config = run_utils.load_config(FLAGS.config, config_dir)
# Check config setting coherences
assert len(config["level_loss_coefs_params"]) == config["pool_count"], \
"level_loss_coefs_params ({} elements) must have model_res_levels ({}) elements".format(
len(config["level_loss_coefs_params"]), config["pool_count"])
# Find data_dir
data_dir = python_utils.choose_first_existing_path(
config["data_dir_candidates"])
if data_dir is None:
print("ERROR: Data directory not found!")
exit()
else:
print("Using data from {}".format(data_dir))
# Setup dataset dirpaths
tfrecords_dirpath_list = [
os.path.join(data_dir, tfrecords_dirpath)
for tfrecords_dirpath in config["tfrecords_partial_dirpath_list"]
]
# Overwrite config ds_fac if FLAGS specify them
if FLAGS.ds_fac is not None:
ds_fac_list = [FLAGS.ds_fac]
ds_repeat_list = [1]
else:
ds_fac_list = config["ds_fac_list"]
ds_repeat_list = config["ds_repeat_list"]
# setup init run directory of one is specified:
if FLAGS.init_run_name is not None:
init_run_dirpath = run_utils.setup_run_dir(config["runs_dirname"],
FLAGS.init_run_name)
else:
init_run_dirpath = None
# setup run directory:
runs_dir = os.path.join(working_dir, config["runs_dirname"])
current_run_dirpath = run_utils.setup_run_dir(runs_dir, FLAGS.run_name,
FLAGS.new_run)
# save config in logs directory
run_utils.save_config(config, current_run_dirpath)
# save FLAGS
FLAGS_filepath = os.path.join(current_run_dirpath, "FLAGS.json")
python_utils.save_json(
FLAGS_filepath, {
"run_name": FLAGS.run_name,
"new_run": FLAGS.new_run,
"batch_size": FLAGS.batch_size,
"ds_fac": FLAGS.ds_fac,
})
train(config, tfrecords_dirpath_list, init_run_dirpath,
current_run_dirpath, FLAGS.batch_size, ds_fac_list, ds_repeat_list)
def main():
# --- Params --- #
seed = 0
data_dir = python_utils.choose_first_existing_path([
"/local/shared/epitome-polygon-deep-learning/data", # Try local node first
"/home/nigirard/epitome-polygon-deep-learning/data",
"/workspace/data", # Try inside Docker image
])
tfrecords_dir_list = [
# os.path.join(data_dir, "AerialImageDataset/tfrecords.mapalign.multires"),
os.path.join(
data_dir,
"bradbury_buildings_roads_height_dataset/tfrecords.mapalign.multires"
),
# os.path.join(data_dir, "mapping_challenge_dataset/tfrecords.mapalign.multires"),
]
print("tfrecords_dir_list:")
print(tfrecords_dir_list)
# downsampling_factors = [1, 2, 4, 8]
# resolution_file_repeats = [1, 4, 16, 64]
tfrecord_filename_format = "ds_fac_{:02d}.{{:06d}}.tfrecord"
downsampling_factors = [1]
resolution_file_repeats = [1]
dataset_filename_list = create_dataset_filename_list(
tfrecords_dir_list,
tfrecord_filename_format,
downsampling_factors,
dataset="train",
resolution_file_repeats=resolution_file_repeats)
print("Length of dataset_filename_list:")
print(len(dataset_filename_list))
patch_outer_res = 220
patch_inner_res = 100
padding = (patch_outer_res - patch_inner_res) // 2
disp_max_abs_value = 4
batch_size = 32
dynamic_range = [-1, 1]
keep_poly_prob = 0.1 # Default: 0.1
data_aug = True
train = True
# --- --- #
# Even when reading in multiple threads, share the filename
# queue.
image, gt_polygons, disp_polygons, gt_polygon_map, disp_field_map, disp_polygon_map = read_and_decode(
dataset_filename_list,
patch_inner_res,
patch_outer_res,
batch_size,
dynamic_range,
keep_poly_prob=keep_poly_prob,
data_aug=data_aug,
train=train,
seed=seed)
# The op for initializing the variables.
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
with tf.Session() as sess:
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
# Let's read off 3 batches just for example
for i in range(30000):
print("---- {} ---".format(i))
if train:
image_batch, gt_polygon_map_batch, disp_field_map_batch, disp_polygon_map_batch = sess.run(
[image, gt_polygon_map, disp_field_map, disp_polygon_map])
else:
image_batch, gt_polygons_batch, disp_polygons_batch, gt_polygon_map_batch, disp_field_map_batch, disp_polygon_map_batch = sess.run(
[
image, gt_polygons, disp_polygons, gt_polygon_map,
disp_field_map, disp_polygon_map
])
print(gt_polygons_batch[0, 0, 0, :])
print(disp_polygons_batch[0, 0, 0, :])
print(gt_polygons_batch.shape)
print(disp_polygons_batch.shape)
print(image_batch.shape)
print(gt_polygon_map_batch.shape)
print(disp_field_map_batch.shape)
print(disp_polygon_map_batch.shape)
# np.set_printoptions(threshold=np.nan)
# print(image_batch)
# print(gt_polygon_map_batch)
# print(disp_field_map_batch)
# print(disp_polygon_map_batch)
print("image_batch:")
print(image_batch.min())
print(image_batch.max())
print("gt_polygon_map_batch:")
print(gt_polygon_map_batch.min())
print(gt_polygon_map_batch.max())
try:
print(disp_field_map_batch[:, :, :, 0].min())
print(disp_field_map_batch[:, :, :, 0].max())
except IndexError:
print(
"Skip min and max of disp_field_map_batch because of wrong rank"
)
# visualization.plot_field_map("disp_field_map", disp_field_map_batch[0])
print("disp_polygon_map_batch:")
print(disp_polygon_map_batch.min())
print(disp_polygon_map_batch.max())
dynamic_range = [-1, 1]
image_batch = (image_batch - dynamic_range[0]) / (
dynamic_range[1] - dynamic_range[0])
disp_field_map_batch = disp_field_map_batch * 2 # Within [-1, 1]
disp_field_map_batch = disp_field_map_batch * disp_max_abs_value # Within [-disp_max_abs_value, disp_max_abs_value]
# gt_polygon_map_batch *= 0 # TODO: Remove
# for batch_index in range(batch_size):
# if train:
# visualization.init_figures(["example"])
# # visualization.plot_example("example",
# # image_batch[batch_index],
# # gt_polygon_map_batch[batch_index],
# # disp_field_map_batch[batch_index],
# # disp_polygon_map_batch[batch_index])
# visualization.plot_example("example",
# image_batch[batch_index],
# disp_polygon_map_batch[batch_index])
# else:
# visualization.init_figures(["example", "example polygons"])
# visualization.plot_example("example",
# image_batch[batch_index],
# gt_polygon_map_batch[batch_index],
# disp_field_map_batch[batch_index],
# disp_polygon_map_batch[batch_index])
# visualization.plot_example_polygons("example polygons",
# image_batch[batch_index],
# gt_polygons_batch[batch_index],
# disp_polygons_batch[batch_index])
# input("Press <Enter> to continue...")
skimage.io.imsave("misaligned_polygon_raster.png",
disp_polygon_map_batch[0])
skimage.io.imsave("image.png", image_batch[0])
disp_field_map_image = visualization.flow_to_image(
disp_field_map_batch[0])
skimage.io.imsave("displacement_field_map.png",
disp_field_map_image)
segmentation = gt_polygon_map_batch[0][padding:-padding,
padding:-padding, :]
skimage.io.imsave("segmentation.png", segmentation)
# input("Press <Enter> to continue...")
coord.request_stop()
coord.join(threads)
def similarity_stats_1d(config, run_name, dataset_params, split_name,
stats_params):
# print("# --- Similarity Stats --- #")
working_dir = os.path.dirname(os.path.abspath(__file__))
# Find data_dir
data_dirpath = python_utils.choose_first_existing_path(
config["data_dir_candidates"])
if data_dirpath is None:
print_utils.print_error("ERROR: Data directory not found!")
exit()
# print_utils.print_info("Using data from {}".format(data_dirpath))
root_dir = os.path.join(data_dirpath, config["data_root_partial_dirpath"])
# setup run directory:
runs_dir = os.path.join(working_dir, config["runs_dirpath"])
run_dirpath = None
try:
run_dirpath = run_utils.setup_run_dir(runs_dir, run_name)
except ValueError:
print_utils.print_error(
"Run name {} was not found. Aborting...".format(run_name))
exit()
# Instantiate dataset
# ds = Synthetic1DDataset(root_dir=root_dir, params=dataset_params, split_name="test",
# distribution="triangular"
# )
ds = Synthetic1DDataset(root_dir=root_dir,
params=dataset_params,
split_name=split_name,
transform=None)
sample_count = len(ds)
# Load grads and pred
grads_dirpath = os.path.join(run_dirpath, "grads")
grads_filepath_list = python_utils.get_filepaths(grads_dirpath,
endswith_str=".npy",
startswith_str="grads.")
grads_list = [
np.load(grads_filepath)
for grads_filepath in tqdm(grads_filepath_list, desc="Loading grads")
]
# print("Grads shape: {}".format(grads_list[0].shape))
pred_filepath_list = python_utils.get_filepaths(grads_dirpath,
endswith_str=".npy",
startswith_str="pred.")
pred_list = [
np.load(pred_filepath)
for pred_filepath in tqdm(pred_filepath_list, desc="Loading pred")
]
# Create stats dir
stats_dirpath = os.path.join(run_dirpath, "stats_1d")
os.makedirs(stats_dirpath, exist_ok=True)
# import time
# t1 = time.clock()
neighbor_count, neighbor_count_no_normalization = netsimilarity_utils.compute_soft_neighbor_count(
grads_list)
neighbors_filepath = os.path.join(stats_dirpath, "neighbors_soft.npy")
np.save(neighbors_filepath, neighbor_count)
neighbors_filepath = os.path.join(stats_dirpath,
"neighbors_soft_no_normalization.npy")
np.save(neighbors_filepath, neighbor_count_no_normalization)
if not COMPUTE_ONLY_NEIGHBORS_SOFT:
# Compute similarity matrix
similarity_mat = netsimilarity_utils.compute_similarity_mat_1d(
grads_list)
# Compute number of neighbors
# Hard-thresholding:
for t in stats_params["neighbors_t"]:
neighbor_count = netsimilarity_utils.compute_neighbor_count(
similarity_mat, "hard", t=t)
neighbors_filepath = os.path.join(
stats_dirpath, "neighbors_hard_t_{}.npy".format(t))
np.save(neighbors_filepath, neighbor_count)
# # Soft estimate
# neighbor_count = netsimilarity_utils.compute_neighbor_count(similarity_mat, "soft")
# neighbors_filepath = os.path.join(stats_dirpath, "neighbors_soft.npy")
# np.save(neighbors_filepath, neighbor_count)
# Mix
for n in stats_params["neighbors_n"]:
neighbor_count = netsimilarity_utils.compute_neighbor_count(
similarity_mat, "less_soft", n=n)
neighbors_filepath = os.path.join(
stats_dirpath, "neighbors_less_soft_n_{}.npy".format(n))
np.save(neighbors_filepath, neighbor_count)
# print("Time to compute number of neighbors:")
# print(time.clock() - t1)
# Save inputs
for key in ["alpha", "x", "density", "gt", "noise", "curvature"]:
filepath = os.path.join(stats_dirpath, "{}.npy".format(key))
values = [sample[key] for sample in ds]
np.save(filepath, values)
# Save outputs
pred_filepath = os.path.join(stats_dirpath, "pred.npy")
pred = [pred[0] for pred in pred_list]
np.save(pred_filepath, pred)
# Error
error_filepath = os.path.join(stats_dirpath, "error.npy")
error = [
np.abs(sample["gt"] - pred[0]) for sample, pred in zip(ds, pred_list)
]
np.save(error_filepath, error)
# Losses
logs_dirpath = os.path.join(run_dirpath, "logs")
final_losses = python_utils.load_json(
os.path.join(logs_dirpath, "final_losses.json"))
train_loss_filepath = os.path.join(stats_dirpath, "train_loss.npy")
np.save(train_loss_filepath, final_losses["train_loss"])
val_loss_filepath = os.path.join(stats_dirpath, "val_loss.npy")
np.save(val_loss_filepath, final_losses["val_loss"])
loss_ratio_filepath = os.path.join(stats_dirpath, "loss_ratio.npy")
np.save(loss_ratio_filepath,
final_losses["val_loss"] / final_losses["train_loss"])
