def main(model_config,
dataset_type,
save_outputs,
output_dir,
data_config,
seed,
small_run,
entry,
device):
# Load the model
model = make_model(**model_config)
# model.sid_obj.to(device)
# print(model)
model.to(device)
# Load the data
_, val, test = load_data()
dataset = test if dataset_type == "test" else val
init_randomness(seed)
if entry is None:
print("Evaluating the model on {} ({})".format(data_config["data_name"],
dataset_type))
evaluate_model_on_dataset(model, dataset, small_run, device, save_outputs, output_dir)
else:
print("Evaluating {}".format(entry))
evaluate_model_on_data_entry(model, dataset, entry, device)
def main(model_config, save_outputs, output_dir, data_config, seed, small_run,
dataset_type, entry, device):
# Load the model
model = make_model(**model_config)
model.sinkhorn_opt.to(device)
from tensorboardX import SummaryWriter
from datetime import datetime
# Load the data
train, test = load_data(dorn_mode=False)
dataset = train if dataset_type == "train" else test
eval_fn = lambda input_, device: model.evaluate(
input_["rgb"], input_["crop"][0, :], input_["depth_cropped"],
torch.ones_like(input_["depth_cropped"]), device)
init_randomness(seed)
if entry is None:
print("Evaluating the model on {}.".format(data_config["data_name"]))
evaluate_model_on_dataset(eval_fn, dataset, small_run, device,
save_outputs, output_dir)
else:
print("Evaluating {}".format(entry))
model.sinkhorn_opt.writer = SummaryWriter(log_dir=os.path.join("runs",
datetime.now().strftime('%b%d'),
datetime.now().strftime('%H-%M-%S_') + \
"densedepth_hist_match_wass"))
evaluate_model_on_data_entry(eval_fn, dataset, entry, device,
save_outputs, output_dir)
def main(model_config, dataset_type, save_outputs, output_dir, data_config,
seed, small_run, device):
# Load the model
model = make_model(**model_config)
model.eval()
model.to(device)
model.sid_obj.to(device)
# Load the data
train, test = load_data(dorn_mode=True)
dataset = test if dataset_type == "test" else train
print(
list((name, entry.shape) for name, entry in dataset[0].items()
if isinstance(entry, torch.Tensor)))
init_randomness(seed)
eval_fn = lambda input_, device: model.evaluate(
input_["bgr"].to(device), input_["bgr_orig"].to(device), input_["crop"]
[0, :], input_["depth_cropped"].to(device), input_["depth"].to(device),
torch.ones_like(input_["depth_cropped"]).to(device), device)
print("Evaluating the model on {} ({})".format(data_config["data_name"],
dataset_type))
evaluate_model_on_dataset(eval_fn, dataset, small_run, device,
save_outputs, output_dir)
def main(model_config, save_outputs, output_dir, data_config, seed, small_run,
entry, device):
# Load the model
model = make_model(**model_config)
# model.sid_obj.to(device)
# print(model)
model.to(device)
from tensorboardX import SummaryWriter
from datetime import datetime
# model.writer = SummaryWriter(log_dir=os.path.join("runs",
# datetime.now().strftime('%b%d'),
# datetime.now().strftime('%H-%M-%S_') + \
# "dorn_sinkhorn_opt"))
# Load the data
dataset = load_data(dorn_mode=True)
eval_fn = lambda input_, device: model.evaluate(
input_["rgb_cropped"].to(device), input_["rgb_cropped_orig"].to(
device), input_["spad"].to(device), input_["mask_orig"].to(device),
input_["depth_cropped_orig"].to(device), device)
init_randomness(seed)
if entry is None:
print("Evaluating the model on {}.".format(data_config["data_name"]))
evaluate_model_on_dataset(eval_fn, dataset, small_run, device,
save_outputs, output_dir)
else:
print("Evaluating {}".format(entry))
evaluate_model_on_data_entry(eval_fn, dataset, entry, device,
save_outputs, output_dir)
def main(model_config, save_outputs, output_dir, data_config, seed, small_run,
entry, device):
# Load the model
model = make_model(**model_config)
model.eval()
model.to(device)
model.sid_obj.to(device)
# Load the data
dataset = load_data(dorn_mode=True)
eval_fn = lambda input_, device: model.evaluate(
input_["rgb_cropped"].to(device), input_["rgb_cropped_orig"].to(
device), input_["depth_cropped_orig"].to(device), input_[
"mask_orig"].to(device), device)
init_randomness(seed)
if entry is None:
print("Evaluating the model on {}.".format(data_config["data_name"]))
evaluate_model_on_dataset(eval_fn, dataset, small_run, device,
save_outputs, output_dir)
else:
print("Evaluating {}".format(entry))
evaluate_model_on_data_entry(eval_fn, dataset, entry, device,
save_outputs, output_dir)
def main(model_config, save_outputs, output_dir, data_config, seed, small_run,
dataset_type, entry):
# Load the model
model = make_model(**model_config)
# model.sid_obj.to(device)
from tensorboardX import SummaryWriter
from datetime import datetime
# Load the data
train, test = load_data(dorn_mode=False)
dataset = train if dataset_type == "train" else test
eval_fn = lambda input_, device: model.evaluate(
input_["rgb"], input_["crop"][0, :], input_["depth_cropped"], input_[
"rawdepth_cropped"], input_["mask_cropped"],
torch.ones_like(input_["depth_cropped"]))
init_randomness(seed)
if entry is None:
print("Evaluating the model on {}.".format(data_config["data_name"]))
evaluate_model_on_dataset(eval_fn, dataset, small_run, None,
save_outputs, output_dir)
else:
print("Evaluating {}".format(entry))
evaluate_model_on_data_entry(eval_fn, dataset, entry, None,
save_outputs, output_dir)
def main(model_config, save_outputs, output_dir, data_config, seed, small_run,
device):
# Load the model
model = make_model(**model_config)
model.eval()
model.to(device)
# model.sid_obj.to(device)
# Load the data
dataset = load_data()
init_randomness(seed)
print("Evaluating the model on {}".format(data_config["data_name"]))
evaluate_model_on_dataset(model, dataset, small_run, device, save_outputs,
output_dir)
def main(model_config, eval_config, data_config, seed, device):
# Load the model
model = make_model(**model_config)
model.to(device)
print(model)
# Load the data
_, val, test = load_data()
dataset = test if eval_config["dataset"] == "test" else val
init_randomness(seed)
# Make dataloader
dataloader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=2,
pin_memory=True,
worker_init_fn=worker_init_randomness)
if eval_config["mode"] == "save_outputs":
# Run the model on everything and save everything to disk.
model.eval()
safe_makedir(eval_config["output_dir"])
with torch.no_grad():
for i, data in enumerate(dataloader):
print("Evaluating {}".format(i))
model.write_eval(
data,
os.path.join(eval_config["output_dir"],
"{}_out.npy".format(i)), device)
elif eval_config["mode"] == "evaluate_metrics":
# Load things and call the model's evaluate function on them.
metrics = model.evaluate_dir(eval_config["output_dir"], device)
with open(os.path.join(eval_config["output_dir"], "metrics.json"),
"w") as f:
json.dump(metrics, f)
else:
print("Unrecognized mode: {}".format(eval_config["mode"]))
def main(model_config, dataset_type, save_outputs, output_dir, data_config,
seed, small_run, entry, device):
# Load the model
model = make_model(**model_config)
model.eval()
model.to(device)
model.sid_obj.to(device)
model.sinkhorn_opt.to(device)
# Load the data
train, test = load_data(dorn_mode=True)
dataset = test if dataset_type == "test" else train
from tensorboardX import SummaryWriter
from datetime import datetime
init_randomness(seed)
eval_fn = lambda input_, device: model.evaluate(
input_["bgr"].to(device), input_["bgr_orig"].to(device), input_["crop"]
[0, :], input_["depth_cropped"].to(device),
torch.ones_like(input_["depth_cropped"]).to(device), device)
if entry is None:
print("Evaluating the model on {} ({}).".format(
data_config["data_name"], dataset_type))
ex.observers.append(
FileStorageObserver.create(os.path.join(output_dir, "runs")))
evaluate_model_on_dataset(eval_fn, dataset, small_run, device,
save_outputs, output_dir)
else:
print("Evaluating {}".format(entry))
model.sinkhorn_opt.writer = SummaryWriter(log_dir=os.path.join("runs",
datetime.now().strftime('%b%d'),
datetime.now().strftime('%H-%M-%S_') + \
"dorn_hist_match_wass"))
evaluate_model_on_data_entry(eval_fn, dataset, entry, device,
save_outputs, output_dir)
def main(model_config,
save_outputs,
output_dir,
data_config,
seed,
small_run,
entry,
device):
# Load the model
model = make_model(**model_config)
# model.sid_obj.to(device)
# print(model)
model.to(device)
from tensorboardX import SummaryWriter
from datetime import datetime
model.writer = SummaryWriter(log_dir=os.path.join("runs",
datetime.now().strftime('%b%d'),
datetime.now().strftime('%H-%M-%S_') + \
"densedepth_sinkhorn_opt"))
# Load the data
dataset = load_data(dorn_mode=False)
eval_fn = lambda input_, device: model.evaluate(input_["rgb"], # RGB input
input_["rgb_cropped"], # rgb cropped for intensity scaling
input_["crop"], # 4-tuple of crop parameters
input_["spad"], # simulated SPAD
input_["mask"], # Cropped mask
input_["depth_cropped"], # Ground truth depth
device)
init_randomness(seed)
if entry is None:
print("Evaluating the model on {}.".format(data_config["data_name"]))
evaluate_model_on_dataset(eval_fn, dataset, small_run, torch_cuda_device, save_outputs, output_dir)
else:
print("Evaluating {}".format(entry))
evaluate_model_on_data_entry(eval_fn, dataset, entry, torch_cuda_device, save_outputs, output_dir)
def main(dataset_type, entry, save_outputs, output_dir, seed, small_run,
device):
# Load the data
dataset = load_data(channels_first=False)
# Load the model
model = DenseDepth()
init_randomness(seed)
if entry is None:
dataloader = DataLoader(
dataset,
batch_size=1,
shuffle=False,
num_workers=0, # needs to be 0 to not crash autograd profiler.
pin_memory=True)
# if eval_config["save_outputs"]:
with torch.no_grad():
metric_list = [
"delta1", "delta2", "delta3", "rel_abs_diff", "rmse", "mse",
"log10", "weight"
]
metrics = np.zeros((len(dataset) if not small_run else small_run,
len(metric_list)))
entry_list = []
outputs = []
for i, data in enumerate(dataloader):
# TESTING
if small_run and i == small_run:
break
entry = data["entry"][0]
entry = entry if isinstance(entry, str) else entry.item()
entry_list.append(entry)
print("Evaluating {}".format(data["entry"][0]))
# pred, pred_metrics = model.evaluate(data, device)
pred, pred_metrics, pred_weight = model.evaluate(
data["rgb"].to(device), data["depth_cropped"].to(device),
torch.ones_like(data["depth_cropped"]).to(device))
for j, metric_name in enumerate(metric_list[:-1]):
metrics[i, j] = pred_metrics[metric_name]
metrics[i, -1] = pred_weight
# Option to save outputs:
if save_outputs:
outputs.append(pred.cpu().numpy())
if save_outputs:
np.save(
os.path.join(
output_dir,
"densedepth_{}_outputs.npy".format(dataset_type)),
np.concatenate(outputs, axis=0))
# Save metrics using pandas
metrics_df = pd.DataFrame(data=metrics,
index=entry_list,
columns=metric_list)
metrics_df.to_pickle(path=os.path.join(
output_dir, "densedepth_{}_metrics.pkl".format(dataset_type)))
# Compute weighted averages:
average_metrics = np.average(metrics_df.ix[:, :-1],
weights=metrics_df.weight,
axis=0)
average_df = pd.Series(data=average_metrics,
index=metric_list[:-1])
average_df.to_csv(os.path.join(
output_dir,
"densedepth_{}_avg_metrics.csv".format(dataset_type)),
header=True)
print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format(
'd1', 'd2', 'd3', 'rel', 'rms', 'log_10'))
print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}".
format(average_metrics[0], average_metrics[1],
average_metrics[2], average_metrics[3],
average_metrics[4], average_metrics[6]))
print("wrote results to {}".format(output_dir))
else:
input_unbatched = dataset.get_item_by_id(entry)
# for key in ["rgb", "albedo", "rawdepth", "spad", "mask", "rawdepth_orig", "mask_orig", "albedo_orig"]:
# input_[key] = input_[key].unsqueeze(0)
from torch.utils.data._utils.collate import default_collate
data = default_collate([input_unbatched])
# Checks
entry = data["entry"][0]
entry = entry if isinstance(entry, str) else entry.item()
print("Entry: {}".format(entry))
# print("remove_dc: ", model.remove_dc)
# print("use_intensity: ", model.use_intensity)
# print("use_squared_falloff: ", model.use_squared_falloff)
pred, pred_metrics, pred_weight = model.evaluate(
data["rgb"].to(device), data["depth_cropped"].to(device),
torch.ones_like(data["depth_cropped"]).to(device))
if save_outputs:
np.save(
os.path.join(output_dir,
"{}_{}_out.npy".format(dataset_type, entry)))
print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format(
'd1', 'd2', 'd3', 'rel', 'rms', 'log_10'))
print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}".
format(pred_metrics["delta1"], pred_metrics["delta2"],
pred_metrics["delta3"], pred_metrics["rel_abs_diff"],
pred_metrics["rms"], pred_metrics["log10"]))
sgd_iters=400,
sinkhorn_iters=40,
sigma=.5,
lam=1e-2,
kde_eps=1e-4,
sinkhorn_eps=1e-4,
remove_dc=spad_config["dc_count"] > 0.,
use_intensity=spad_config["use_intensity"],
use_squared_falloff=spad_config["use_squared_falloff"],
lr=1e3)
model.to(device)
_, _, test = load_data(**data_config, spad_config=spad_config)
dataloader = DataLoader(test, shuffle=True)
start = perf_counter()
init_randomness(95290421)
input_ = test.get_item_by_id("kitchen_0002/1121")
for key in [
"rgb", "rgb_orig", "rawdepth", "spad", "mask", "rawdepth_orig",
"mask_orig"
]:
input_[key] = input_[key].unsqueeze(0).to(device)
data_load_time = perf_counter() - start
print("dataloader: {}".format(data_load_time))
# print(input_["entry"])
# print(model.hints_extractor[0].weight)
# Checks
print(input_["entry"])
print("remove_dc: ", model.remove_dc)
print("use_intensity: ", model.use_intensity)
