default=23,
help="Number of residual blocks in G")
opt = parser.parse_args()
print(opt)
os.makedirs("images/outputs", exist_ok=True)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Define model and load model checkpoint
generator = GeneratorRRDB(opt.channels,
filters=64,
num_res_blocks=opt.residual_blocks).to(device)
generator.load_state_dict(torch.load(opt.checkpoint_model))
generator.eval()
transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(mean, std)])
# Prepare input
image_tensor = Variable(transform(Image.open(
opt.image_path))).to(device).unsqueeze(0)
# Upsample image
with torch.no_grad():
sr_image = denormalize(generator(image_tensor)).cpu()
# Save image
fn = opt.image_path.split("/")[-1]
save_image(sr_image, f"images/outputs/sr-{fn}")
""" 9) Load Weights """
LOAD_WEIGHTS = True
EMBEDDING_WEIGHT_FILE = 'checkpoints/BIGDATASET-weights-embedding-epoch-3.pt'
ENCODER_WEIGHT_FILE = 'checkpoints/BIGDATASET-weights-encoder-epoch-3.pt'
DECODER_WEIGHT_FILE = 'checkpoints/BIGDATASET-weights-decoder-epoch-3.pt'
if LOAD_WEIGHTS:
print("Loading pretrained weights...")
word_embedding.load_state_dict(torch.load(EMBEDDING_WEIGHT_FILE))
image_encoder.load_state_dict(torch.load(ENCODER_WEIGHT_FILE))
image_decoder.load_state_dict(torch.load(DECODER_WEIGHT_FILE))
""" 10) Device Setup"""
device = 'cuda:1'
device = torch.device(device)
word_embedding = word_embedding.to(device)
image_encoder = image_encoder.to(device)
image_decoder = image_decoder.to(device)
print(vocab.word_to_index('yooooo'))
for i, batch in enumerate(val_loader):
image_batch, word_ids_batch = batch[0].to(device), batch[1].to(device)
for image in image_batch:
sentence = generate_caption(image, image_encoder, image_decoder,
word_embedding, vocab, device)
image = denormalize(image.cpu())
plt.imshow(image)
plt.title(sentence)
plt.show()
plt.pause(1)
def predict_dataset(
self,
dataset: Dataset,
output_folder: str = None,
if_blobs: bool = False,
review: bool = False,
binary_threshold: float = None,
):
elements_count = len(dataset)
for i in range(elements_count):
image, mask_true = dataset[i]
# mask_pred is an np array of shape (256, 256, 1)
# with values in range(0,1)
mask_pred = (self.predict(image) * 255).astype(np.uint8)
# binary mask if needed
if binary_threshold:
mask_pred = ((mask_pred > binary_threshold) * 255).astype(
np.uint8)
# find blobs on prediction
if if_blobs:
detector = SkimageBlobDetector(images=None)
# detector = OpenCVBlobDetector(images=None)
try:
with open(f"best_blob_params_{detector.name}.pickle",
"rb") as f:
ext_params = pickle.load(f)
except Exception as e:
print(e)
ext_params = {}
try:
noise_threshold = ext_params.pop("noise_threshold")
# filter little dark gray noise
mask_pred[mask_pred < noise_threshold] = 0
min_radius = ext_params.pop("min_radius")
max_radius = ext_params.pop("max_radius")
except Exception as e:
print("Params not found", e)
min_radius = 0
max_radius = np.inf
# invert colors only for detector
keypoints = self.detect_blobs(
image=255 -
mask_pred if detector.name == "opencv" else mask_pred,
detector=detector,
ext_params=ext_params,
)
keypoints_filtered = detector.filter_keypoints(
keypoints=keypoints,
min_radius=min_radius,
max_radius=max_radius,
)
mask_pred = detector.draw_blobs(
image=mask_pred,
keypoints=keypoints_filtered,
)
# obtaining filename for saving if needed
(
base_name,
file_format,
) = os.path.split(dataset.masks_fps[i])[-1].split(".")
save_name = os.path.join(
output_folder if output_folder else "",
f"{base_name}"
f"{'_' + str(i) if self.if_crop else ''}"
f"{'_' + str(len(keypoints_filtered)) if if_blobs else ''}"
f"{'_blobs' if if_blobs else ''}."
f"{file_format}",
)
print(save_name)
# mode for saving image
if review:
visualize(
save_name=save_name,
image=denormalize(image.squeeze()),
mask_true=mask_true,
mask_pred=mask_pred,
)
else:
self.save_image(
mask_pred,
filename=save_name,
)
def test_fullres():
# argument parser
parser = argparse.ArgumentParser(description='AIM')
parser.add_argument('--raw_dir',
type=str,
default='/data/raw_to_rgb/FullResTestingPhoneRaw/')
parser.add_argument('--rgb_dir', type=str, default=None)
parser.add_argument('--batch_size', type=int, default=1)
parser.add_argument('--first_weights_path',
type=str,
default='trained_model/p4_2.net')
parser.add_argument('--second_weights_path',
type=str,
default='trained_model/p4_2_second.net')
parser.add_argument('--save_folder', type=str, default='Full_Results/')
args = parser.parse_args()
print(args)
# data loader
dataset = Raw2RgbDataset(raw_dir=args.raw_dir,
rgb_dir=args.rgb_dir,
full_res=True)
data_loader = DataLoader(dataset=dataset, batch_size=args.batch_size)
# model
model = CAUNet(in_channels=3)
model_front = CAUNet(in_channels=4)
model_last = CAUNet(in_channels=3)
model_front.eval()
model_last.eval()
model_front.cuda()
model_last.cuda()
model_front.load_state_dict(load(args.first_weights_path))
model_last.load_state_dict(load(args.second_weights_path)['state_dict'])
# save folder
save_folder = args.save_folder
makedirs(save_folder, exist_ok=True)
# eval metric
total_psnr = 0
total_ssim = 0
# for each batch
for batch_idx, data in enumerate(data_loader):
raw_images = data
raw_images = raw_images.cuda()
h, w = raw_images.size(2), raw_images.size(3)
new_h, new_w = 16 * (h // 16 + 1), 16 * (w // 16 + 1)
pad_h, pad_w = new_h - h, new_w - w
pad_t, pad_b = pad_h // 2, pad_h - pad_h // 2
pad_l, pad_r = pad_w // 2, pad_w - pad_w // 2
input_image = pad(raw_images, (pad_l, pad_r, pad_t, pad_b),
mode='replicate')
with torch.no_grad():
out_front = model_front(input_image)
out = model_last(out_front)
out = denormalize(out)
out = out[:, :, pad_t:-pad_b, pad_l:-pad_r]
for i in range(out.size(0)):
index = batch_idx * args.batch_size + i + 1
image_path = save_folder + str(index) + '.png'
save_image(tensor=out[i], filename=image_path)
message = 'id: {}'.format(index)
print(message)
# average
total_psnr /= len(dataset)
total_ssim /= len(dataset)
# Print psnr, ssim
message = '\t {}: {:.2f}\t {}: {:.4f}'.format('psnr', total_psnr, 'ssim',
total_ssim)
print(message)
def visualize(config, model, epoch, n_particles, images,
positions, groups, is_train, is_eval=False):
"""
images: (B, T, C, H, W)
positions: (B, T, N, 3)
groups: (B, N)
"""
dir_split = 'train' if is_train else 'valid'
if is_eval:
save_dir = os.path.join(
config.run_dir, 'eval', 'batch_{}'.format(epoch))
else:
save_dir = os.path.join(
config.run_dir, 'vis', 'epoch{:03d}_{}'.format(epoch, dir_split))
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
model.eval()
_, _, C, H, W = images.shape
B = config.vis_samples
T_vis = config.vis_frames_per_sample
vis_gt_pos, vis_pred_pos, vis_pred_grp = None, None, None
for j in range(T_vis):
vis_images = images[:B, j:j + config.n_frames,
...].to(_DEVICE)
gt_pos = positions[:B, j:j + config.n_frames,
...].transpose(2, 3).to(_DEVICE)
pred_pos, pred_grp = model(vis_images)
# Take the last time step
new_gt_pos = gt_pos[:, -1, ...].detach().cpu().unsqueeze(1).numpy()
new_pred_pos = pred_pos[:, -1, ...].detach().cpu().unsqueeze(1).numpy()
new_pred_grp = pred_grp[:, -1, ...].detach().cpu().unsqueeze(1).numpy()
if vis_gt_pos is None:
vis_gt_pos = new_gt_pos
vis_pred_pos = new_pred_pos
vis_pred_grp = new_pred_grp
else:
vis_gt_pos = np.concatenate(
(vis_gt_pos, new_gt_pos), 1)
vis_pred_pos = np.concatenate(
(vis_pred_pos, new_pred_pos), 1)
vis_pred_grp = np.concatenate(
(vis_pred_grp, new_pred_grp), 1)
for k in range(B):
vis_input = denormalize(
images[k, config.n_frames - 1:config.n_frames + T_vis - 1,
...].cpu().numpy())
vis_input = (vis_input * 255).astype('uint8')
# randomly select 5 particles to highlight with
# different colors for tracking temporal consistency
highlight_idxs = np.random.choice(
range(n_particles), 5, replace=False)
# use special z-axis range for MassRope visualization
if config.dataset == 'MassRope':
zlim = (0.3, 1.7)
else:
zlim = (0, 0.7)
# visualize ground truth groups (N,)
vis_gt_frames = utils.toy_render(
np.transpose(vis_gt_pos[k], (0, 2, 1)),
title='Ground Truth',
gt_groups=groups[k],
zlim=zlim)
vis_pred_frames = utils.toy_render(
np.transpose(vis_pred_pos[k], (0, 2, 1)),
title='Prediction',
highlight_idxs=highlight_idxs,
groups=vis_pred_grp[k],
zlim=zlim)
vis_frames = np.concatenate(
(vis_gt_frames, vis_pred_frames), 2)
imageio.mimwrite(
os.path.join(
save_dir, 'sample{:02d}_input.mp4'.format(k)),
vis_input,
fps=15)
imageio.mimwrite(
os.path.join(
save_dir,
'sample{:02d}_particles.mp4'.format(k)),
vis_frames,
fps=15)
def test():
cudnn.benchmark = True
# argument parser
parser = argparse.ArgumentParser(description='AIM')
parser.add_argument('--raw_dir',
type=str,
default='/data/raw_to_rgb/TestingPhoneRaw/')
parser.add_argument('--rgb_dir', type=str, default=None)
parser.add_argument('--batch_size', type=int, default=1)
parser.add_argument('--weights_path', type=str, default=None)
parser.add_argument('--save_folder',
type=str,
default='fidelity_result_images/')
args = parser.parse_args()
print(args)
# data loader
dataset = Raw2RgbDataset(raw_dir=args.raw_dir, rgb_dir=args.rgb_dir)
data_loader = DataLoader(dataset=dataset, batch_size=args.batch_size)
# model
model = CAUNet(in_channels=3)
# load weights
model.load_state_dict(load('trained_model/p4_2_second.net')['state_dict'])
model.eval()
model.cuda()
model_first = CAUNet()
model_first.eval()
model_first.cuda()
model_first.load_state_dict(load('trained_model/p4_2.net'))
second_model_front = CAUNet(in_channels=4)
second_model_last = CAUNet(in_channels=3)
second_model_front.eval()
second_model_last.eval()
second_model_front.cuda()
second_model_last.cuda()
second_model_front.load_state_dict(
load('trained_model/p4_1_hue.net')['state_dict'])
second_model_last.load_state_dict(
load('trained_model/p4_1_hue_second.net')['state_dict'])
third_model_front = CAUNet(in_channels=4)
third_model_front.eval()
third_model_front.cuda()
third_model_front.load_state_dict(
load('trained_model/p5_1_hue.pkl')['state_dict'])
# save folder
save_folder = args.save_folder
makedirs(save_folder, exist_ok=True)
# eval metric
total_psnr = 0
total_ssim = 0
# for each batch
for batch_idx, data in enumerate(data_loader):
raw_images = data
raw_images = raw_images.cuda()
with torch.no_grad():
first_out_front = model_first(raw_images)
first_out = model(first_out_front)
first_out = denormalize(first_out)
second_out_front = second_model_front(raw_images)
second_out = second_model_last(second_out_front)
second_out = denormalize(second_out)
thrid_out_front = third_model_front(raw_images)
thrid_out = denormalize(thrid_out_front)
out = (first_out + second_out + thrid_out) / 3
# save image, compute psnr, ssim
for i in range(out.size(0)):
index = batch_idx * args.batch_size + i
image_path = save_folder + str(index) + '.png'
save_image(tensor=out[i], filename=image_path)
message = 'id: {}'.format(index)
print(message)
# average
total_psnr /= len(dataset)
total_ssim /= len(dataset)
# Print psnr, ssim
message = '\t {}: {:.2f}\t {}: {:.4f}'.format('psnr', total_psnr, 'ssim',
total_ssim)
print(message)