def get_feature_RGBD(data_root, meta_fname, batch, num_images):
if torch.cuda.is_available():
device = 'cuda'
else:
device = 'cpu'
dataset = DIODE(meta_fname, data_root, splits=['train', 'val'], scene_types=['indoors', 'outdoor'],
num_images=num_images)
dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch, collate_fn=collate_skip_empty, shuffle=True)
features = None
labels = []
image_paths = []
for idx, batch in enumerate(tqdm(dataloader, desc='Extracting RGB-D features')):
im_file, cls, im, dm, dm_valid = batch
im = torch.tensor(im, dtype=torch.float32)
pass
current_features = torch.flatten(list_new_dm)
if features is not None:
features = np.concatenate((features, current_features))
else:
features = current_features
return features, labels, image_paths
pass
def main():
depth_dict = {}
dset = DIODE(meta_fname, data_root, splits=['train', 'val'], scene_types=['indoors', 'outdoor'])
print(dset.classes)
for _, dm, mask in tqdm(dset, "analysing data distribution", len(dset)): # for each image in the dataset...
for indx, (row_dm, row_mask) in enumerate(zip(dm, mask)): # for each row...
if row_mask[indx] == 1.:
try:
depth_dict[row_dm[indx]] += 1 # for each value in each row..
except KeyError:
depth_dict[row_dm[indx]] = 1
def save_depth_dict():
depth_dict = {}
# dset = DIODE(meta_fname, data_root, splits=['train', 'val'], scene_types=['indoors', 'outdoors'])
dset = DIODE(meta_fname, data_root, splits=['train', 'val'], scene_types=['indoors'])
for _, dm, mask in tqdm(dset, "analysing data distribution", len(dset)): # for each image in the dataset...
for indx, (row_dm, row_mask) in enumerate(zip(dm, mask)): # for each row...
if row_mask[indx] == 1.:
try:
depth_dict[row_dm[indx]] += 1 # for each value in each row..
except KeyError:
depth_dict[row_dm[indx]] = 1
a_file = open("data_distribution_indoors.pkl", "wb")
pickle.dump(depth_dict, a_file)
a_file.close()
print(depth_dict)
def show_outlier_images_from_depth_dict():
depth_dict = {}
# dset = DIODE(meta_fname, data_root, splits=['train', 'val'], scene_types=['indoors', 'outdoors'])
dset = DIODE(meta_fname, data_root, splits=['train', 'val'], scene_types=['indoors'])
for img, dm, mask in tqdm(dset, "analysing data distribution", len(dset)): # for each image in the dataset...
has_outliers = False
for indx, (row_dm, row_mask) in enumerate(zip(dm, mask)): # for each row...
if row_mask[indx] == 1.:
if row_dm[indx] > 20: # d > 20m
has_outliers = True
try:
depth_dict[row_dm[indx]] += 1 # for each value in each row..
except KeyError:
depth_dict[row_dm[indx]] = 1
if has_outliers:
im = Image.fromarray(np.uint8(img))
im.show()
plot_depth_map(dm, mask)
def get_features(data_root, meta_fname, batch, num_images):
# move the input and model to GPU for speed if available
if torch.cuda.is_available():
device = 'cuda'
else:
device = 'cpu'
# initialize our implementation of ResNet
# TODO: other model...
# model = loadModel()
# model = resnest101(pretrained=True)
model = resnest200(pretrained=True)
model.eval()
model.to(device)
# read the dataset and initialize the data loader
# dataset = DIODE(dataset, num_images)
dataset = DIODE(meta_fname, data_root, splits=['train', 'val'], scene_types=['indoors', 'outdoor'], num_images=num_images)
dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch, collate_fn=collate_skip_empty, shuffle=True)
# we'll store the features as NumPy array of size num_images x feature_size
features = None
# we'll also store the image labels and paths to visualize them later
labels = []
image_paths = []
for idx, batch in enumerate(tqdm(dataloader, desc='Running the model inference')):
im_file, cls, im, dm, dm_valid = batch
im = torch.tensor(im, dtype=torch.float32)
newsize = (384, 512)
# newsize = (192, 256)
# newsize = (3, 4)
# dms = []
# for one_dm, one_dm_valid in zip(dm, dm_valid):
# save_plot_dm(one_dm, one_dm_valid)
# one_dm = Image.open("AAA.png")
# one_dm = one_dm.convert('RGB')
# one_dm = one_dm.resize(newsize) # limited GPU resources
# one_dm = transforms.ToTensor()(one_dm)
# dms.append(one_dm)
# dms = torch.cat(dms, dim=1)
# print(dms)
# im = dms
# im = torch.reshape(im, [64, 3, 768, 1024]) # cuda ran out of memory...
# im = torch.reshape(im, [64, 3, 192, 256])
# im = torch.reshape(im, [1, 3, 192, 256])
try:
im = torch.reshape(im, [32, 3, 384, 512])
except RuntimeError as err:
dim = int(str(err).split(" ")[-1])
batch_size = dim // 3 // 384 // 512
im = torch.reshape(im, [batch_size, 3, 384, 512])
pass
# im = torch.reshape(im, [1, 3, 3, 4]) # 1/10/32/64 is the batch size
images = im.to(device)
# idk what I am doing
# labels.append(cls) # append for batchsize == 1
# labels += list(cls) # extend / += for batchsize >= 2
labels += list(cls) # extend / += for batchsize >= 2
# image_paths.append(im_file)
image_paths += list(im_file)
with torch.no_grad():
print(images.size())
output = model.forward(images)
# # TODO: copied from inference code
# my_image = images[0].cuda().unsqueeze(0)
#
# output = model.evaluate(my_image)
# predict = torch.max(output, 1)[1].cpu().numpy() + 1
#
# mask = get_mask_pallete(predict, 'ade20k') # this should change when using DE
# mask.show()
# plt.plot(images[0].cpu().numpy()[0])
# plt.show()
# something = output[0]
# something_else = something[0]
# my_img = images[0]
#
# trans = transforms.ToPILImage()
# trans1 = transforms.ToTensor()
#
# img = my_img.cpu().numpy()[0]
# # img = np.transpose(img, (1, 2, 0))
# # show the image
# plt.imshow(img)
# plt.show()
#
# plt.imshow(trans(trans1(my_img).convert("RGB")))
#
# width, height, pred_img = inference(model, transforms.ToPILImage(my_img).convert("RGB"))
current_features = output.cpu().numpy()
if features is not None:
features = np.concatenate((features, current_features))
else:
features = current_features
# print(features)
print(labels)
print(image_paths)
return features, labels, image_paths
def get_features(data_root, meta_fname, batch, num_images):
# move the input and model to GPU for speed if available
if torch.cuda.is_available():
device = 'cuda'
else:
device = 'cpu'
# initialize our implementation of ResNet
# model = resnest101(pretrained=True)
model = resnest200(pretrained=True)
model.eval()
model.to(device)
# read the dataset and initialize the data loader
# dataset = DIODE(dataset, num_images)
dataset = DIODE(meta_fname, data_root, splits=['train', 'val'], scene_types=['indoors', 'outdoor'],
num_images=num_images)
dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch, collate_fn=collate_skip_empty, shuffle=True)
# we'll store the features as NumPy array of size num_images x feature_size
features = None
# we'll also store the image labels and paths to visualize them later
labels = []
image_paths = []
for idx, batch in enumerate(tqdm(dataloader, desc='Running the model inference')):
im_file, cls, im, dm, dm_valid = batch
im = torch.tensor(im, dtype=torch.float32)
newsize = (384, 512)
# newsize = (192, 256)
# newsize = (3, 4)
# dms = []
# for one_dm, one_dm_valid in zip(dm, dm_valid):
# save_plot_dm(one_dm, one_dm_valid)
# one_dm = Image.open("AAA.png")
# one_dm = one_dm.convert('RGB')
# one_dm = one_dm.resize(newsize) # limited GPU resources
# one_dm = transforms.ToTensor()(one_dm)
# dms.append(one_dm)
# dms = torch.cat(dms, dim=1)
# print(dms)
# im = dms
# im = torch.reshape(im, [64, 3, 768, 1024]) # cuda ran out of memory...
# im = torch.reshape(im, [64, 3, 192, 256])
# im = torch.reshape(im, [1, 3, 192, 256])
try:
im = torch.reshape(im, [32, 3, 384, 512])
except RuntimeError as err:
dim = int(str(err).split(" ")[-1])
batch_size = dim // 3 // 384 // 512
im = torch.reshape(im, [batch_size, 3, 384, 512])
pass
# im = torch.reshape(im, [1, 3, 3, 4]) # 1/10/32/64 is the batch size
images = im.to(device)
# idk what I am doing
# labels.append(cls) # append for batchsize == 1
# labels += list(cls) # extend / += for batchsize >= 2
labels += list(cls) # extend / += for batchsize >= 2
# image_paths.append(im_file)
image_paths += list(im_file)
with torch.no_grad():
# print(images.size())
output = model.forward(images)
current_features = output.cpu().numpy()
if features is not None:
features = np.concatenate((features, current_features))
else:
features = current_features
# print(features)
# print(labels)
# print(image_paths)
a_file = open("features_labels_imagepaths_resnest200_256.pkl", "wb")
pickle.dump(zip(features, labels, image_paths), a_file)
a_file.close()
return features, labels, image_paths
def get_feature_depths(data_root, meta_fname, batch, num_images):
if torch.cuda.is_available():
device = 'cuda'
else:
device = 'cpu'
# initialize our implementation of ResNet
# model = resnest101(pretrained=True)
model = resnest200(pretrained=True)
model.eval()
model.to(device)
# read the dataset and initialize the data loader
# dataset = DIODE(dataset, num_images)
dataset = DIODE(meta_fname, data_root, splits=['train', 'val'], scene_types=['indoors', 'outdoor'],
num_images=num_images)
dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch, collate_fn=collate_skip_empty, shuffle=True)
# we'll store the features as NumPy array of size num_images x feature_size
features = None
# we'll also store the image labels and paths to visualize them later
labels = []
image_paths = []
skip_step = 100 # this is for compressing images
# thr = 8.820631664646312
# thr = 3.636924135853406
# thr = 3.6
for idx, batch in enumerate(tqdm(dataloader, desc='Running the model inference')):
im_file, cls, im, dm, dm_valid = batch
im = torch.tensor(im, dtype=torch.float32)
list_cls = []
list_new_dm = []
list_new_dm_valid = []
for one_dm, one_dm_valid in zip(dm, dm_valid):
# plot_depth_map(one_dm, one_dm_valid)
avg_depth = 0
count = 0
new_dm = []
new_dm_valid = []
# process one depth map
for row_indx in range(0, len(one_dm), skip_step):
new_row = []
new_row_valid = []
for cell_indx in range(0, len(one_dm[row_indx]), skip_step):
new_row.append(one_dm[row_indx][cell_indx])
new_row_valid.append(one_dm_valid[row_indx][cell_indx])
if one_dm_valid[row_indx][cell_indx]:
avg_depth += one_dm[row_indx][cell_indx]
count += 1
new_dm.append(new_row)
new_dm_valid.append(new_row_valid)
# NOT RIGHT NOW ! (because we will transform them later)
# new_dm = torch.FloatTensor(new_dm)
# new_dm_valid = torch.FloatTensor(new_dm_valid)
# NOT RIGHT NOW !
# print(len(new_dm))
# print(len(new_dm[0]))
# new_dm = transforms.ToPILImage(new_dm)
# new_dm.show()
# plt.imshow(new_dm)
# plt.imshow(new_dm_valid)
plot_depth_map(new_dm, new_dm_valid)
list_new_dm.append(new_dm)
list_new_dm_valid.append(new_dm_valid)
avg = avg_depth / count
# if avg > thr:
# list_cls.append('outdoor')
# else:
# list_cls.append('indoors')
list_new_dm = torch.FloatTensor(list_new_dm)
list_new_dm_valid = torch.FloatTensor(list_new_dm_valid)
# newsize = (384, 512)
# newsize = (192, 256)
# newsize = (3, 4)
# try:
# im = torch.reshape(im, [32, 3, 384, 512])
# except RuntimeError as err:
# dim = int(str(err).split(" ")[-1])
# batch_size = dim // 3 // 384 // 512
# im = torch.reshape(im, [batch_size, 3, 384, 512])
# pass
# images = im.to(device)
labels += list(cls)
# labels = list_cls
image_paths += list(im_file)
# with torch.no_grad():
# # print(images.size())
# output = model.forward(images)
#
# current_features = output.cpu().numpy()
print("unflattened: ", list_new_dm.size())
current_features = torch.flatten(list_new_dm)
print("flattened: ", current_features.size())
if features is not None:
features = np.concatenate((features, current_features))
else:
features = current_features
# print(features)
# print(labels)
# print(image_paths)
# a_file = open("features_labels_imagepaths_resnest200_256.pkl", "wb")
# pickle.dump(zip(features, labels, image_paths), a_file)
# a_file.close()
return features, labels, image_paths
def get_features(data_root, meta_fname, batch, num_images):
# move the input and model to GPU for speed if available
if torch.cuda.is_available():
device = 'cuda'
else:
device = 'cpu'
# TODO: other model...
model = loadModel()
model.eval()
model.to(device)
# read the dataset and initialize the data loader
dataset = DIODE(meta_fname,
data_root,
splits=['train', 'val'],
scene_types=['indoors', 'outdoor'],
num_images=num_images)
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=batch,
collate_fn=collate_skip_empty,
shuffle=True)
# we'll store the features as NumPy array of size num_images x feature_size
features = None
# we'll also store the image labels and paths to visualize them later
labels = []
image_paths = []
for idx, batch in enumerate(
tqdm(dataloader, desc='Running the model inference')):
im_file, cls, im, dm, dm_valid = batch
# TODO: removed this just for testing...
# im = torch.tensor(im, dtype=torch.float32)
resized_im = None
to_img = transforms.ToPILImage()
to_ten = transforms.ToTensor()
for one_im in im:
# print(one_im.size())
# plt.imshow(one_im)
# plt.show()
# TODO: RESIZE TODO: RESIZE TODO: RESIZE TODO: RESIZE TODO: RESIZE TODO: RESIZE TODO: RESIZE TODO: RESIZE
# one_im = torch.reshape(one_im, [3, 384, 512])
# print(one_im.size())
#
# one_im = to_img(one_im)
#
# plt.imshow(one_im)
# plt.show()
#
# # keep 1024:768 ratio (4:3)
# newsize = (508, 381)
# # newsize = (1024, 768)
# # newsize = (1024, 768)
# # newsize = (256, 192)
#
# one_resized = one_im.resize(newsize)
# # one_resized.show()
#
# one_ten = to_ten(one_resized)
# print(one_ten.size())
#
# one_ten = torch.reshape(one_ten, [newsize[0], newsize[1], 3])
# TODO: RESIZE TODO: RESIZE TODO: RESIZE TODO: RESIZE TODO: RESIZE TODO: RESIZE TODO: RESIZE TODO: RESIZE
one_ten = []
my_step_row = 32
my_step_col = 32
i = 0
j = 0
for row in one_im:
one_row = []
if i % my_step_row == 0:
for rgb in row:
if j % my_step_col == 0:
# tup = [int(rgb[0]), int(rgb[1]), int(rgb[2])]
red = int(rgb[0])
green = int(rgb[1])
blue = int(rgb[2])
# tup = (0xFF << 24) | (red << 16) | (green << 8) | blue
tup = (red << 16) | (green << 8) | blue
one_row.append(tup)
j += 1
one_ten.append(one_row)
i += 1
one_ten = torch.IntTensor(one_ten)
# plt.imshow(one_ten)
# plt.show()
if resized_im is not None:
resized_im = np.concatenate(
(resized_im, torch.flatten(one_ten)))
else:
resized_im = torch.flatten(one_ten)
labels += list(cls)
image_paths += list(im_file)
if features is not None:
features = np.concatenate((features, resized_im.flatten()))
else:
features = resized_im.flatten()
print(features)
# print(labels)
# print(image_paths)
return features, labels, image_paths