def dense_process_data(index):
images = list()
for ind in indices['dense']:
ptr = int(ind)
if ptr <= record.num_frames:
imgs = self._load_image(record.path, ptr)
else:
imgs = self._load_image(record.path, record.num_frames)
images.extend(imgs)
if self.phase == 'Fntest':
images = [np.asarray(im) for im in images]
clip_input = np.concatenate(images, axis=2)
self.t = transforms.Compose([
transforms.Resize(256)])
clip_input = self.t(clip_input)
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
if record.crop_pos == 0:
self.transform = transforms.Compose([
transforms.CenterCrop((256, 256)),
transforms.ToTensor(),
normalize,
])
elif record.crop_pos == 1:
self.transform = transforms.Compose([
transforms.CornerCrop2((256, 256),),
transforms.ToTensor(),
normalize,
])
elif record.crop_pos == 2:
self.transform = transforms.Compose([
transforms.CornerCrop1((256, 256)),
transforms.ToTensor(),
normalize,
])
return self.transform(clip_input)
return self.transform(images)
print("Preprocessing finished!")
cuda_available = torch.cuda.is_available()
# directory results
if not os.path.exists(RESULTS_PATH):
os.makedirs(RESULTS_PATH)
# Load dataset
mean = m
std_dev = s
transform_train = transforms.Compose([
transforms.RandomApply([transforms.ColorJitter(0.1, 0.1, 0.1, 0.1)],
p=0.5),
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize(mean, std_dev)
])
transform_test = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize(mean, std_dev)
])
training_set = LocalDataset(IMAGES_PATH,
TRAINING_PATH,
transform=transform_train)
validation_set = LocalDataset(IMAGES_PATH,
VALIDATION_PATH,
default="facades",
help="Name of the dataset: ['facades', 'maps', 'cityscapes']")
parser.add_argument("--batch_size",
type=int,
default=1,
help="Size of the batches")
parser.add_argument("--lr",
type=float,
default=0.0002,
help="Adams learning rate")
args = parser.parse_args()
device = ('cuda:0' if torch.cuda.is_available() else 'cpu')
transforms = T.Compose([
T.Resize((256, 256)),
T.ToTensor(),
T.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
])
# models
print('Defining models!')
generator = UnetGenerator().to(device)
discriminator = ConditionalDiscriminator().to(device)
# optimizers
g_optimizer = torch.optim.Adam(generator.parameters(),
lr=args.lr,
betas=(0.5, 0.999))
d_optimizer = torch.optim.Adam(discriminator.parameters(),
lr=args.lr,
betas=(0.5, 0.999))
# loss functions
if args.preprocess:
print ("Preprocessing..")
preprocessing()
print ("Preprocessing finished!")
cuda_available = torch.cuda.is_available()
# directory results
if not os.path.exists(RESULTS_PATH):
os.makedirs(RESULTS_PATH)
# Load dataset
mean=m
std_dev=s
transform = transforms.Compose([transforms.Resize((224,224)),
transforms.ToTensor(),
transforms.Normalize(mean, std_dev)])
training_set = LocalDataset(IMAGES_PATH, TRAINING_PATH, transform=transform)
validation_set = LocalDataset(IMAGES_PATH, VALIDATION_PATH, transform=transform)
training_set_loader = DataLoader(dataset=training_set, batch_size=BATCH_SIZE, num_workers=THREADS, shuffle=True)
validation_set_loader = DataLoader(dataset=validation_set, batch_size=BATCH_SIZE, num_workers=THREADS, shuffle=False)
def train_model(model_name, model, lr=LEARNING_RATE, epochs=EPOCHS, momentum=MOMENTUM, weight_decay=0, train_loader=training_set_loader, test_loader=validation_set_loader):
if not os.path.exists(RESULTS_PATH + "/" + model_name):
os.makedirs(RESULTS_PATH + "/" + model_name)
criterion = nn.CrossEntropyLoss()
parser.add_argument('--device', type=str, default='cuda:0',
help='cpu or cuda:0 or cuda:1')
args = parser.parse_args() if string is None else parser.parse_args(string)
return args
if __name__=='__main__':
args = parse_args()
wandb.init(config=args,
project=f'dlcv_naive_{args.source}2{args.target}')
size = 64
t0 = transforms.Compose([
transforms.Resize(size),
transforms.ColorJitter(),
transforms.RandomRotation(15, fill=(0,)),
transforms.Grayscale(3),
transforms.ToTensor(),
transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5))
])
t1 = transforms.Compose([
transforms.Resize(size),
transforms.Grayscale(3),
transforms.ToTensor(),
transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5))
])
root = '../hw3_data/digits/'
# dataset
def inference(args):
if args.target=='mnistm':
args.source = 'usps'
elif args.target=='usps':
args.source = 'svhn'
elif args.target=='svhn':
args.source = 'mnistm'
else:
raise NotImplementedError(f"{args.target}: not implemented!")
size = args.img_size
t1 = transforms.Compose([
transforms.Resize(size),
transforms.Grayscale(3),
transforms.ToTensor(),
transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5))
])
valid_target_dataset = Digits_Dataset_Test(args.dataset_path, t1)
valid_target_dataloader = DataLoader(valid_target_dataset,
batch_size=512,
num_workers=6)
load = torch.load(
f"./p3/result/3_2/{args.source}2{args.target}/best_model.pth",
map_location='cpu')
feature_extractor = FeatureExtractor()
feature_extractor.load_state_dict(load['F'])
feature_extractor.cuda()
feature_extractor.eval()
label_predictor = LabelPredictor()
label_predictor.load_state_dict(load['C'])
label_predictor.cuda()
label_predictor.eval()
out_preds = []
out_fnames = []
count=0
for i,(imgs, fnames) in enumerate(valid_target_dataloader):
bsize = imgs.size(0)
imgs = imgs.cuda()
features = feature_extractor(imgs)
class_output = label_predictor(features)
_, preds = class_output.max(1)
preds = preds.detach().cpu()
out_preds.append(preds)
out_fnames += fnames
count+=bsize
print(f"\t [{count}/{len(valid_target_dataloader.dataset)}]",
end=" \r")
out_preds = torch.cat(out_preds)
out_preds = out_preds.cpu().numpy()
d = {'image_name':out_fnames, 'label':out_preds}
df = pd.DataFrame(data=d)
df = df.sort_values('image_name')
df.to_csv(args.out_csv, index=False)
print(f' [Info] finish predicting {args.dataset_path}')
type=str,
default='cuda:0',
help='cpu or cuda:0 or cuda:1')
args = parser.parse_args() if string is None else parser.parse_args(string)
return args
if __name__ == '__main__':
args = parse_args()
wandb.init(config=args, project='dlcv_gan_face')
transform = transforms.Compose([
transforms.Resize(args.img_size),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.5] * 3, [0.5] * 3)
])
train_dataset = Face_Dataset('../hw3_data/face/train', transform)
valid_dataset = Face_Dataset('../hw3_data/face/test', transform)
train_dataloader = DataLoader(train_dataset,
batch_size=args.batch,
shuffle=True,
num_workers=args.num_workers)
valid_dataloader = DataLoader(valid_dataset,
batch_size=args.batch,
num_workers=args.num_workers)
train(args, train_dataloader, valid_dataloader)
def transform_val(self, input_data):
rgb = np.array(input_data["image"]).astype(np.float32)
lidar_depth = np.array(input_data["lidar_depth"]).astype(np.float32)
radar_depth = np.array(input_data["radar_depth"]).astype(np.float32)
if 'index_map' in input_data.keys():
index_map = np.array(input_data["index_map"]).astype(np.int)
# Then, we add model-aware resizing
if self.transform_mode == "DORN":
if cfg.scaling is True:
h, w, _ = tuple((np.array(rgb.shape)).astype(np.int32))
else:
h, w, _ = tuple((np.array(rgb.shape) * 0.5).astype(np.int32))
h_new = self.t_cfg.crop_size_train[0]
w_new = w
resize_image_method = transforms.Resize([h_new, w_new],
interpolation="bilinear")
resize_depth_method = transforms.Resize([h_new, w_new],
interpolation="nearest")
elif self.transform_mode == "sparse-to-dense":
h_new = self.t_cfg.crop_size_train[0]
w_new = self.t_cfg.crop_size_train[1]
resize_image_method = transforms.Resize([h_new, w_new],
interpolation="bilinear")
resize_depth_method = transforms.Resize([h_new, w_new],
interpolation="nearest")
transform_rgb = transforms.Compose([
# resize_image_method,
transforms.CenterCrop(self.t_cfg.crop_size_val)
])
transform_depth = transforms.Compose([
# resize_depth_method,
transforms.CenterCrop(self.t_cfg.crop_size_val)
])
rgb = transform_rgb(rgb)
rgb = rgb / 255.
lidar_depth = transform_depth(lidar_depth)
rgb = np.array(rgb).astype(np.float32)
lidar_depth = np.array(lidar_depth).astype(np.float32)
rgb = to_tensor(rgb)
lidar_depth = to_tensor(lidar_depth)
radar_depth = transform_depth(radar_depth)
radar_depth = np.array(radar_depth).astype(np.float32)
radar_depth = to_tensor(radar_depth)
# Perform transform on index map
if 'index_map' in input_data.keys():
index_map = transform_depth(index_map)
index_map = np.array(index_map).astype(np.int)
index_map = to_tensor(index_map)
index_map = index_map.unsqueeze(0)
# Normalize to imagenet mean and std
if self.transform_mode == "DORN":
rgb = transforms.normalization_imagenet(rgb)
####################
## Filtering part ##
####################
if self.sparsifier == "radar_filtered":
# Indicating the invalid entries
invalid_mask = ~input_data['valid_mask']
invalid_index = np.where(invalid_mask)[0]
invalid_index_mask = invalid_index[None, None,
...].transpose(2, 0, 1)
# Constructing mask for dense depth
dense_mask = torch.ByteTensor(
np.sum(index_map.numpy() == invalid_index_mask, axis=0))
radar_depth_filtered = radar_depth.clone()
radar_depth_filtered[dense_mask.to(torch.bool)] = 0.
radar_depth_filtered = radar_depth_filtered.unsqueeze(0)
# ipdb.set_trace()
####################
######################################
## Filtering using predicted labels ##
######################################
if self.sparsifier == "radar_filtered2":
# ipdb.set_trace()
invalid_mask = ~input_data['pred_labels']
invalid_index = np.where(invalid_mask)[0]
invalid_index_mask = invalid_index[None, None,
...].transpose(2, 0, 1)
dense_mask = torch.ByteTensor(
np.sum(index_map.numpy() == invalid_index_mask, axis=0))
radar_depth_filtered2 = radar_depth.clone()
radar_depth_filtered2[dense_mask.to(torch.bool)] = 0.
radar_depth_filtered2 = radar_depth_filtered2.unsqueeze(0)
######################################
lidar_depth = lidar_depth.unsqueeze(0)
radar_depth = radar_depth.unsqueeze(0)
# Return different data for different modality
################ Input sparsifier #########
if self.modality == "rgb":
inputs = rgb
elif self.modality == "rgbd":
if self.sparsifier == "radar":
# Filter out the the points exceeding max_depth
mask = (radar_depth > self.max_depth)
radar_depth[mask] = 0
inputs = torch.cat((rgb, radar_depth), dim=0)
elif self.sparsifier == "radar_filtered":
# Filter out the points exceeding max_depth
mask = (radar_depth_filtered > self.max_depth)
radar_depth_filtered[mask] = 0
inputs = torch.cat((rgb, radar_depth_filtered), dim=0)
# Using the learned classifyer
elif self.sparsifier == "radar_filtered2":
# Filter out the points exceeding max_depth
mask = (radar_depth_filtered2 > self.max_depth)
radar_depth_filtered2[mask] = 0
inputs = torch.cat((rgb, radar_depth_filtered2), dim=0)
else:
s_depth = self.get_sparse_depth(lidar_depth, radar_depth)
inputs = torch.cat((rgb, s_depth), dim=0)
else:
raise ValueError("[Error] Unsupported modality. Consider ",
self.avail_modality)
labels = lidar_depth
output_dict = {
"rgb": rgb,
"lidar_depth": lidar_depth,
"radar_depth": radar_depth,
"inputs": inputs,
"labels": labels
}
if self.sparsifier == "radar_filtered":
output_dict["radar_depth_filtered"] = radar_depth_filtered
if self.sparsifier == "radar_filtered2":
output_dict["radar_depth_filtered2"] = radar_depth_filtered2
# For 'index_map' compatibility
if 'index_map' in input_data.keys():
output_dict["index_map"] = index_map
return output_dict
def transform_train(self, input_data):
# import ipdb; ipdb.set_trace()
# Fetch the data
rgb = np.array(input_data["image"]).astype(np.float32)
lidar_depth = np.array(input_data["lidar_depth"]).astype(np.float32)
radar_depth = np.array(input_data["radar_depth"]).astype(np.float32)
if 'index_map' in input_data.keys():
index_map = np.array(input_data["index_map"]).astype(np.int)
# Define augmentation factor
scale_factor = np.random.uniform(
self.t_cfg.scale_factor_train[0],
self.t_cfg.scale_factor_train[1]) # random scaling
angle_factor = np.random.uniform(
-self.t_cfg.rotation_factor,
self.t_cfg.rotation_factor) # random rotation degrees
flip_factor = np.random.uniform(0.0,
1.0) < 0.5 # random horizontal flip
# Compose customized transform for RGB and Depth separately
color_jitter = transforms.ColorJitter(0.2, 0.2, 0.2)
resize_image = transforms.Resize(scale_factor,
interpolation="bilinear")
resize_depth = transforms.Resize(scale_factor, interpolation="nearest")
# # First, we uniformly downsample all the images by half
# resize_image_initial = transforms.Resize(0.5, interpolation="bilinear")
# resize_depth_initial = transforms.Resize(0.5, interpolation="nearest")
# Then, we add model-aware resizing
if self.transform_mode == "DORN":
if cfg.scaling is True:
h, w, _ = tuple((np.array(rgb.shape)).astype(np.int32))
else:
h, w, _ = tuple((np.array(rgb.shape) * 0.5).astype(np.int32))
# ipdb.set_trace()
h_new = self.t_cfg.crop_size_train[0]
w_new = w
resize_image_method = transforms.Resize([h_new, w_new],
interpolation="bilinear")
resize_depth_method = transforms.Resize([h_new, w_new],
interpolation="nearest")
elif self.transform_mode == "sparse-to-dense":
h_new = self.t_cfg.crop_size_train[0]
w_new = self.t_cfg.crop_size_train[1]
resize_image_method = transforms.Resize([h_new, w_new],
interpolation="bilinear")
resize_depth_method = transforms.Resize([h_new, w_new],
interpolation="nearest")
# Get the border of random crop
h_scaled, w_scaled = math.floor(h_new * scale_factor), math.floor(
(w_new * scale_factor))
h_bound, w_bound = h_scaled - self.t_cfg.crop_size_train[
0], w_scaled - self.t_cfg.crop_size_train[1]
h_startpoint = round(np.random.uniform(0, h_bound))
w_startpoint = round(np.random.uniform(0, w_bound))
# Compose the transforms for RGB
transform_rgb = transforms.Compose([
transforms.Rotate(angle_factor), resize_image,
transforms.Crop(h_startpoint, w_startpoint,
self.t_cfg.crop_size_train[0],
self.t_cfg.crop_size_train[1]),
transforms.HorizontalFlip(flip_factor)
])
# Compose the transforms for Depth
transform_depth = transforms.Compose([
transforms.Rotate(angle_factor), resize_depth,
transforms.Crop(h_startpoint, w_startpoint,
self.t_cfg.crop_size_train[0],
self.t_cfg.crop_size_train[1]),
transforms.HorizontalFlip(flip_factor)
])
# Perform transform on rgb data
# ToDo: whether we need to - imagenet mean here
rgb = transform_rgb(rgb)
rgb = color_jitter(rgb)
rgb = rgb / 255.
# Perform transform on lidar depth data
lidar_depth /= float(scale_factor)
lidar_depth = transform_depth(lidar_depth)
rgb = np.array(rgb).astype(np.float32)
lidar_depth = np.array(lidar_depth).astype(np.float32)
rgb = to_tensor(rgb)
lidar_depth = to_tensor(lidar_depth)
# Perform transform on radar depth data
radar_depth /= float(scale_factor)
radar_depth = transform_depth(radar_depth)
radar_depth = np.array(radar_depth).astype(np.float32)
radar_depth = to_tensor(radar_depth)
# Perform transform on index map
if 'index_map' in input_data.keys():
index_map = transform_depth(index_map)
index_map = np.array(index_map).astype(np.int)
index_map = to_tensor(index_map)
index_map = index_map.unsqueeze(0)
# Normalize rgb using imagenet mean and std
# ToDo: only do imagenet normalization on DORN
if self.transform_mode == "DORN":
rgb = transforms.normalization_imagenet(rgb)
if self.sparsifier == "radar_filtered":
####################
## Filtering part ##
####################
# Indicating the invalid entries
invalid_mask = ~input_data['valid_mask']
invalid_index = np.where(invalid_mask)[0]
invalid_index_mask = invalid_index[None, None,
...].transpose(2, 0, 1)
# Constructing mask for dense depth
dense_mask = torch.ByteTensor(
np.sum(index_map.numpy() == invalid_index_mask, axis=0))
radar_depth_filtered = radar_depth.clone()
radar_depth_filtered[dense_mask.to(torch.bool)] = 0.
radar_depth_filtered = radar_depth_filtered.unsqueeze(0)
if self.sparsifier == "radar_filtered2":
######################################
## Filtering using predicted labels ##
######################################
invalid_mask = ~input_data['pred_labels']
invalid_index = np.where(invalid_mask)[0]
invalid_index_mask = invalid_index[None, None,
...].transpose(2, 0, 1)
dense_mask = torch.ByteTensor(
np.sum(index_map.numpy() == invalid_index_mask, axis=0))
radar_depth_filtered2 = radar_depth.clone()
radar_depth_filtered2[dense_mask.to(torch.bool)] = 0.
radar_depth_filtered2 = radar_depth_filtered2.unsqueeze(0)
######################################
lidar_depth = lidar_depth.unsqueeze(0)
radar_depth = radar_depth.unsqueeze(0)
# Return different data for different modality
if self.modality == "rgb":
inputs = rgb
elif self.modality == "rgbd":
if self.sparsifier == "radar":
# Filter out the the points exceeding max_depth
mask = (radar_depth > self.max_depth)
radar_depth[mask] = 0
inputs = torch.cat((rgb, radar_depth), dim=0)
# Using the generated groundtruth
elif self.sparsifier == "radar_filtered":
# Filter out the points exceeding max_depth
mask = (radar_depth_filtered > self.max_depth)
radar_depth_filtered[mask] = 0
inputs = torch.cat((rgb, radar_depth_filtered), dim=0)
# Using the learned classifyer
elif self.sparsifier == "radar_filtered2":
# Filter out the points exceeding max_depth
mask = (radar_depth_filtered2 > self.max_depth)
radar_depth_filtered2[mask] = 0
inputs = torch.cat((rgb, radar_depth_filtered2), dim=0)
else:
s_depth = self.get_sparse_depth(lidar_depth, radar_depth)
inputs = torch.cat((rgb, s_depth), dim=0)
else:
raise ValueError("[Error] Unsupported modality. Consider ",
self.avail_modality)
labels = lidar_depth
# Gathering output results
output_dict = {
"rgb": rgb,
"lidar_depth": lidar_depth,
"radar_depth": radar_depth,
"inputs": inputs,
"labels": labels
}
if self.sparsifier == "radar_filtered":
output_dict["radar_depth_filtered"] = radar_depth_filtered
if self.sparsifier == "radar_filtered2":
output_dict["radar_depth_filtered2"] = radar_depth_filtered2
if 'index_map' in input_data.keys():
output_dict["index_map"] = index_map
return output_dict