def get_cars_features(self): img_name = os.path.join(self.data_dir, self.car_annotations[0][-1][0]) img_height, img_width, _ = np.array(Image.open(img_name)).shape total_train_size = len(self.car_annotations) num_of_classes = self.get_num_of_class() dataset_feature = get_word_list_embeddings('cars', self.car_names) add_vector_to_file('cars', img_height, img_width, total_train_size, num_of_classes, dataset_feature)
def get_features(self): img_path = os.path.join(self.root_dir, self.base_folder, self.sub_directory, self.csv_data.iloc[0, 0]) img_height, img_width, _ = np.array(Image.open(img_path)).shape total_train_size = len(self.csv_data) num_of_classes = self.get_num_of_classes() dataset_feature = get_word_list_embeddings( 'traffic', ['stop', 'turn right', 'turn left']) add_vector_to_file('traffic', img_height, img_width, total_train_size, num_of_classes, dataset_feature)
def get_features(self): img_path, classid = self.final_list[0] img_height, img_width, _ = np.array(Image.open(img_path)).shape total_train_size = len(self.final_list) num_of_classes = self.get_num_of_classes() dataset_feature = get_word_list_embeddings( 'flower', ['daisy', 'rose', 'sunflower', 'tuplip', 'flowers', 'dandelion']) add_vector_to_file('flower', img_height, img_width, total_train_size, num_of_classes, dataset_feature)
def load_FashionMNIST(data_dir, transform=None, validation_split = 0.2, batch_size=256, add_feature=False): root = os.path.join(data_dir, 'FashionMNIST') if transform is None: normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) transform = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Lambda(lambda x: x.repeat(3, 1, 1)), normalize, ]) download = True if os.path.exists(os.path.join(root,'processed/test.pt')) and os.path.exists(os.path.join(root,'processed/training.pt')): download = False # load the dataset train_dataset = datasets.FashionMNIST( root=data_dir, train=True, download=download, transform=transform, ) valid_dataset = datasets.FashionMNIST( root=data_dir, train=True, download=download, transform=transform, ) test_dataset = datasets.FashionMNIST( root=data_dir, train=False, download=download, transform=transform, ) num_train = len(train_dataset) indices = list(range(num_train)) split = int(np.floor(validation_split * num_train)) np.random.shuffle(indices) train_idx, valid_idx = indices[split:], indices[:split] if add_feature == True: image, label = train_dataset.__getitem__(0) _, img_h, img_w = image.shape train_size = len(train_dataset) num_classes = 10 labels = ["T-shirt/top","Trouser","Pullover","Dress","Coat","Sandal","Shirt","Sneaker","Bag"] word_embed = get_word_list_embeddings('fashion-mnist', labels) add_vector_to_file("FashionMnist",img_h, img_w, train_size,num_classes, word_embed) train_sampler = SubsetRandomSampler(train_idx) valid_sampler = SubsetRandomSampler(valid_idx) train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=4, pin_memory=False, ) valid_loader = torch.utils.data.DataLoader( valid_dataset, batch_size=batch_size, sampler=valid_sampler, num_workers=4, pin_memory=False, ) test_loader = torch.utils.data.DataLoader( test_dataset, batch_size=batch_size, num_workers=4, pin_memory=False, ) print("Got total number test data ", len(test_dataset)) return {'train_loader': train_loader, 'validation_loader': valid_loader, 'test_loader': test_loader, 'num_classes': 10, 'name': 'fashion-mnist'}
def load_cifar100(data_dir, transform=None, validation_split=0.2, batch_size=256, add_feature=False): if transform is None: normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) transform = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), normalize, ]) download = True if os.path.exists(os.path.join(data_dir,'cifar-100-python.tar.gz')): download = False # load the dataset train_dataset = datasets.CIFAR100( root=data_dir, train=True, download=download, transform=transform, ) valid_dataset = datasets.CIFAR100( root=data_dir, train=True, download=download, transform=transform, ) test_dataset = datasets.CIFAR100( root=data_dir, train=False, download=download, transform=transform, ) num_train = len(train_dataset) indices = list(range(num_train)) split = int(np.floor(validation_split * num_train)) np.random.shuffle(indices) train_idx, valid_idx = indices[split:], indices[:split] if add_feature: image, label = train_dataset.__getitem__(0) _, img_h, img_w = image.shape train_size = len(train_dataset) num_classes = 100 fp = open(data_dir + '/cifar-100-python/meta','rb') labels = list(pickle.load(fp)['fine_label_names']) word_embed = get_word_list_embeddings('cifar100', labels) add_vector_to_file("Cifar100",img_h, img_w, train_size,num_classes, word_embed) train_sampler = SubsetRandomSampler(train_idx) valid_sampler = SubsetRandomSampler(valid_idx) train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=4, pin_memory=False, ) valid_loader = torch.utils.data.DataLoader( valid_dataset, batch_size=batch_size, sampler=valid_sampler, num_workers=4, pin_memory=False, ) test_loader = torch.utils.data.DataLoader( test_dataset, batch_size=batch_size, num_workers=4, pin_memory=False, ) print("Got total number test data ", len(test_dataset)) return {'train_loader': train_loader, 'validation_loader': valid_loader, 'test_loader': test_loader, 'num_classes': 100, 'name': 'cifar100'}