def get_cars_features(self):
img_name = os.path.join(self.data_dir, self.car_annotations[0][-1][0])
img_height, img_width, _ = np.array(Image.open(img_name)).shape
total_train_size = len(self.car_annotations)
num_of_classes = self.get_num_of_class()
dataset_feature = get_word_list_embeddings('cars', self.car_names)
add_vector_to_file('cars', img_height, img_width, total_train_size, num_of_classes, dataset_feature)
def get_features(self):
img_path = os.path.join(self.root_dir, self.base_folder,
self.sub_directory, self.csv_data.iloc[0, 0])
img_height, img_width, _ = np.array(Image.open(img_path)).shape
total_train_size = len(self.csv_data)
num_of_classes = self.get_num_of_classes()
dataset_feature = get_word_list_embeddings(
'traffic', ['stop', 'turn right', 'turn left'])
add_vector_to_file('traffic', img_height, img_width, total_train_size,
num_of_classes, dataset_feature)
def get_features(self):
img_path, classid = self.final_list[0]
img_height, img_width, _ = np.array(Image.open(img_path)).shape
total_train_size = len(self.final_list)
num_of_classes = self.get_num_of_classes()
dataset_feature = get_word_list_embeddings(
'flower',
['daisy', 'rose', 'sunflower', 'tuplip', 'flowers', 'dandelion'])
add_vector_to_file('flower', img_height, img_width, total_train_size,
num_of_classes, dataset_feature)
def load_FashionMNIST(data_dir, transform=None, validation_split = 0.2, batch_size=256, add_feature=False):
root = os.path.join(data_dir, 'FashionMNIST')
if transform is None:
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Lambda(lambda x: x.repeat(3, 1, 1)),
normalize,
])
download = True
if os.path.exists(os.path.join(root,'processed/test.pt')) and os.path.exists(os.path.join(root,'processed/training.pt')):
download = False
# load the dataset
train_dataset = datasets.FashionMNIST(
root=data_dir, train=True,
download=download, transform=transform,
)
valid_dataset = datasets.FashionMNIST(
root=data_dir, train=True,
download=download, transform=transform,
)
test_dataset = datasets.FashionMNIST(
root=data_dir, train=False,
download=download, transform=transform,
)
num_train = len(train_dataset)
indices = list(range(num_train))
split = int(np.floor(validation_split * num_train))
np.random.shuffle(indices)
train_idx, valid_idx = indices[split:], indices[:split]
if add_feature == True:
image, label = train_dataset.__getitem__(0)
_, img_h, img_w = image.shape
train_size = len(train_dataset)
num_classes = 10
labels = ["T-shirt/top","Trouser","Pullover","Dress","Coat","Sandal","Shirt","Sneaker","Bag"]
word_embed = get_word_list_embeddings('fashion-mnist', labels)
add_vector_to_file("FashionMnist",img_h, img_w, train_size,num_classes, word_embed)
train_sampler = SubsetRandomSampler(train_idx)
valid_sampler = SubsetRandomSampler(valid_idx)
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=batch_size, sampler=train_sampler,
num_workers=4, pin_memory=False,
)
valid_loader = torch.utils.data.DataLoader(
valid_dataset, batch_size=batch_size, sampler=valid_sampler,
num_workers=4, pin_memory=False,
)
test_loader = torch.utils.data.DataLoader(
test_dataset, batch_size=batch_size,
num_workers=4, pin_memory=False,
)
print("Got total number test data ", len(test_dataset))
return {'train_loader': train_loader, 'validation_loader': valid_loader,
'test_loader': test_loader, 'num_classes': 10, 'name': 'fashion-mnist'}
def load_cifar100(data_dir, transform=None, validation_split=0.2, batch_size=256, add_feature=False):
if transform is None:
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
normalize,
])
download = True
if os.path.exists(os.path.join(data_dir,'cifar-100-python.tar.gz')):
download = False
# load the dataset
train_dataset = datasets.CIFAR100(
root=data_dir, train=True,
download=download, transform=transform,
)
valid_dataset = datasets.CIFAR100(
root=data_dir, train=True,
download=download, transform=transform,
)
test_dataset = datasets.CIFAR100(
root=data_dir, train=False,
download=download, transform=transform,
)
num_train = len(train_dataset)
indices = list(range(num_train))
split = int(np.floor(validation_split * num_train))
np.random.shuffle(indices)
train_idx, valid_idx = indices[split:], indices[:split]
if add_feature:
image, label = train_dataset.__getitem__(0)
_, img_h, img_w = image.shape
train_size = len(train_dataset)
num_classes = 100
fp = open(data_dir + '/cifar-100-python/meta','rb')
labels = list(pickle.load(fp)['fine_label_names'])
word_embed = get_word_list_embeddings('cifar100', labels)
add_vector_to_file("Cifar100",img_h, img_w, train_size,num_classes, word_embed)
train_sampler = SubsetRandomSampler(train_idx)
valid_sampler = SubsetRandomSampler(valid_idx)
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=batch_size, sampler=train_sampler,
num_workers=4, pin_memory=False,
)
valid_loader = torch.utils.data.DataLoader(
valid_dataset, batch_size=batch_size, sampler=valid_sampler,
num_workers=4, pin_memory=False,
)
test_loader = torch.utils.data.DataLoader(
test_dataset, batch_size=batch_size,
num_workers=4, pin_memory=False,
)
print("Got total number test data ", len(test_dataset))
return {'train_loader': train_loader, 'validation_loader': valid_loader,
'test_loader': test_loader, 'num_classes': 100, 'name': 'cifar100'}
