def test_siamesize_train_ds_epoch():
n_epochs = 10
dataset = DeepFashionDataset(root_dir, 'val')
# make it trainable
dataset.train = True
siamese_ds = Siamesize(dataset)
for epoch in range(n_epochs):
for idx1 in range(len(siamese_ds)):
idx2, is_similar = siamese_ds._get_idx2_and_target(idx1)
def test_siamesize_train_ds():
"""
One epoch testing
"""
dataset = DeepFashionDataset(root_dir, 'train')
siamese_ds = Siamesize(dataset)
targets = list()
for idx1 in range(len(siamese_ds)):
idx2, is_similar = siamese_ds._get_idx2_and_target(idx1)
# store the targets
targets.append(is_similar)
# empirically check probability of the target getting 0 or 1
prob = np.mean(targets)
assert 0.48 <= prob <= 0.52
def test_siamesize_train_ds_basic():
"""
Basic testing
"""
dataset = DeepFashionDataset(root_dir, 'train')
siamese_ds = Siamesize(dataset)
targets = list()
for idx1 in range(5000):
idx2, is_similar = siamese_ds._get_idx2_and_target(idx1)
assert is_similar in [0, 1]
assert idx1 != idx2
# store the targets
targets.append(is_similar)
# empirically check probability of the target getting 0 or 1
prob = np.mean(targets)
assert 0.4 <= prob <= 0.6
def prepare_data_loaders(self):
"""
Our target is to construct embbeding space. Therefore we use the "test set"
as validation
"""
# alias
cfg = FashionMNISTConfig
# data transform
data_transform = Compose(
[ToTensor(), Normalize((cfg.mean, ), (cfg.std, ))])
# ----------------------------
# Consturct data loader
# ----------------------------
ds_kwargs = {'root': cfg.root, 'transform': data_transform}
train_ds = FashionMNIST(train=True, download=True, **ds_kwargs)
val_ds = FashionMNIST(train=False, download=False, **ds_kwargs)
# ---------------------------------------------------
# Returns pairs of images and target same/different
# ---------------------------------------------------
siamese_train_ds = Siamesize(train_ds)
siamese_test_ds = Siamesize(val_ds)
# ----------------------------
# Consturct loader
# ----------------------------
self.train_loader = DataLoader(train_ds,
shuffle=False,
batch_size=self.hparams.batch_size)
self.val_loader = DataLoader(val_ds,
shuffle=False,
batch_size=self.hparams.batch_size)
batch_size = self.hparams.batch_size
self.siamese_train_loader = DataLoader(siamese_train_ds,
batch_size=batch_size,
shuffle=True)
self.siamese_val_loader = torch.utils.data.DataLoader(
siamese_test_ds, batch_size=batch_size, shuffle=False)
self.train_loader_len = len(self.siamese_train_loader)
def test_siamesize_mnist_basic():
cfg = FashionMNISTConfig
ds_kwargs = {
'root': cfg.root,
}
train_ds = FashionMNIST(train=True, download=True, **ds_kwargs)
siamese_ds = Siamesize(train_ds)
targets = list()
for idx1 in range(5000):
idx2, is_similar = siamese_ds._get_idx2_and_target(idx1)
assert is_similar in [0, 1]
assert idx1 != idx2
# store the targets
targets.append(is_similar)
# empirically check probability of the target getting 0 or 1
prob = np.mean(targets)
assert 0.4 <= prob <= 0.6
def datasets(self):
if self._datasets is None:
trans = Compose(
[Resize(cfg.sizes),
ToTensor(),
Normalize(cfg.mean, cfg.std)])
# dataset
train_ds = DeepFashionDataset(cfg.root_dir,
'train',
transform=trans)
val_ds = DeepFashionDataset(cfg.root_dir, 'val', transform=trans)
test_ds = DeepFashionDataset(cfg.root_dir, 'test', transform=trans)
siam_train_ds = Siamesize(train_ds)
siam_val_ds = Siamesize(val_ds)
siam_test_ds = Siamesize(test_ds)
# Subset if needed
if self._debug:
train_samples = np.random.choice(len(train_ds),
1000,
replace=False)
val_samples = np.random.choice(len(val_ds), 100, replace=False)
# Subset the datasets
train_ds = Subset(train_ds, train_samples)
val_ds = Subset(val_ds, val_samples)
test_ds = Subset(test_ds, val_samples)
siam_train_ds = Subset(siam_train_ds, train_samples)
siam_val_ds = Subset(siam_val_ds, val_samples)
siam_test_ds = Subset(siam_test_ds, val_samples)
# -------------------------------------------------------
# pack them up
self._datasets = {
"train": train_ds,
"val": val_ds,
"test": test_ds,
"siam_train": siam_train_ds,
"siam_val": siam_val_ds,
"siam_test": siam_test_ds,
}
return self._datasets
def test_siamesize_test_ds():
dataset = DeepFashionDataset(root_dir, 'test')
siamese_ds = Siamesize(dataset)
assert not siamese_ds.train
# first round
pairs = list()
targets = list()
for idx1 in range(len(siamese_ds)):
idx2, is_similar = siamese_ds._get_idx2_and_target(idx1)
assert is_similar in [0, 1]
assert idx1 != idx2
# store the targets
pairs.append((idx1, idx2))
targets.append(is_similar)
prob = np.mean(targets)
assert 0.48 <= prob <= 0.52
# second round
for idx1 in range(len(siamese_ds)):
idx2, is_similar = siamese_ds._get_idx2_and_target(idx1)
assert (idx1, idx2) in pairs
def prepare_data_loaders(self):
"""
Our target is to construct embbeding space. Therefore we use the "test set"
as validation
"""
# data transform
trans = Compose([
Resize(cfg.sizes),
ToTensor(),
Normalize(cfg.mean, cfg.std),
])
self.train_ds = DeepFashionDataset(cfg.root_dir,
'train',
transform=trans)
self.val_ds = DeepFashionDataset(cfg.root_dir, 'val', transform=trans)
# ---------------------------------------------------
# Returns pairs of images and target same/different
# ---------------------------------------------------
siamese_train_ds = Siamesize(self.train_ds)
# siamese_val_ds = Siamesize(self.val_ds)
if self._debug:
# For debug use
self.train_ds = Subset(self.train_ds, range(5000))
self.val_ds = Subset(self.val_ds, range(1000))
siamese_train_ds = Subset(siamese_train_ds, range(5000))
# siamese_val_ds = Subset(siamese_train_ds, range(1000))
# ----------------------------
# Consturct data loader
# ----------------------------
loader_kwargs = {
'pin_memory': True,
'batch_size': self.hparams.batch_size,
'num_workers': os.cpu_count(),
}
self.siamese_train_loader = DataLoader(siamese_train_ds,
**loader_kwargs)
self.siamese_val_loader = DataLoader(siamese_train_ds, **loader_kwargs)
# Set the lenght for tqdm
self.train_loader_len = len(self.siamese_train_loader)
from torchvision.transforms import Compose
from torchvision.transforms import Resize
from torchvision.transforms import ToTensor
from torchvision.transforms import Normalize
from config.deep_fashion import DeepFashionConfig as cfg
from torch.utils.data import DataLoader
from utils.datasets import Siamesize
trans = Compose([
Resize(cfg.sizes),
ToTensor(),
Normalize(cfg.mean, cfg.std),
])
# dataset
train_ds = DeepFashionDataset(cfg.root_dir, 'train', transform=trans)
val_ds = DeepFashionDataset(cfg.root_dir, 'val', transform=trans)
siamese_train_ds = Siamesize(train_ds)
# loader
loader_kwargs = {
'pin_memory': True,
'batch_size': 100,
'num_workers': 4,
}
s_train_loader = DataLoader(siamese_train_ds, **loader_kwargs)
train_loader = DataLoader(val_ds, **loader_kwargs)
val_loader = DataLoader(val_ds, **loader_kwargs)
# Optim
import torch.optim as optim
optimizer = optim.Adam(clsf_net.parameters(), lr=1e-4)
# Loss functions
import torch.nn as nn
from torchvision.transforms import Resize
from torchvision.transforms import ToTensor
from torchvision.transforms import Normalize
from config.deep_fashion import DeepFashionConfig as cfg
from torch.utils.data import DataLoader
from torch.utils.data import Subset
from torch.utils.data import TensorDataset
from utils.datasets import Siamesize
trans = Compose([Resize(cfg.sizes), ToTensor(),
Normalize(cfg.mean, cfg.std), ])
# dataset
train_ds = DeepFashionDataset(
cfg.root_dir, 'train', transform=trans)
val_ds = DeepFashionDataset(
cfg.root_dir, 'val', transform=trans)
siamese_train_ds = Siamesize(train_ds)
siamese_val_ds = Siamesize(val_ds)
if False: # For overfitting test
train_samples = np.random.choice(len(train_ds), 300, replace=False)
val_samples = np.random.choice(len(val_ds), 100, replace=False)
train_ds = Subset(train_ds, train_samples)
val_ds = Subset(val_ds, val_samples)
siamese_train_ds = Subset(siamese_train_ds, train_samples)
siamese_val_ds = Subset(siamese_val_ds, val_samples)
# loader
import os
batch_size = 128
loader_kwargs = {
'pin_memory': pin_memory,
'batch_size': batch_size,