def __init__(self, root=None, batch_size=100,
train_shuffle=True, val_shuffle=False, test_shuffle=False,
train_aug=True, val_aug=False, test_aug=False,
size=32, color=True, val_ratio=0.5, aug_list=None, domain_id=None, sample_ratio=1.0, resize_pad=False, double_aug=False):
assert(sample_ratio == 1.0)
assert(resize_pad == False)
## default tforms
tforms_dft = [
Map2Tuple(),
Resize([size, size]) if size!=32 else Identity(),
Identity() if color else RGB2Gray(),
Float(),
Normalize(0.5, 0.5),
#PerImageStd(),
]
## data augmentation ops
if aug_list is not None:
tforms_aug = [DuplicateX() if double_aug else Identity()]
tforms_aug += [aug_op for aug_op in compose_augment_seq(aug_list, is_training=True)]
else:
tforms_aug = []
## domain label
tforms_dom = [DomainLabel(domain_id)] if domain_id is not None else []
## tforms
tforms_train = tforms_dft + tforms_aug if train_aug else tforms_dft
tforms_val = tforms_dft + tforms_aug if val_aug else tforms_dft
tforms_test = tforms_dft + tforms_aug if test_aug else tforms_dft
if domain_id is not None:
tforms_train_dom = tforms_train + tforms_dom
tforms_val_dom = tforms_val + tforms_dom
tforms_test_dom = tforms_test + tforms_dom
## load data
tf.random.set_seed(0)
ds_train = tfds.load('svhn_cropped', split='train', shuffle_files=True)
ds_val = tfds.load('svhn_cropped', split='test[:%d'%(round(val_ratio*100.))+'%]', shuffle_files=True)
ds_test = tfds.load('svhn_cropped', split='test[%d'%(round(val_ratio*100.))+'%:]', shuffle_files=True)
## construct data loaders
self.train = self._init_loader(ds_train, batch_size*10, train_shuffle, batch_size, tforms_train)
self.val = self._init_loader(ds_val, batch_size*10, val_shuffle, batch_size, tforms_val)
self.test = self._init_loader(ds_test, batch_size*10, test_shuffle, batch_size, tforms_test)
if domain_id is not None:
self.train_dom = self._init_loader(ds_train, batch_size*10, train_shuffle, batch_size, tforms_train_dom)
self.val_dom = self._init_loader(ds_val, batch_size*10, val_shuffle, batch_size, tforms_val_dom)
self.test_dom = self._init_loader(ds_test, batch_size*10, test_shuffle, batch_size, tforms_test_dom)
def __init__(self,
root=None,
batch_size=100,
train_shuffle=True,
val_shuffle=False,
test_shuffle=False,
train_aug=True,
val_aug=False,
test_aug=False,
size=32,
color=True,
val_ratio=0.5,
aug_list=None,
domain_id=None,
sample_ratio=1.0,
resize_pad=False,
double_aug=False):
assert (size >= 28)
## default tforms
tforms_dft = [
Mat2RGB() if color else Mat2Gray(),
#Pad([size-28, size-28]) if size>28 else Identity(),
#Resize([size, size]) if size!=28 else Identity(),
Identity() if size == 28 else
Pad([size - 28, size -
28]) if resize_pad else Resize([size, size]),
Cast(tf.float32, tf.int32),
Scaling(1.0 / 255.0),
Normalize(0.5, 0.5),
#PerImageStd(),
]
## data augmentation ops
if aug_list is not None:
tforms_aug = [DuplicateX() if double_aug else Identity()]
tforms_aug += [
aug_op
for aug_op in compose_augment_seq(aug_list, is_training=True)
]
else:
tforms_aug = []
## domain label
tforms_dom = [DomainLabel(domain_id)] if domain_id is not None else []
## tforms
tforms_train = tforms_dft + tforms_aug if train_aug else tforms_dft
tforms_val = tforms_dft + tforms_aug if val_aug else tforms_dft
tforms_test = tforms_dft + tforms_aug if test_aug else tforms_dft
if domain_id is not None:
tforms_train_dom = tforms_train + tforms_dom
tforms_val_dom = tforms_val + tforms_dom
tforms_test_dom = tforms_test + tforms_dom
##TODO: tfds?
## load data
(x_train, y_train), (x_test,
y_test) = tf.keras.datasets.mnist.load_data()
(x_val, y_val), (x_test,
y_test) = self._split_data(x_test, y_test, val_ratio,
True)
## reduce train/val samples (shuffle before sampling)
x_train, y_train = shuffle_labeled_examples(x_train, y_train)
x_train, y_train = x_train[:round(x_train.shape[0] * sample_ratio
)], y_train[:round(y_train.shape[0] *
sample_ratio)]
x_val, y_val = shuffle_labeled_examples(x_val, y_val)
x_val, y_val = x_val[:round(x_val.shape[0] * sample_ratio
)], y_val[:round(y_val.shape[0] *
sample_ratio)]
## init meta data
self.n_train = y_train.shape[0]
self.n_val = y_val.shape[0]
self.n_test = y_test.shape[0]
## construct data loaders
self.train = self._init_loader(
tf.data.Dataset.from_tensor_slices((x_train, y_train)),
self.n_train, train_shuffle, batch_size, tforms_train)
self.val = self._init_loader(
tf.data.Dataset.from_tensor_slices((x_val, y_val)), self.n_val,
val_shuffle, batch_size, tforms_val)
self.test = self._init_loader(
tf.data.Dataset.from_tensor_slices((x_test, y_test)), self.n_test,
test_shuffle, batch_size, tforms_test)
if domain_id is not None:
self.train_dom = self._init_loader(
tf.data.Dataset.from_tensor_slices((x_train, y_train)),
self.n_train, train_shuffle, batch_size, tforms_train_dom)
self.val_dom = self._init_loader(
tf.data.Dataset.from_tensor_slices((x_val, y_val)), self.n_val,
val_shuffle, batch_size, tforms_val_dom)
self.test_dom = self._init_loader(
tf.data.Dataset.from_tensor_slices((x_test, y_test)),
self.n_test, test_shuffle, batch_size, tforms_test_dom)
def __init__(self,
root,
batch_size=100,
train_shuffle=True,
val_shuffle=False,
test_shuffle=False,
train_aug=True,
val_aug=False,
test_aug=False,
size=32,
val_ratio=0.5,
aug_list=None,
domain_id=None):
## default tforms
tforms_dft = [
File2Tensor(3),
Resize([size, size]),
Cast(tf.float32, tf.int32),
Scaling(1.0 / 255.0),
Normalize(0.5, 0.5),
]
## data augmentation ops
tforms_aug = [
aug_op
for aug_op in compose_augment_seq(aug_list, is_training=True)
] if aug_list is not None else []
## domain label
tforms_dom = [DomainLabel(domain_id)] if domain_id is not None else []
## tforms
tforms_train = tforms_dft + tforms_aug if train_aug else tforms_dft
tforms_val = tforms_dft + tforms_aug if val_aug else tforms_dft
tforms_test = tforms_dft + tforms_aug if test_aug else tforms_dft
if domain_id is not None:
tforms_train_dom = tforms_train + tforms_dom
tforms_val_dom = tforms_val + tforms_dom
tforms_test_dom = tforms_test + tforms_dom
## load data
np.random.seed(0)
tr_fns = glob.glob(os.path.join(root, 'imgs_train', '*/*.jpg'))
te_fns = glob.glob(os.path.join(root, 'imgs_valid', '*/*.jpg'))
np.random.shuffle(tr_fns)
np.random.shuffle(te_fns)
val_fns = te_fns[:round(len(te_fns) * val_ratio)]
te_fns = te_fns[round(len(te_fns) * val_ratio):]
ds_train = tf.data.Dataset.from_tensor_slices(tr_fns)
ds_val = tf.data.Dataset.from_tensor_slices(val_fns)
ds_test = tf.data.Dataset.from_tensor_slices(te_fns)
## construct data loaders
self.train = self._init_loader(ds_train, batch_size * 10,
train_shuffle, batch_size, tforms_train)
self.val = self._init_loader(ds_val, batch_size * 10, val_shuffle,
batch_size, tforms_val)
self.test = self._init_loader(ds_test, batch_size * 10, test_shuffle,
batch_size, tforms_test)
if domain_id is not None:
self.train_dom = self._init_loader(ds_train, batch_size * 10,
train_shuffle, batch_size,
tforms_train_dom)
self.val_dom = self._init_loader(ds_val, batch_size * 10,
val_shuffle, batch_size,
tforms_val_dom)
self.test_dom = self._init_loader(ds_test, batch_size * 10,
test_shuffle, batch_size,
tforms_test_dom)
def __init__(self,
root,
batch_size=100,
train_shuffle=True,
val_shuffle=False,
test_shuffle=False,
train_aug=True,
val_aug=False,
test_aug=False,
size=32,
color=True,
val_ratio=0.5,
aug_list=None,
domain_id=None,
sample_ratio=1.0,
resize_pad=False,
double_aug=False):
assert (size >= 28)
assert (resize_pad == False)
## default tforms
tforms_dft = [
Mat2RGB() if color else Mat2Gray(),
#Pad([size-28, size-28]) if size>28 else Identity(),
Resize([size, size]) if size != 16 else Identity(),
Normalize(0.5, 0.5),
Cast(None, tf.int32)
]
## data augmentation ops
if aug_list is not None:
tforms_aug = [DuplicateX() if double_aug else Identity()]
tforms_aug += [
aug_op
for aug_op in compose_augment_seq(aug_list, is_training=True)
]
else:
tforms_aug = []
## domain label
tforms_dom = [DomainLabel(domain_id)] if domain_id is not None else []
## tforms
tforms_train = tforms_dft + tforms_aug if train_aug else tforms_dft
tforms_val = tforms_dft + tforms_aug if val_aug else tforms_dft
tforms_test = tforms_dft + tforms_aug if test_aug else tforms_dft
if domain_id is not None:
tforms_train_dom = tforms_train + tforms_dom
tforms_val_dom = tforms_val + tforms_dom
tforms_test_dom = tforms_test + tforms_dom
## load data
with h5py.File(os.path.join(root, 'usps.h5'), 'r') as hf:
train = hf.get('train')
x_train, y_train = np.reshape(train.get('data')[:],
(-1, 16, 16)), train.get('target')[:]
test = hf.get('test')
x_test, y_test = np.reshape(test.get('data')[:],
(-1, 16, 16)), test.get('target')[:]
(x_val, y_val), (x_test,
y_test) = self._split_data(x_test, y_test, val_ratio,
True)
## reduce train/val samples (shuffle before sampling)
x_train, y_train = shuffle_labeled_examples(x_train, y_train)
x_train, y_train = x_train[:round(x_train.shape[0] * sample_ratio
)], y_train[:round(y_train.shape[0] *
sample_ratio)]
x_val, y_val = shuffle_labeled_examples(x_val, y_val)
x_val, y_val = x_val[:round(x_val.shape[0] * sample_ratio
)], y_val[:round(y_val.shape[0] *
sample_ratio)]
## init meta data
self.n_train = y_train.shape[0]
self.n_val = y_val.shape[0]
self.n_test = y_test.shape[0]
## construct data loaders
self.train = self._init_loader(
tf.data.Dataset.from_tensor_slices((x_train, y_train)),
self.n_train, train_shuffle, batch_size, tforms_train)
self.val = self._init_loader(
tf.data.Dataset.from_tensor_slices((x_val, y_val)), self.n_val,
val_shuffle, batch_size, tforms_val)
self.test = self._init_loader(
tf.data.Dataset.from_tensor_slices((x_test, y_test)), self.n_test,
test_shuffle, batch_size, tforms_test)
if domain_id is not None:
self.train_dom = self._init_loader(
tf.data.Dataset.from_tensor_slices((x_train, y_train)),
self.n_train, train_shuffle, batch_size, tforms_train_dom)
self.val_dom = self._init_loader(
tf.data.Dataset.from_tensor_slices((x_val, y_val)), self.n_val,
val_shuffle, batch_size, tforms_val_dom)
self.test_dom = self._init_loader(
tf.data.Dataset.from_tensor_slices((x_test, y_test)),
self.n_test, test_shuffle, batch_size, tforms_test_dom)
def __init__(self,
root,
batch_size=100,
train_shuffle=True,
val_shuffle=False,
test_shuffle=False,
train_aug=True,
val_aug=False,
test_aug=False,
color=True,
size=32,
aug_list=None,
domain_id=None):
assert (size == 32)
## default tforms
tforms_dft = [
Cast(tf.float32, tf.int32),
Scaling(1.0 / 255.0),
Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
]
## default augment
tforms_size_aug_rnd = [
#RandomCrop((28, 28, 3)),
RandomHorizontalFlip(),
]
tforms_size_aug_no_rnd = [
#CenterCrop(28, 32),
]
## data augmentation ops
tforms_aug = [
aug_op
for aug_op in compose_augment_seq(aug_list, is_training=True)
] if aug_list is not None else []
## domain label
tforms_dom = [DomainLabel(domain_id)] if domain_id is not None else []
## tforms
tforms_train = tforms_dft + tforms_aug if train_aug else tforms_dft
tforms_train += tforms_size_aug_rnd if train_shuffle else tforms_size_aug_no_rnd
tforms_val = tforms_dft + tforms_aug if val_aug else tforms_dft
tforms_val += tforms_size_aug_rnd if val_shuffle else tforms_size_aug_no_rnd
tforms_test = tforms_dft + tforms_aug if test_aug else tforms_dft
tforms_test += tforms_size_aug_rnd if test_shuffle else tforms_size_aug_no_rnd
if domain_id is not None:
tforms_train_dom = tforms_train + tforms_dom
tforms_val_dom = tforms_val + tforms_dom
tforms_test_dom = tforms_test + tforms_dom
## construct data loaders
self.train = DataFolderLoader(os.path.join(root, 'train'), batch_size,
train_shuffle, tforms_train)
self.val = DataFolderLoader(os.path.join(root, 'val'), batch_size,
val_shuffle, tforms_val)
self.test = DataFolderLoader(os.path.join(root, 'test'), batch_size,
test_shuffle, tforms_test)
if domain_id is not None:
self.train_dom = DataFolderLoader(os.path.join(root, 'train'),
batch_size, train_shuffle,
tforms_train_dom)
self.val_dom = DataFolderLoader(os.path.join(root,
'val'), batch_size,
val_shuffle, tforms_val_dom)
self.test_dom = DataFolderLoader(os.path.join(root,
'test'), batch_size,
test_shuffle, tforms_test_dom)