def build_input_transform(rng, h, w, upscale_factor: int):
return Compose([
ColorJitter(rng),
RandomApply([AddNoise()], 0.3),
RandomChoice([
RandomApply([MotionBlur()], 0.3),
RandomApply([DefocusBlur()], 0.3)
]),
RandomApply([JpegCompress()], 0.3),
Resize((h // upscale_factor, w // upscale_factor),
interpolation=Image.BICUBIC),
ToTensor(),
])
def __init__(self,
root=None,
label_map={
0: 0,
255: 1
},
fg_imgs=None,
annfile=None,
val=False,
test_mode=False):
self.root = root
self.label_map = label_map
self.num_classes = len(list(label_map.keys()))
self.test_mode = test_mode
self.val = val
if not test_mode:
assert fg_imgs is None
assert annfile is not None
with open(annfile, 'r') as f:
lines = f.readlines()
self.imgpaths = [l.rstrip('\n') for l in lines]
# transforms
self.label_map_transform = LabelMap(label_map)
if not val:
self.spatial_transforms = Compose([
RandomHorizontalFlip(),
RandomVerticalFlip(),
RandomAffine(degrees=45,
translate=(0.2, 0.2),
scale=(0.8, 1.2),
shear=10,
fillcolor=103)
])
self.color_transforms = Compose([
RandomApply([ColorJitter(brightness=0.2, contrast=0.2)],
p=0.8),
RandomApply([GaussianBlur(sigma=[.1, 1.])], p=0.2)
])
## test
else:
assert fg_imgs is not None
assert annfile is None
self.imgs = fg_imgs # the results of robust PCA
## mutual transform
self.to_tensor = Compose([
ToTensor(),
# Normalize(mean=[0.4026], std=[0.4025])
])
def get_transform(
target_size=(288, 288),
transform_list='random_crop, horizontal_flip', # random_crop | keep_aspect
augment_ratio=0.5,
is_train=True,
):
transform = list()
transform_list = transform_list.split(', ')
augments = list()
for transform_name in transform_list:
if transform_name == 'random_crop':
scale = (0.5, 1.0) if is_train else (0.8, 1.0)
transform.append(RandomResizedCrop(target_size, scale=(0.8, 1.0)))
elif transform_name == 'keep_aspect':
transform.append(KeepAsepctResize(target_size))
elif transform_name == 'horizontal_flip':
augments.append(RandomHorizontalFlip())
elif transform_name == 'vertical_flip':
augments.append(RandomVerticalFlip())
elif transform_name == 'random_rotate':
augments.append(RandomRotate())
elif transform_name == 'color_jitter':
brightness = 0.1 if is_train else 0.05
contrast = 0.1 if is_train else 0.05
augments.append(
ColorJitter(
brightness=brightness,
contrast=contrast,
saturation=0,
hue=0,
))
transform.append(RandomApply(augments, p=augment_ratio))
return Compose(transform)
def _get_img_transformations(means: typing.List[float],
stds: typing.List[float]
) -> Compose:
""" Get all transformations to carry out on image.
Helpful while creating GestureDataset.
Arguments
----------
means : typing.Tuple
Mean values for data normalization
stds : typing.Tuple
Std values for data normalization
Returns
-------
torchvision.transforms.Compose
composition of transformations
"""
return Compose([
Resize((512, 256)),
ColorJitter(brightness=0.1, contrast=0.1), # will be applied RANDOMLY
RandomVerticalFlip(p=0.1),
ToTensor(),
Normalize(means, stds),
RandomApply([Lambda(lambda x: x + torch.randn_like(x))], p=0.3) # noise
])
def get_train_transforms(img_size: int) -> Compose:
"""Returns data transformations/augmentations for train dataset.
Args:
img_size: The resolution of the input image (img_size x img_size)
"""
return Compose([
RandomApply([
ColorJitter(brightness=0.3, contrast=0.01, saturation=0.01, hue=0),
RandomAffine(0.1,
translate=(0.04, 0.04),
scale=(0.04, 0.04),
shear=0.01,
resample=2),
#Grayscale(num_output_channels=3),
#RandomCrop(30),
RandomPerspective(0.1)
]),
Resize([img_size, img_size], interpolation=3),
ToTensor(),
#RandomApply([
#RandomErasing(p=0.2, scale=(0.02, 0.33), ratio=(0.3, 3.3))
#]),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
])
def __init__(self,
base_dataset,
img_size=(32, 32),
color_dist_strength=0.5):
"""
Parameters
----------
base_dataset : Dataset
img_size : tuple
color_dist_strength : float
"""
self.base_dataset = base_dataset
# get color distortion random transform, Color distortion (see Pytorch
# pseudocode in arXiv:2002.05709)
color_jitter = ColorJitter(
0.8 * color_dist_strength,
0.8 * color_dist_strength,
0.8 * color_dist_strength,
0.2 * color_dist_strength,
)
rnd_color_jitter = RandomApply([color_jitter], p=0.8)
rnd_gray = RandomGrayscale(p=0.2)
self.transform = Compose([
RandomResizedCrop(img_size), rnd_color_jitter, rnd_gray,
ToTensor()
])
def get_image_transformer(self):
transformations = [
ColorJitter(brightness=0.3, contrast=0.3, saturation=0.1, hue=0.1),
RandomGrayscale(p=0.5),
RandomRotation(degrees=[0., 45.]),
RandomResizedCrop(size=[224, 224], scale=(0.3, 1.0))
]
return Compose([RandomApply(transformations, p=0.7), ToTensor()])
def test_augment():
from torchvision.transforms import RandomApply, ColorJitter, RandomAffine, ToTensor, Normalize
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
fill_color = (255, 255, 255) # (209, 200, 193)
min_scale, max_scale = 2 / 3, 2
imgW = 1024
imgH = 64
transform_train = transforms.Compose([
# RandomApply([cnx_aug_thin_characters()], p=0.2),
# RandomApply([cnx_aug_bold_characters()], p=0.4),
# cnd_aug_randomResizePadding(imgH, imgW, min_scale, max_scale, fill=fill_color),
cnd_aug_resizePadding(imgW, imgH, fill=fill_color),
RandomApply([cnd_aug_add_line()], p=0.5),
RandomApply([cnx_aug_blur()], p=0.3),
ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),
RandomApply(
[
RandomAffine(
shear=(-20, 20),
translate=(0.0, 0.1),
degrees=0,
# degrees=2,
# scale=(0.8, 1),
fillcolor=fill_color)
],
p=0.5)
# ,ToTensor()
# ,Normalize(mean, std)
])
path = "/data/train_data_29k_29Feb_update30Mar/cinnamon_data/cinamon_test_115/"
# path='/data/SDV/cropped_img'
list_img = list_files1(path, "jpg")
print(list_img)
for l in list_img:
path_img = os.path.join(path, l)
img = cv2.imread(path_img)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = Image.fromarray(img)
img_rs = transform_train(img)
numpy_image = np.array(img_rs)
# numpy_image = augment_random_rotate(img, -5, 5)
cv_img = cv2.cvtColor(numpy_image, cv2.COLOR_RGB2BGR)
cv2.imshow("result", cv_img)
cv2.waitKey(0)
def __init__(self):
self.random_resized_crop = RandomResizedCrop(range=(0.85,1.0))
self.random_horizontal_flip = RandomHorizontalFlip(p=1.0)
self.other_transform = Compose([
RandomApply([ColorJitter(0.8, 0.8, 0.2)], p=0.3),
RandomGrayscale(p=0.3),
RandomGaussianBlur(p=0.3),
ToTensor()
])
def __init__(self, hp: HParams):
s = hp.colour_distortion
self.simclr_augment = Compose([
RandomResizedCrop(hp.image_size),
RandomHorizontalFlip(),
RandomApply([ColorJitter(0.8 * s, 0.8 * s, 0.8 * s, 0.2 * s)],
p=0.8),
RandomGrayscale(p=0.2),
ToTensor(),
])
def prepare_datasets(meta_path, mels_dir, random_state=123):
df = pd.read_csv(meta_path)
group_ids = df['group_id'].unique()
group_ebird_codes = [
df[df['group_id'] == x].iloc[0]['ebird_code'] for x in group_ids
]
train_group_ids, test_group_ids = train_test_split(
group_ids,
test_size=0.2,
stratify=group_ebird_codes,
random_state=random_state)
train_group_ids = set(train_group_ids)
test_group_ids = set(test_group_ids)
train_df = df[df['group_id'].isin(train_group_ids)]
test_df = df[df['group_id'].isin(test_group_ids)]
train_mixup_dataset = BirdMelTrainDataset(
train_df, mels_dir, True,
Compose([
SpecTransform(RandomTimeShift(1.0)),
RandomApply(
[SpecTransform(RandomTimeResize(resize_mode='bilinear'))],
0.5),
]))
train_dataset = BirdMelTrainDataset(
train_df, mels_dir, True,
Compose([
RandomApply([SpecMixup(train_mixup_dataset, 0.5, True)], 0.5),
SpecTransform(RandomTimeShift(1.0)),
RandomApply(
[SpecTransform(RandomTimeResize(resize_mode='bilinear'))],
0.5),
SpecTransform(TimeMasking(10)),
SpecTransform(FrequencyMasking(8)),
]))
test_dataset = BirdMelTrainDataset(test_df, mels_dir, True)
return train_dataset, test_dataset
def __init__(self, opts):
self.dataroot = opts.dataroot
# A
images_A = os.listdir(os.path.join(self.dataroot, opts.phase + 'A'))
self.A = [
os.path.join(self.dataroot, opts.phase + 'A', x) for x in images_A
]
# B
images_B = os.listdir(os.path.join(self.dataroot, opts.phase + 'B'))
self.B = [
os.path.join(self.dataroot, opts.phase + 'B', x) for x in images_B
]
self.A_size = len(self.A)
self.B_size = len(self.B)
self.dataset_size = max(self.A_size, self.B_size)
self.input_dim_A = opts.input_dim_a
self.input_dim_B = opts.input_dim_b
# setup image transformation
transforms = [
Resize((opts.resize_size, opts.resize_size), Image.BICUBIC),
CenterCrop(opts.crop_size),
RandomApply([ColorJitter(0.4, 0.4, 0.4, 0.1)], p=0.8),
RandomGrayscale(p=0.2),
RandomApply([moco.loader.GaussianBlur([.1, 2.])], p=0.5),
RandomHorizontalFlip(),
]
if opts.phase == 'train':
transforms.append(RandomCrop(opts.crop_size))
else:
#ALERT: what to do!
transforms.append(CenterCrop(opts.crop_size))
if not opts.no_flip:
transforms.append(RandomHorizontalFlip())
transforms.append(ToTensor())
transforms.append(Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]))
self.transforms = Compose(transforms)
print('A: %d, B: %d images' % (self.A_size, self.B_size))
return
def ColorDistort(options):
s = options.jitter_strength
# No need to normalize after color jitter?
# Or normalize before color jitter?
return Compose([
RandomApply([
ColorJitter(0.8*s, 0.8*s, 0.8*s, 0.2*s)
], p=0.8),
RandomGrayscale(p=0.2),
ToTensor(),
Normalize(options.image_mean, options.image_std),
])
def _photometric_augmentations(self, x):
if self.photometric_augmentations["random_color_jitter"]:
color_jitter = ColorJitter(brightness=0.8, contrast=0.8, saturation=0.8, hue=0.2)
x = RandomApply([color_jitter], p=0.8)(x)
if self.photometric_augmentations["random_grayscale"]:
x = RandomGrayscale(0.2)(x)
if self.photometric_augmentations["random_gaussian_blur"]:
w, h = x.size
x = GaussianBlur(kernel_size=int((0.1 * min(w, h) // 2 * 2) + 1))(x)
return x
def __init__(self, opts, setname, input_dim):
self.dataroot = opts.dataroot
images = os.listdir(os.path.join(self.dataroot, opts.phase + setname))
self.img = [
os.path.join(self.dataroot, opts.phase + setname, x)
for x in images
]
self.size = len(self.img)
self.input_dim = input_dim
# setup image transformation
normalize = Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
if args.aug_plus:
# MoCo v2's aug: similar to SimCLR https://arxiv.org/abs/2002.05709
augmentation = [
#transforms.RandomResizedCrop(224, scale=(0.2, 1.)),
Resize((opts.resize_size, opts.resize_size), Image.BICUBIC),
CenterCrop(opts.crop_size),
RandomApply([ColorJitter(0.4, 0.4, 0.4, 0.1)], p=0.8),
RandomGrayscale(p=0.2),
RandomApply([moco.loader.GaussianBlur([.1, 2.])], p=0.5),
RandomHorizontalFlip(),
ToTensor(),
normalize
]
else:
# MoCo v1's aug: the same as InstDisc https://arxiv.org/abs/1805.01978
augmentation = [
#RandomResizedCrop(224, scale=(0.2, 1.)),
Resize((opts.resize_size, opts.resize_size), Image.BICUBIC),
CenterCrop(opts.crop_size),
RandomGrayscale(p=0.2),
ColorJitter(0.4, 0.4, 0.4, 0.4),
RandomHorizontalFlip(),
ToTensor(),
normalize
]
self.transforms = Compose(augmentation)
print('%s: %d images' % (setname, self.size))
return
def _transform(cls,
cache_image,
resize_dim=256,
mode='center',
augment=False):
norm = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
if augment:
rotate = RandomApply([RandomRotation(10.0, expand=True)], p=0.5)
color = RandomApply([ColorJitter(brightness=0.1, contrast=0.1)],
p=0.5)
augs = [rotate, color]
if mode == cls.TRANSFORM_CENTER:
augs += [Resize(resize_dim), CenterCrop(224)]
elif mode == cls.TRANSFORM_PAD:
augs += [PadSquare(), Resize(224)]
else:
raise ValueError('Unknown transform mode {0}'.format(mode))
else:
augs = []
if cache_image:
if mode == cls.TRANSFORM_CENTER:
return Compose([Resize(resize_dim),
CenterCrop(224)
]), Compose(augs + [ToTensor(), norm])
elif mode == cls.TRANSFORM_PAD:
return Compose([PadSquare(), Resize(224)
]), Compose(augs + [ToTensor(), norm])
else:
raise ValueError('Unknown transform mode {0}'.format(mode))
else:
if mode == cls.TRANSFORM_CENTER:
return None, Compose(
[Resize(resize_dim), CenterCrop(224)] + augs +
[ToTensor(), norm])
elif mode == cls.TRANSFORM_PAD:
return None, Compose([PadSquare(), Resize(224)] + augs +
[ToTensor(), norm])
else:
raise ValueError('Unknown transform mode {0}'.format(mode))
def train_hr_transform(crop_size):
return Compose([
RandomApply(torch.nn.ModuleList([
Resize(960),
Resize(720),
Resize(576),
Resize(480)
]), p=0.7),
RandomCrop(crop_size),
RandomHorizontalFlip(p=0.5),
#RandomVerticalFlip(p=0.5),
RandomApply(torch.nn.ModuleList([
ColorJitter(brightness=0.1,
contrast=0.1,
saturation=0.1,
hue=0.05),
]), p=0.3),
ToTensor(),
])
def get_transform(
target_size=(512, 512),
transform_list='horizontal_flip', # random_crop | keep_aspect
augment_ratio=0.5,
is_train=True,
):
transform = list()
transform_list = transform_list.split(', ')
augments = list()
for transform_name in transform_list:
if transform_name == 'random_crop':
scale = (0.6, 1.0) if is_train else (0.8, 1.0)
transform.append(RandomResizedCrop(target_size, scale=scale))
# elif transform_name == 'resize':
# transform.append(Resize(target_size))
elif transform_name == 'keep_aspect':
transform.append(KeepAsepctResize(target_size))
elif transform_name == 'Affine':
augments.append(
RandomAffine(degrees=(-180, 180),
scale=(0.8889, 1.0),
shear=(-36, 36)))
elif transform_name == 'centor_crop':
augments.append(CenterCrop(target_size))
elif transform_name == 'horizontal_flip':
augments.append(RandomHorizontalFlip())
elif transform_name == 'vertical_flip':
augments.append(RandomVerticalFlip())
elif transform == 'random_grayscale':
p = 0.5 if is_train else 0.25
transform.append(RandomGrayscale(p))
elif transform_name == 'random_rotate':
augments.append(RandomRotation(180))
elif transform_name == 'color_jitter':
brightness = 0.1 if is_train else 0.05
contrast = 0.1 if is_train else 0.05
augments.append(
ColorJitter(
brightness=brightness,
contrast=contrast,
saturation=0,
hue=0,
))
transform.append(RandomApply(augments, p=augment_ratio))
transform.append(ToTensor())
transform.append(
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]))
return Compose(transform)
def create_list_of_transforms(transforms_list, ovino_ir=False):
transforms = []
if transforms_list:
for transform in transforms_list:
transform_name = transform.pop('name')
transform_prob = transform.pop('prob', 1.0)
transforms.append(
RandomApply([TRANSFORMS[transform_name](**transform)],
p=transform_prob))
if ovino_ir:
transforms.append(TransformOvinoIR())
else:
transforms.append(TransformToTensor())
return Compose(transforms)
def get_transforms():
'''Get Transforms'''
transform1 = Compose([
Resize(IMAGE_SIZE),
CenterCrop(IMAGE_SIZE)
])
random_transforms = [RandomRotation(degrees = 5)]
transform2 = Compose([
RandomHorizontalFlip(p = 0.5),
RandomApply(random_transforms, p = 0.3),
ToTensor(),
Normalize(
(0.5, 0.5, 0.5),
(0.5, 0.5, 0.5)
)
])
return transform1, transform2
def GaussBlur(options):
kernel_size = math.floor(options.image_size / 10)
# ensure odd kernel size so that we can
# reflect pad with (kernel_size - 1)/2 to get same size output
if kernel_size % 2 == 0:
kernel_size += 1
return Compose([
ToTensor(),
Normalize(options.image_mean, options.image_std),
RandomApply([
GaussianSmoothing(
channels=options.image_channels,
kernel_size=kernel_size,
sigma=random.uniform(0.1, 2.0),
dim=2
)
], p=0.5),
])
def __init__(self,
size: tuple = (32, 32),
mode: str = 'pretrain',
**kwargs):
if mode == 'pretrain':
self.transforms = Compose([
RandomResizedCrop(size=size),
RandomHorizontalFlip(p=0.5),
RandomApply([ColorJitter(0.4, 0.4, 0.4, 0.1)], p=0.8),
RandomGrayscale(p=0.2),
ToTensor(),
Normalize(mean=self.mean, std=self.std)
])
elif mode == 'test':
self.transforms = Compose(
[ToTensor(),
Normalize(mean=self.mean, std=self.std)])
else:
raise NotImplementedError
self.size = size
self.mode = mode
def augmentations(args):
"""Applies random augmentations for the input images based on the transform probability.
Note:
Many methods are taken from https://www.kaggle.com/c/tgs-salt-identification-challenge/discussion/63974.
The user can specify between geometric, image or both types of transforms to the images since sometimes
some transformations work well for certain datasets.
:param args:
image_size (int) : size of the image to be resized.
transform_prob (float) : probability to apply transformations on the data.
:return:
a compose of transformations.
"""
augment_type = 'geometric'
transform_prob = args.transform_prob
if augment_type == 'geometric':
geometric_transforms = Compose([
RandomApply([HorizontalShear(max_scale=0.07)], p=transform_prob),
RandomApply([Rotation(max_angle=15)], p=transform_prob),
RandomApply([ElasticDeformation(max_distort=0.15)],
p=transform_prob),
ResizeToNxN(args.image_size),
ConvertToTensor()
])
return geometric_transforms
elif augment_type == 'image':
brightness_transform = Compose([
RandomApply([BrightnessShift(max_value=0.1)], p=transform_prob),
RandomApply([BrightnessScaling(max_value=0.08)], p=transform_prob),
RandomApply([GammaChange(max_value=0.08)], p=transform_prob),
ResizeToNxN(args.image_size),
ConvertToTensor()
])
return brightness_transform
elif augment_type == 'both':
both_transforms = Compose([
RandomApply([HorizontalShear(max_scale=0.07)], p=transform_prob),
RandomApply([Rotation(max_angle=15)], p=transform_prob),
RandomApply([ElasticDeformation(max_distort=0.15)],
p=transform_prob),
RandomApply([BrightnessShift(max_value=0.1)], p=transform_prob),
RandomApply([BrightnessScaling(max_value=0.08)], p=transform_prob),
RandomApply([GammaChange(max_value=0.08)], p=transform_prob),
ResizeToNxN(args.image_size),
ConvertToTensor()
])
return both_transforms
from torchvision.transforms import ColorJitter, ToTensor, Normalize
from common.dataset import FilesFromCsvDataset
from common.data_loaders import get_data_loader
from models.inceptionresnetv2_ssd_like import FurnitureInceptionResNetV4350SSDLike_v3
SEED = 17
DEBUG = True
DEVICE = 'cuda'
OUTPUT_PATH = "output"
size = 350
TRAIN_TRANSFORMS = Compose([
RandomApply([
RandomAffine(degrees=10, resample=3, fillcolor=(255, 255, 255)),
],
p=0.5),
RandomResizedCrop(size, scale=(0.7, 1.0), interpolation=3),
RandomHorizontalFlip(p=0.5),
ColorJitter(hue=0.12, brightness=0.12),
ToTensor(),
Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
])
VAL_TRANSFORMS = TRAIN_TRANSFORMS
BATCH_SIZE = 24
NUM_WORKERS = 15
dataset = FilesFromCsvDataset("output/unique_filtered_train_dataset.csv")
TRAIN_LOADER = get_data_loader(dataset,
parser.add_argument('--epochs', type=int, default=10, help='number of epochs')
parser.add_argument('--epochs2', type=int, default=-1, help='number of epochs')
parser.add_argument('--lr', type=float, default=1e-4, help='learning rate')
parser.add_argument('--lr2', type=float, default=-1, help='learning rate')
parser.add_argument('--tasks', default=10, type=int, help='no. of tasks')
parser.add_argument('--hidden_size', default=32, type=int, help='hidden neurons')
parser.add_argument('--im_size', default=28, type=int, help='image dimensions')
parser.add_argument('--save_path', default='./saved_models/default.pt', type=str, help='save path')
parser.add_argument('--load_path', default='', type=str, help='load path')
args = parser.parse_args()
train_data_aug = Compose([
Resize(size = args.im_size),
RandomApply(
[RandomAffine(degrees=(-10, 10), scale=(0.8, 1.2), translate=(0.05, 0.05))],
p=0.5
),
ToTensor(),
])
train_data_aug_simple = Compose([
Resize(size = args.im_size),
ToTensor()
])
test_data_aug = Compose([
Resize(size = args.im_size),
ToTensor()
])
opt.adadelta = True
opt.lr = 1
# opt.STN_type = 'Affine'
# opt.tps_inputsize = [32, 64]
opt.STN_type = 'TPS'
opt.tps_inputsize = [48, 128]
opt.tps_outputsize = [96, 256]
htr = BaseHTR(opt, dataset_name)
htr.nheads = 1
l1 = ['pn', 'bn', 'od', 'gu'] # indo-aryan languages smaller wdith
l2 = ['kn', 'ma', 'ta']
if lang in l1:
elastic_alpha = 0.3
else:
elastic_alpha = 0.2
htr.train_transforms = Compose([
GD(0.5),
IRescale(max_width=htr.opt.imgW, height=htr.opt.imgH),
ElasticTransformation(0.5, alpha=elastic_alpha),
AffineTransformation(0.5, rotate=5, shear=0.5),
RandomApply([ColorJitter(brightness=0.5, contrast=0.5)], p=0.5),
ToTensor()])
htr.test_transforms = Compose([IRescale(max_width=htr.opt.imgW, height=htr.opt.imgH),
ToTensor()])
htr.run()
def DataSets(dataset="MNIST",
data_path="../data",
tensor_size=None,
n_samples=64,
cpus=multiprocessing.cpu_count(),
augment=False):
r"""Train, validation and test dataset iterator for
MNIST/FashionMNIST/CIFAR10/CIFAR100
Args:
dataset (string): name of dataset, MNIST/FashionMNIST/CIFAR10/CIFAR100
data_path (string): path to dataset, default = "../data"
tensor_size (list/tuple, optional): BCHW of output, default = based on
dataset
n_samples (int): samples per batch
cpus (int, optional): numbers of cpus used by dataloader,
default = cpu_count
augment (bool, optional): when True, does color jitter, random crop
and random rotation
Return:
train data iterator, test data iterator and n_labels
"""
dataset = dataset.lower()
assert dataset in ["mnist", "fashionmnist", "cifar10", "cifar100"],\
"DataSets: available options MNIST/FashionMNIST/CIFAR10/CIFAR100"
# data path
folder = os.path.join(data_path, dataset)
if not os.path.isdir(folder):
os.mkdir(folder)
basics = [ToTensor()]
if dataset in ["mnist", "fashionmnist", "cifar10", "cifar100"]:
n_labels = 10
if dataset == "mnist":
loader = datasets.MNIST
if tensor_size is None:
tensor_size = (1, 1, 28, 28)
basics += [Normalize((0.1307,), (0.3081,))]
if dataset == "fashionmnist":
loader = datasets.FashionMNIST
if tensor_size is None:
tensor_size = (1, 1, 28, 28)
basics += [Normalize((0.5,), (0.5,))]
if dataset == "cifar10":
loader = datasets.CIFAR10
if tensor_size is None:
tensor_size = (1, 3, 32, 32)
basics += [Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010))]
if dataset == "cifar100":
n_labels = 100
loader = datasets.CIFAR100
if tensor_size is None:
tensor_size = (1, 3, 32, 32)
basics += [Normalize((0.5071, 0.4867, 0.4408),
(0.2675, 0.2565, 0.2761))]
elif dataset == "emnist":
# TODO
pass
# test data
teData = loader(root=folder, train=False, download=True,
transform=Compose(basics))
teData = DataLoader(teData, batch_size=n_samples,
shuffle=False, num_workers=cpus)
# validation data
vaData = None
# train data
if augment:
h, w = tensor_size[2:]
few_augs = [ColorJitter(.5, .5, .2),
RandomResizedCrop((w, h), scale=(0.7, 1.0),
ratio=(0.75, 1.33), interpolation=2),
RandomRotation(16, resample=False)]
if dataset not in ["mnist", "fashionmnist"]:
few_augs += [RandomHorizontalFlip(p=0.25)]
basics = [RandomApply(few_augs, p=0.8)] + basics
trData = loader(root=folder, train=True, download=False,
transform=Compose(basics))
#trData = DataLoader(trData, batch_size=n_samples,
# shuffle=True, num_workers=cpus)
return trData, vaData, teData, n_labels, tensor_size
from torchvision.transforms import Compose, Normalize, RandomApply, ColorJitter, RandomGrayscale, RandomResizedCrop, RandomHorizontalFlip, ToTensor
CIFAR10_MEAN = (0.4914, 0.4822, 0.4465)
CIFAR10_STD = (0.247, 0.243, 0.261)
transforms_train = Compose([
RandomResizedCrop((32, 32)),
RandomHorizontalFlip(p=0.5),
RandomApply([
ColorJitter(0.8, 0.8, 0.8, 0.2),
], p=0.8),
RandomGrayscale(p=0.2),
ToTensor(),
# Normalize(mean=CIFAR10_MEAN, std=CIFAR10_STD)
])
transforms_valid = transforms_train
transforms_test = Compose([
ToTensor(),
# Normalize(mean=CIFAR10_MEAN, std=CIFAR10_STD)
])
from common.dataset import get_data_loaders
from models.dpn import FurnitureDPN131_350
SEED = 12345
DEBUG = True
OUTPUT_PATH = "output"
DATASET_PATH = Path(
"/home/fast_storage/imaterialist-challenge-furniture-2018/")
size = 350
TRAIN_TRANSFORMS = [
RandomApply([
RandomAffine(
degrees=45, translate=(0.1, 0.1), scale=(0.7, 1.2), resample=2),
],
p=0.5),
RandomCrop(size=size),
RandomHorizontalFlip(p=0.5),
RandomVerticalFlip(p=0.5),
ColorJitter(hue=0.1, brightness=0.1),
ToTensor(),
Normalize(mean=[124 / 255, 117 / 255, 104 / 255],
std=[1 / (.0167 * 255)] * 3)
]
VAL_TRANSFORMS = [
CenterCrop(size=size),
RandomHorizontalFlip(p=0.5),
RandomVerticalFlip(p=0.5),
ToTensor(),
workers = config_crnn.workers_train
batch_size = config_crnn.batch_size
train_file = config_crnn.train_file
val_file = config_crnn.val_file
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
fill_color = (255, 255, 255) # (209, 200, 193)
min_scale, max_scale = 2 / 3, 2
transform_train = transforms.Compose([
# RandomApply([cnx_aug_thin_characters()], p=0.2),
# RandomApply([cnx_aug_bold_characters()], p=0.4),
# cnd_aug_randomResizePadding(imgH, imgW, min_scale, max_scale, fill=fill_color),
cnd_aug_resizePadding(imgW, imgH, fill=fill_color),
RandomApply([cnd_aug_add_line()], p=0.3),
RandomApply([cnx_aug_blur()], p=0.3),
ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),
RandomApply([RandomAffine(shear=(-20, 20),
translate=(0.0, 0.05),
degrees=0,
# degrees=2,
# scale=(0.8, 1),
fillcolor=fill_color)], p=0.3)
, ToTensor()
, Normalize(mean, std)
])
transform_test = transforms.Compose([
# cnd_aug_randomResizePadding(imgH, imgW, min_scale, max_scale, fill=fill_color, train=False),
cnd_aug_resizePadding(imgW, imgH, fill=fill_color, train=False),
