def init_model(self):
"""Initialize model and other data for procedure"""
if self.optical_flow is True:
self.in_planes = 2
elif self.data_type in ["depth", "ir"]:
self.in_planes = 1
else:
self.in_planes = 3
# Selecting correct model and normalization variable based on type variable
self.net = GestureTransoformer(
self.backbone,
self.in_planes,
self.n_classes,
pretrained=self.configer.get("network", "pretrained"),
n_head=self.configer.get("network", "n_head"),
dropout_backbone=self.configer.get("network", "dropout2d"),
dropout_transformer=self.configer.get("network", "dropout1d"),
dff=self.configer.get("network", "ff_size"),
n_module=self.configer.get("network", "n_module"))
self.net, _, _, _ = self.model_utility.load_net(self.net)
# Selecting Dataset and DataLoader
if self.dataset == "briareo":
Dataset = Briareo
self.transforms = iaa.CenterCropToFixedSize(200, 200)
elif self.dataset == "nvgestures":
Dataset = NVGesture
self.transforms = iaa.CenterCropToFixedSize(256, 192)
else:
raise NotImplementedError(
f"Dataset not supported: {self.configer.get('dataset')}")
# Setting Dataloaders
self.data_loader = DataLoader(Dataset(self.configer,
self.data_path,
split="test",
data_type=self.data_type,
transforms=self.transforms,
n_frames=self.clip_length,
optical_flow=self.optical_flow),
batch_size=1,
shuffle=False,
drop_last=True,
num_workers=self.configer.get(
'solver', 'workers'),
pin_memory=True,
worker_init_fn=worker_init_fn)
def get_valid_augmentation(**kwargs):
seq = iaa.Sequential([
iaa.Resize({
'height': kwargs['crop_sz'],
'width': kwargs['crop_sz']
}),
iaa.CenterCropToFixedSize(height=kwargs['inp_sz'],
width=kwargs['inp_sz'])
])
return seq
def __init__(self, testing=False):
if not testing:
self.aug = iaa.Sequential([
iaa.Fliplr(0.5),
iaa.Affine(rotate=(-180, 180),
order=[0, 1, 3],
mode="symmetric"),
iaa.Sometimes(0.5, iaa.GaussianBlur(sigma=(0, 2.0))),
iaa.CenterCropToFixedSize(256, 256),
])
else:
self.aug = iaa.Sequential([
iaa.CropToFixedSize(256, 256),
])
def get_moments(data_root=os.path.join('/media','user','disk0'),
clip_length=8,
clip_size=256,
val_clip_length=None,
val_clip_size=None,
train_interval=2,
val_interval=2,
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
seed=torch.distributed.get_rank() if torch.distributed.is_initialized() else 0,
**kwargs):
""" data iter for Moments in Time
"""
logging.debug("VideoIter:: clip_length = {}, interval = [train: {}, val: {}], seed = {}".format( \
clip_length, train_interval, val_interval, seed))
sometimes_aug = lambda aug: iaa.Sometimes(0.4, aug)
sometimes_seq = lambda aug: iaa.Sometimes(0.9, aug)
train_sampler = sampler.RandomSampling(num=clip_length,
interval=train_interval,
speed=[1.0, 1.0],
seed=(seed+0))
train = VideoIter(dataset_location=os.path.join(data_root, 'data' ,
'Moments_in_Time_videos','jpg'),
csv_filepath=os.path.join(data_root, 'labels', 'Moments_in_Time_train_1.csv'),
include_timeslices = False,
sampler=train_sampler,
video_size=(clip_length,clip_size,clip_size),
video_transform = transforms.Compose(
transforms=iaa.Sequential([
iaa.Resize({"shorter-side": 294, "longer-side": "keep-aspect-ratio"}),
iaa.CropToFixedSize(width=224, height=224, position='uniform'),
sometimes_seq(iaa.Sequential([
sometimes_aug(iaa.GaussianBlur(sigma=[0.1,0.2,0.3])),
sometimes_aug(iaa.Add((-5, 15), per_channel=True)),
#sometimes_aug(iaa.AdditiveGaussianNoise(scale=0.03*255, per_channel=True)),
#sometimes_aug(iaa.pillike.EnhanceColor(factor=(1.2, 1.6))),
#sometimes_aug(iaa.MotionBlur([3,5,7])),
sometimes_aug(iaa.AddToHueAndSaturation((-16, 16), per_channel=True)),
sometimes_aug(iaa.LinearContrast((0.85, 1.115))),
sometimes_aug(
iaa.OneOf([
iaa.PerspectiveTransform(scale=(0.02, 0.05), keep_size=True),
iaa.Rotate(rotate=(-10,10)),
])
)
])),
iaa.Fliplr(0.5)
]),
normalise=[mean,std]
),
name='train',
shuffle_list_seed=(seed+2))
# Only return train iterator
if (val_clip_length is None and val_clip_size is None):
return train
val_sampler = sampler.SequentialSampling(num=clip_length,
interval=val_interval,
fix_cursor=True,
shuffle=True)
val = VideoIter(dataset_location=os.path.join(data_root, 'data',
'Moments_in_Time_videos','jpg'),
csv_filepath=os.path.join(data_root, 'labels', 'Moments_in_Time_val.csv'),
include_timeslices = False,
sampler=val_sampler,
video_size=(val_clip_length,val_clip_size,val_clip_size),
video_transform=transforms.Compose(
transforms=iaa.Sequential([
iaa.Resize({"shorter-side": 294, "longer-side": "keep-aspect-ratio"}),
iaa.CenterCropToFixedSize(width=val_clip_size, height=val_clip_size)
]),
normalise=[mean,std]),
name='val')
return (train, val)
def augment_raw_frames(raw_frames, method='val', img_size=224):
if method == 'ten': # 4 corners + center crop. 50% flipped
aug = iaa.Sequential([
iaa.Fliplr(0.5),
iaa.OneOf([
iaa.CenterCropToFixedSize(img_size, img_size),
iaa.CropToFixedSize(img_size, img_size, position='left-top'),
iaa.CropToFixedSize(img_size, img_size,
position='left-bottom'),
iaa.CropToFixedSize(img_size, img_size, position='right-top'),
iaa.CropToFixedSize(img_size,
img_size,
position='right-bottom')
])
])
elif method == 'center-crop':
aug = iaa.Sequential([
iaa.CenterCropToFixedSize(img_size, img_size),
])
elif method == '04-20':
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
aug = iaa.Sequential([
iaa.Fliplr(0.5),
iaa.CropToFixedSize(img_size, img_size),
iaa.LinearContrast(),
iaa.Flipud(0.2),
sometimes(iaa.Rotate())
])
elif method == 'kitchen-sink':
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
aug = iaa.Sequential([
iaa.CropToFixedSize(img_size, img_size),
# apply the following augmenters to most images
iaa.Fliplr(0.5), # horizontally flip 50% of all images
iaa.Flipud(0.2), # vertically flip 20% of all images
# crop images by -5% to 10% of their height/width
sometimes(
iaa.CropAndPad(percent=(-0.05, 0.1),
pad_mode=ia.ALL,
pad_cval=(0, 255))),
sometimes(
iaa.Affine(
scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
},
# scale images to 80-120% of their size, individually per axis
translate_percent={
"x": (-0.2, 0.2),
"y": (-0.2, 0.2)
},
# translate by -20 to +20 percent (per axis)
rotate=(-45, 45), # rotate by -45 to +45 degrees
shear=(-16, 16), # shear by -16 to +16 degrees
order=[
0,
1
], # use nearest neighbour or bilinear interpolation (fast)
cval=(
0, 255
), # if mode is constant, use a cval between 0 and 255
mode=ia.
ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
)),
# execute 0 to 5 of the following (less important) augmenters per image
# don't execute all of them, as that would often be way too strong
iaa.SomeOf(
(0, 5),
[
sometimes(
iaa.Superpixels(p_replace=(0, 1.0),
n_segments=(20, 200))),
# convert images into their superpixel representation
iaa.OneOf([
iaa.GaussianBlur(
(0, 3.0
)), # blur images with a sigma between 0 and 3.0
iaa.AverageBlur(k=(2, 7)),
# blur image using local means with kernel sizes between 2 and 7
iaa.MedianBlur(k=(3, 11)),
# blur image using local medians with kernel sizes between 2 and 7
]),
iaa.Sharpen(alpha=(0, 1.0),
lightness=(0.75, 1.5)), # sharpen images
iaa.Emboss(alpha=(0, 1.0),
strength=(0, 2.0)), # emboss images
# search either for all edges or for directed edges,
# blend the result with the original image using a blobby mask
iaa.SimplexNoiseAlpha(
iaa.OneOf([
iaa.EdgeDetect(alpha=(0.5, 1.0)),
iaa.DirectedEdgeDetect(alpha=(0.5, 1.0),
direction=(0.0, 1.0)),
])),
iaa.AdditiveGaussianNoise(
loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5),
# add gaussian noise to images
iaa.OneOf([
iaa.Dropout(
(0.01, 0.1), per_channel=0.5
), # randomly remove up to 10% of the pixels
iaa.CoarseDropout((0.03, 0.15),
size_percent=(0.02, 0.05),
per_channel=0.2),
]),
iaa.Invert(0.05,
per_channel=True), # invert color channels
iaa.Add((-10, 10), per_channel=0.5),
# change brightness of images (by -10 to 10 of original value)
iaa.AddToHueAndSaturation(
(-20, 20)), # change hue and saturation
# either change the brightness of the whole image (sometimes
# per channel) or change the brightness of subareas
iaa.OneOf([
iaa.Multiply((0.5, 1.5), per_channel=0.5),
iaa.FrequencyNoiseAlpha(
exponent=(-4, 0),
first=iaa.Multiply((0.5, 1.5), per_channel=True),
second=iaa.LinearContrast((0.5, 2.0)))
]),
iaa.LinearContrast(
(0.5, 2.0),
per_channel=0.5), # improve or worsen the contrast
iaa.Grayscale(alpha=(0.0, 1.0)),
sometimes(
iaa.ElasticTransformation(alpha=(0.5, 3.5),
sigma=0.25)),
# move pixels locally around (with random strengths)
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))),
# sometimes move parts of the image around
sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
],
random_order=True)
])
elif method == 'val':
aug = iaa.Sequential([
iaa.Resize({
"shorter-side": img_size,
"longer-side": "keep-aspect-ratio"
}),
iaa.CenterCropToFixedSize(img_size, img_size)
])
else:
aug = augmentations_for_method(method)
augDet = aug.to_deterministic()
num_frames = len(raw_frames)
h = img_size
w = img_size
c = raw_frames[0].shape[2]
results = np.zeros((num_frames, h, w, c), raw_frames[0].dtype)
for i in range(num_frames):
results[i] = augDet.augment_image(raw_frames[i])
return results
seq = iaa.Sequential([
iaa.Fliplr(p=0),# basically this is original one
iaa.Sometimes(0.05,(iaa.Crop(px=(22, 45),keep_size=True))), # crop images from each side by 0 to 16px (randomly chosen)
iaa.Sometimes(0.5,(iaa.Fliplr(1))), # horizontally flip 50% of the images
iaa.Sometimes(0.02,iaa.GaussianBlur(sigma=(5, 7.0))), # blur images with a sigma of 0 to 3.0
iaa.Sometimes(0.02 ,iaa.ImpulseNoise(p=(0.6,1))),
iaa.Sometimes(0.02 ,iaa.EdgeDetect(alpha=(0.09,1))),
#iaa.AddToBrightness(add=(100,124)),
iaa.Sometimes(0.02 ,iaa.Canny(alpha=(0.8,0.9))),
iaa.Sometimes(0.5 ,iaa.Grayscale(alpha=1.00)),
iaa.Sometimes(0.5 ,iaa.ChannelShuffle(p=1)),
#iaa.Sometimes(0.02 ,(iaa.geometric.Affine( scale=2,rotate=22,order=1))),
iaa.Sometimes(0.5 ,iaa.Cartoon(blur_ksize=(11,13))),
iaa.Sometimes(0.02 ,iaa.CenterCropToAspectRatio(1)),
iaa.Sometimes(0.02 ,iaa.CenterCropToFixedSize(100,100)),
iaa.Sometimes(0.12 ,iaa.ChangeColorTemperature(kelvin=(2222,3333))),
#iaa.segmentation(),
iaa.Sometimes(0.12 ,iaa.CLAHE(clip_limit=(4,8))),
iaa.Sometimes(0.8 ,iaa.Rotate(rotate=(-90,90),order=1))
])
plt.figure(figsize=(12,12))
ls=glob.glob(path)
res=[]
for idx,l in enumerate(ls):
res.append(Parser.myType(l,idx,classes=['bird','zebra']))
def init_model(self):
"""Initialize model and other data for procedure"""
if self.optical_flow is True:
self.in_planes = 2
elif self.data_type in ["depth", "ir"]:
self.in_planes = 1
else:
self.in_planes = 3
self.loss = nn.CrossEntropyLoss().to(self.device)
# Selecting correct model and normalization variable based on type variable
self.net = GestureTransoformer(
self.backbone,
self.in_planes,
self.n_classes,
pretrained=self.configer.get("network", "pretrained"),
n_head=self.configer.get("network", "n_head"),
dropout_backbone=self.configer.get("network", "dropout2d"),
dropout_transformer=self.configer.get("network", "dropout1d"),
dff=self.configer.get("network", "ff_size"),
n_module=self.configer.get("network", "n_module"))
# Initializing training
self.iters = 0
self.epoch = None
phase = self.configer.get('phase')
# Starting or resuming procedure
if phase == 'train':
self.net, self.iters, self.epoch, optim_dict = self.model_utility.load_net(
self.net)
else:
raise ValueError('Phase: {} is not valid.'.format(phase))
if self.epoch is None:
self.epoch = 0
# ToDo Restore optimizer and scheduler from checkpoint
self.optimizer, self.lr = self.model_utility.update_optimizer(
self.net, self.iters)
self.scheduler = MultiStepLR(self.optimizer,
self.configer["solver", "decay_steps"],
gamma=0.1)
# Resuming training, restoring optimizer value
if optim_dict is not None:
print("Resuming training from epoch {}.".format(self.epoch))
self.optimizer.load_state_dict(optim_dict)
# Selecting Dataset and DataLoader
if self.dataset == "briareo":
Dataset = Briareo
self.train_transforms = iaa.Sequential([
iaa.Resize((0.85, 1.15)),
iaa.CropToFixedSize(width=190, height=190),
iaa.Rotate((-15, 15))
])
self.val_transforms = iaa.CenterCropToFixedSize(200, 200)
elif self.dataset == "nvgestures":
Dataset = NVGesture
self.train_transforms = iaa.Sequential([
iaa.Resize((0.8, 1.2)),
iaa.CropToFixedSize(width=256, height=192),
iaa.Rotate((-15, 15))
])
self.val_transforms = iaa.CenterCropToFixedSize(256, 192)
else:
raise NotImplementedError(
f"Dataset not supported: {self.configer.get('dataset')}")
# Setting Dataloaders
self.train_loader = DataLoader(
Dataset(self.configer,
self.data_path,
split="train",
data_type=self.data_type,
transforms=self.train_transforms,
n_frames=self.clip_length,
optical_flow=self.optical_flow),
batch_size=self.configer.get('data', 'batch_size'),
shuffle=True,
drop_last=True,
num_workers=self.configer.get('solver', 'workers'),
pin_memory=True,
worker_init_fn=worker_init_fn)
self.val_loader = DataLoader(
Dataset(self.configer,
self.data_path,
split="val",
data_type=self.data_type,
transforms=self.val_transforms,
n_frames=self.clip_length,
optical_flow=self.optical_flow),
batch_size=self.configer.get('data', 'batch_size'),
shuffle=False,
drop_last=True,
num_workers=self.configer.get('solver', 'workers'),
pin_memory=True,
worker_init_fn=worker_init_fn)
if self.dataset == "nvgestures":
self.test_loader = None
else:
self.test_loader = DataLoader(
Dataset(self.configer,
self.data_path,
split="test",
data_type=self.data_type,
transforms=self.val_transforms,
n_frames=self.clip_length,
optical_flow=self.optical_flow),
batch_size=1,
shuffle=False,
drop_last=True,
num_workers=self.configer.get('solver', 'workers'),
pin_memory=True,
worker_init_fn=worker_init_fn)
seq = iaa.Sequential([
iaa.Fliplr(p=0),# basically this is original one
iaa.Crop(px=(22, 45),keep_size=False), # crop images from each side by 0 to 16px (randomly chosen)
iaa.Fliplr(1), # horizontally flip 50% of the images
iaa.GaussianBlur(sigma=(5, 7.0)), # blur images with a sigma of 0 to 3.0
iaa.ImpulseNoise(p=(0.6,1)),
iaa.EdgeDetect(alpha=(0.9,1)),
#iaa.AddToBrightness(add=(100,124)),
iaa.Canny(alpha=(0.8,0.9)),
iaa.Grayscale(alpha=1.00),
iaa.ChannelShuffle(p=1),
iaa.geometric.Affine( scale=2,rotate=22, backend='cv2'),
iaa.Cartoon(blur_ksize=(11,13)),
iaa.CenterCropToAspectRatio(1),
iaa.CenterCropToFixedSize(100,100),
iaa.ChangeColorTemperature(kelvin=(2222,3333)),
#iaa.segmentation(),
iaa.CLAHE(clip_limit=(4,8)),
iaa.Rotate(rotate=(-30,90))
])
plt.figure(figsize=(12,12))
for idx,Augmentor in enumerate(seq):
# print(1)
ax=plt.subplot(4,4,idx+1)
ax.axis('off')
plt.tight_layout()
title=str(Augmentor).split('(')[0]
#plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0,
transformed_image = transform(image=image)
elif augmentation == 'crop_to_aspect_ratio':
transform = iaa.CropToAspectRatio(2.0)
transformed_image = transform(image=image)
elif augmentation == 'crop_to_square':
transform = iaa.CropToSquare()
transformed_image = transform(image=image)
elif augmentation == 'center_crop':
transform = CenterCrop(always_apply=True, width=400, height=400)
transformed_image = transform(image=image)['image']
elif augmentation == 'center_crop_to_fixed_size':
transform = iaa.CenterCropToFixedSize(width=300, height=300)
transformed_image = transform(image=image)
elif augmentation == 'center_crop_to_multiples_of':
transform = iaa.CenterCropToPowersOf(height_base=3, width_base=2)
transformed_image = transform(image=image)
elif augmentation == 'center_crop_to_powers_of':
transform = iaa.CenterCropToMultiplesOf(height_multiple=32, width_multiple=32)
transformed_image = transform(image=image)
elif augmentation == 'center_crop_to_aspect_ratio':
transform = iaa.CenterCropToAspectRatio(2.0)
transformed_image = transform(image=image)
elif augmentation == 'center_crop_to_square':
depth_proj = depth.astype(np.float32) / 1000 * 256
depth_proj = depth_proj.astype(np.uint16)
# OB
# depth[depth > 5000] = 5000
# depth_proj = depth.astype(np.float32) / 100 * 256
# depth_proj = depth_proj.astype(np.uint16)
# KITTI
# depth_proj = depth
crop_h = 352
crop_w = 1216
seq1 = iaa.Sequential([
iaa.CenterPadToFixedSize(height=crop_h, width=crop_w), # 保证可crop
iaa.CenterCropToFixedSize(height=crop_h, width=crop_w),
], random_order=True)
seq2 = iaa.Sequential([
# iaa.CoarseDropout(0.19, size_px=200),
iaa.Dropout(1-0.05),
], random_order=True)
img_aug = seq1(image=img)
# depth_aug = seq1(image=depth_proj)
depth_aug = seq1(image=seq2(image=depth_proj))
show(img_aug)
show(depth)
show(depth_aug)
name = '\\nyu-1-kitti.png'
rotate = iaa.Affine(rotate=(-50, 30))
#flipping image verically
flip_vr = iaa.Flipud(p=1.0)
#flipping image horizontally
flip_hr = iaa.Fliplr(p=1.0)
#Affine
move = iaa.Affine(translate_percent={"x": 0.1, "y": 0.1}, scale=0.8)
#contrast
contrast = iaa.GammaContrast(gamma=2.0)
#resize
resize = iaa.Resize({
"height": 480,
"width": 640
}) #"height": 480, "width": 640
#crop
crop = iaa.CenterCropToFixedSize(height=300, width=650)
#brightness
brightness = iaa.WithBrightnessChannels(iaa.Add((-50, 50)))
augmentations = {
'rotate': rotate,
'flip_hr': flip_hr,
'flip_vr': flip_vr,
'affine': move,
'contrast': contrast,
'resize': resize,
'crop': crop,
'brightness': brightness
}
