def generate_init_samples(self, im: torch.Tensor, target_pos, sample_scale) -> TensorList: # Compute augmentation size aug_expansion_factor = getattr(self.params, 'augmentation_expansion_factor', None) aug_expansion_sz = self.img_sample_sz.clone() aug_output_sz = None if aug_expansion_factor is not None and aug_expansion_factor != 1: aug_expansion_sz = (self.img_sample_sz * aug_expansion_factor).long() aug_expansion_sz += (aug_expansion_sz - self.img_sample_sz.long()) % 2 aug_expansion_sz = aug_expansion_sz.float() aug_output_sz = self.img_sample_sz.long().tolist() # Random shift operator get_rand_shift = lambda: None # Create transofmations self.transforms = [augmentation.Identity(aug_output_sz)] if 'shift' in self.params.augmentation_method: self.transforms.extend([augmentation.Translation(shift, aug_output_sz) for shift in self.params.augmentation_method['shift']]) if 'relativeshift' in self.params.augmentation_method: get_absolute = lambda shift: (torch.Tensor(shift) * self.img_sample_sz/2).long().tolist() self.transforms.extend([augmentation.Translation(get_absolute(shift), aug_output_sz) for shift in self.params.augmentation_method['relativeshift']]) if 'fliplr' in self.params.augmentation_method and self.params.augmentation_method['fliplr']: self.transforms.append(augmentation.FlipHorizontal(aug_output_sz, get_rand_shift())) if 'blur' in self.params.augmentation_method: self.transforms.extend([augmentation.Blur(sigma, aug_output_sz, get_rand_shift()) for sigma in self.params.augmentation_method['blur']]) if 'scale' in self.params.augmentation_method: self.transforms.extend([augmentation.Scale(scale_factor, aug_output_sz, get_rand_shift()) for scale_factor in self.params.augmentation_method['scale']]) if 'rotate' in self.params.augmentation_method: self.transforms.extend([augmentation.Rotate(angle, aug_output_sz, get_rand_shift()) for angle in self.params.augmentation_method['rotate']]) init_sample = sample_patch(im, target_pos, sample_scale*aug_expansion_sz, aug_expansion_sz) init_samples = torch.cat([T(init_sample) for T in self.transforms]) if not self.params.augmentation: init_samples = init_samples[0:1,...] return init_samples
def generate_init_samples(self, im: torch.Tensor) -> TensorList: # Do data augmentation transforms = [augmentation.Identity()] if 'shift' in self.params.augmentation: transforms.extend([augmentation.Translation(shift) for shift in self.params.augmentation['shift']]) if 'fliplr' in self.params.augmentation and self.params.augmentation['fliplr']: transforms.append(augmentation.FlipHorizontal()) if 'rotate' in self.params.augmentation: transforms.extend([augmentation.Rotate(angle) for angle in self.params.augmentation['rotate']]) if 'blur' in self.params.augmentation: transforms.extend([augmentation.Blur(sigma) for sigma in self.params.augmentation['blur']]) init_samples = self.params.features.extract_transformed(im, self.pos.round(), self.target_scale, self.img_sample_sz, transforms) # Remove augmented samples for those that shall not have for i, use_aug in enumerate(self.fparams.attribute('use_augmentation')): if not use_aug: init_samples[i] = init_samples[i][0:1, ...] if 'dropout' in self.params.augmentation: num, prob = self.params.augmentation['dropout'] for i, use_aug in enumerate(self.fparams.attribute('use_augmentation')): if use_aug: init_samples[i] = torch.cat([init_samples[i], F.dropout2d(init_samples[i][0:1,...].expand(num,-1,-1,-1), p=prob, training=True)]) return init_samples
def generate_init_samples(self, im: torch.Tensor) -> TensorList: """Generate augmented initial samples.""" # Compute augmentation size aug_expansion_factor = getattr(self.params, 'augmentation_expansion_factor', None) aug_expansion_sz = self.img_sample_sz.clone() aug_output_sz = None if aug_expansion_factor is not None and aug_expansion_factor != 1: aug_expansion_sz = (self.img_sample_sz * aug_expansion_factor).long() aug_expansion_sz += (aug_expansion_sz - self.img_sample_sz.long()) % 2 aug_expansion_sz = aug_expansion_sz.float() aug_output_sz = self.img_sample_sz.long().tolist() # Random shift operator get_rand_shift = lambda: None random_shift_factor = getattr(self.params, 'random_shift_factor', 0) if random_shift_factor > 0: get_rand_shift = lambda: ((torch.rand(2) - 0.5) * self.img_sample_sz * random_shift_factor).long().tolist() # Create transofmations self.transforms = [augmentation.Identity(aug_output_sz)] if 'shift' in self.params.augmentation: self.transforms.extend([augmentation.Translation(shift, aug_output_sz) for shift in self.params.augmentation['shift']]) if 'relativeshift' in self.params.augmentation: get_absolute = lambda shift: (torch.Tensor(shift) * self.img_sample_sz/2).long().tolist() self.transforms.extend([augmentation.Translation(get_absolute(shift), aug_output_sz) for shift in self.params.augmentation['relativeshift']]) if 'fliplr' in self.params.augmentation and self.params.augmentation['fliplr']: self.transforms.append(augmentation.FlipHorizontal(aug_output_sz, get_rand_shift())) if 'blur' in self.params.augmentation: self.transforms.extend([augmentation.Blur(sigma, aug_output_sz, get_rand_shift()) for sigma in self.params.augmentation['blur']]) if 'scale' in self.params.augmentation: self.transforms.extend([augmentation.Scale(scale_factor, aug_output_sz, get_rand_shift()) for scale_factor in self.params.augmentation['scale']]) if 'rotate' in self.params.augmentation: self.transforms.extend([augmentation.Rotate(angle, aug_output_sz, get_rand_shift()) for angle in self.params.augmentation['rotate']]) # Generate initial samples init_samples = self.params.features.extract_transformed(im, self.pos, self.target_scale, aug_expansion_sz, self.transforms) # Remove augmented samples for those that shall not have for i, use_aug in enumerate(self.fparams.attribute('use_augmentation')): if not use_aug: init_samples[i] = init_samples[i][0:1, ...] # Add dropout samples if 'dropout' in self.params.augmentation: num, prob = self.params.augmentation['dropout'] self.transforms.extend(self.transforms[:1]*num) for i, use_aug in enumerate(self.fparams.attribute('use_augmentation')): if use_aug: init_samples[i] = torch.cat([init_samples[i], F.dropout2d(init_samples[i][0:1,...].expand(num,-1,-1,-1), p=prob, training=True)]) return init_samples
def generate_init_samples(self, im: torch.Tensor) -> TensorList: # Do data augmentation transforms = [augmentation.Identity()] if 'shift' in self.params.augmentation: transforms.extend([ augmentation.Translation(shift) for shift in self.params.augmentation['shift'] ]) if 'fliplr' in self.params.augmentation and self.params.augmentation[ 'fliplr']: transforms.append(augmentation.FlipHorizontal()) if 'rotate' in self.params.augmentation: transforms.extend([ augmentation.Rotate(angle) for angle in self.params.augmentation['rotate'] ]) if 'blur' in self.params.augmentation: transforms.extend([ augmentation.Blur(sigma) for sigma in self.params.augmentation['blur'] ]) im_patch = F.interpolate(im, self.output_sz.long().tolist(), mode='bilinear') im_patches = torch.cat([T(im_patch) for T in transforms]) init_samples = TensorList([ f.get_feature(im_patches) for f in self.params.features.features ]).unroll() # Remove augmented samples for those that shall not have for i, use_aug in enumerate( self.fparams.attribute('use_augmentation')): if not use_aug: init_samples[i] = init_samples[i][0:1, ...] if 'dropout' in self.params.augmentation: num, prob = self.params.augmentation['dropout'] for i, use_aug in enumerate( self.fparams.attribute('use_augmentation')): if use_aug: init_samples[i] = torch.cat([ init_samples[i], F.dropout2d(init_samples[i][0:1, ...].expand( num, -1, -1, -1), p=prob, training=True) ]) return init_samples
def generate_init_samples(self, im: torch.Tensor) -> TensorList: """Perform data augmentation to generate initial training samples.""" if getattr(self.params, 'border_mode', 'replicate') == 'inside': # Get new sample size if forced inside the image im_sz = torch.Tensor([im.shape[2], im.shape[3]]) sample_sz = self.target_scale * self.img_sample_sz shrink_factor = (sample_sz.float() / im_sz).max().clamp(1) sample_sz = (sample_sz.float() / shrink_factor) self.init_sample_scale = (sample_sz / self.img_sample_sz).prod().sqrt() tl = self.pos - (sample_sz - 1) / 2 br = self.pos + sample_sz / 2 + 1 global_shift = - ((-tl).clamp(0) - (br - im_sz).clamp(0)) / self.init_sample_scale else: self.init_sample_scale = self.target_scale global_shift = torch.zeros(2) self.init_sample_pos = self.pos.round() # Compute augmentation size aug_expansion_factor = getattr(self.params, 'augmentation_expansion_factor', None) aug_expansion_sz = self.img_sample_sz.clone() aug_output_sz = None if aug_expansion_factor is not None and aug_expansion_factor != 1: aug_expansion_sz = (self.img_sample_sz * aug_expansion_factor).long() aug_expansion_sz += (aug_expansion_sz - self.img_sample_sz.long()) % 2 aug_expansion_sz = aug_expansion_sz.float() aug_output_sz = self.img_sample_sz.long().tolist() # Random shift for each sample get_rand_shift = lambda: None random_shift_factor = getattr(self.params, 'random_shift_factor', 0) if random_shift_factor > 0: get_rand_shift = lambda: ((torch.rand(2) - 0.5) * self.img_sample_sz * random_shift_factor + global_shift).long().tolist() # Always put identity transformation first, since it is the unaugmented sample that is always used self.transforms = [augmentation.Identity(aug_output_sz, global_shift.long().tolist())] augs = self.params.augmentation if getattr(self.params, 'use_augmentation', True) else {} # Add all augmentations if 'shift' in augs: self.transforms.extend([augmentation.Translation(shift, aug_output_sz, global_shift.long().tolist()) for shift in augs['shift']]) if 'relativeshift' in augs: get_absolute = lambda shift: (torch.Tensor(shift) * self.img_sample_sz/2).long().tolist() self.transforms.extend([augmentation.Translation(get_absolute(shift), aug_output_sz, global_shift.long().tolist()) for shift in augs['relativeshift']]) if 'fliplr' in augs and augs['fliplr']: self.transforms.append(augmentation.FlipHorizontal(aug_output_sz, get_rand_shift())) if 'blur' in augs: self.transforms.extend([augmentation.Blur(sigma, aug_output_sz, get_rand_shift()) for sigma in augs['blur']]) if 'scale' in augs: self.transforms.extend([augmentation.Scale(scale_factor, aug_output_sz, get_rand_shift()) for scale_factor in augs['scale']]) if 'rotate' in augs: self.transforms.extend([augmentation.Rotate(angle, aug_output_sz, get_rand_shift()) for angle in augs['rotate']]) # Extract augmented image patches im_patches = sample_patch_transformed(im, self.init_sample_pos, self.init_sample_scale, aug_expansion_sz, self.transforms) # Extract initial backbone features with torch.no_grad(): init_backbone_feat = self.net.extract_backbone(im_patches) return init_backbone_feat
def generate_init_samples(self, im: torch.Tensor, init_mask): """ Generate initial training sample.""" mode = self.params.get('border_mode', 'replicate') if 'inside' in mode: # Get new sample size if forced inside the image im_sz = torch.Tensor([im.shape[2], im.shape[3]]) sample_sz = self.target_scale * self.img_sample_sz shrink_factor = (sample_sz.float() / im_sz) if mode == 'inside': shrink_factor = shrink_factor.max() elif mode == 'inside_major': shrink_factor = shrink_factor.min() shrink_factor.clamp_(min=1, max=self.params.get('patch_max_scale_change', None)) sample_sz = (sample_sz.float() / shrink_factor) init_sample_scale = (sample_sz / self.img_sample_sz).prod().sqrt() tl = self.pos - (sample_sz - 1) / 2 br = self.pos + sample_sz / 2 + 1 global_shift = -((-tl).clamp(0) - (br - im_sz).clamp(0)) / init_sample_scale else: init_sample_scale = self.target_scale global_shift = torch.zeros(2) init_sample_pos = self.pos.round() # Compute augmentation size aug_expansion_factor = 2.0 aug_expansion_sz = self.img_sample_sz.clone() aug_output_sz = None if aug_expansion_factor is not None and aug_expansion_factor != 1: aug_expansion_sz = (self.img_sample_sz * aug_expansion_factor).long() aug_expansion_sz += (aug_expansion_sz - self.img_sample_sz.long()) % 2 aug_expansion_sz = aug_expansion_sz.float() aug_output_sz = self.img_sample_sz.long().tolist() # Can be extended to include data augmentation on the initial frame self.transforms = [ augmentation.Identity(aug_output_sz, global_shift.long().tolist()) ] # Extract image patches im_patches = sample_patch_transformed(im, init_sample_pos, init_sample_scale, aug_expansion_sz, self.transforms) init_masks = sample_patch_transformed(init_mask, init_sample_pos, init_sample_scale, aug_expansion_sz, self.transforms, is_mask=True) init_masks = init_masks.to(self.params.device) # Extract initial backbone features with torch.no_grad(): init_backbone_feat = self.net.extract_backbone(im_patches) return init_backbone_feat, init_masks
def generate_init_samples(self, im: np.ndarray) -> TensorList: """Generate augmented initial samples.""" # Compute augmentation size aug_expansion_factor = getattr(self.params, 'augmentation_expansion_factor', None) aug_expansion_sz = self.img_sample_sz.copy() aug_output_sz = None if aug_expansion_factor is not None and aug_expansion_factor != 1: aug_expansion_sz = (self.img_sample_sz * aug_expansion_factor).astype('long') aug_expansion_sz += (aug_expansion_sz - self.img_sample_sz.astype('long')) % 2 aug_expansion_sz = aug_expansion_sz.astype('float32') aug_output_sz = self.img_sample_sz.astype('long').tolist() # Random shift operator get_rand_shift = lambda: None random_shift_factor = getattr(self.params, 'random_shift_factor', 0) if random_shift_factor > 0: get_rand_shift = lambda: ( (np.random.uniform(size=[2]) - 0.5) * self.img_sample_sz * random_shift_factor).astype('long').tolist() # Create transofmations self.transforms = [augmentation.Identity(aug_output_sz)] if 'shift' in self.params.augmentation: self.transforms.extend([ augmentation.Translation(shift, aug_output_sz) for shift in self.params.augmentation['shift'] ]) if 'relativeshift' in self.params.augmentation: get_absolute = lambda shift: (np.array(shift, 'float32') * self. img_sample_sz / 2).astype('long' ).tolist() self.transforms.extend([ augmentation.Translation(get_absolute(shift), aug_output_sz) for shift in self.params.augmentation['relativeshift'] ]) if 'fliplr' in self.params.augmentation and self.params.augmentation[ 'fliplr']: self.transforms.append( augmentation.FlipHorizontal(aug_output_sz, get_rand_shift())) if 'blur' in self.params.augmentation: self.transforms.extend([ augmentation.Blur(sigma, aug_output_sz, get_rand_shift()) for sigma in self.params.augmentation['blur'] ]) if 'scale' in self.params.augmentation: self.transforms.extend([ augmentation.Scale(scale_factor, aug_output_sz, get_rand_shift()) for scale_factor in self.params.augmentation['scale'] ]) if 'rotate' in self.params.augmentation: self.transforms.extend([ augmentation.Rotate(angle, aug_output_sz, get_rand_shift()) for angle in self.params.augmentation['rotate'] ]) # Generate initial samples init_samples = self.params.features.extract_transformed( im, self.pos, self.target_scale, aug_expansion_sz, self.transforms) # Remove augmented samples for those that shall not have for i, use_aug in enumerate( self.fparams.attribute('use_augmentation')): if not use_aug: init_samples[i] = init_samples[i][0:1] # Add dropout samples if 'dropout' in self.params.augmentation: num, prob = self.params.augmentation['dropout'] self.transforms.extend(self.transforms[:1] * num) with fluid.dygraph.guard(): for i, use_aug in enumerate( self.fparams.attribute('use_augmentation')): if use_aug: init_samples[i] = np.concatenate([ init_samples[i], dropout2d(layers.expand(n2p(init_samples[i][0:1]), (num, 1, 1, 1)), prob, is_train=True).numpy() ]) return init_samples