def soft(image_iteration): iteration = image_iteration / (120 * 1.5) frequency_factor = 0.05 + float(iteration) / 1200000.0 color_factor = float(iteration) / 1200000.0 dropout_factor = 0.198667 + (0.03856658 - 0.198667) / (1 + (iteration / 196416.6)**1.863486) blur_factor = 0.5 + (0.5 * iteration / 120000.0) add_factor = 10 + 10 * iteration / 170000.0 multiply_factor_pos = 1 + (2.5 * iteration / 800000.0) multiply_factor_neg = 1 - (0.91 * iteration / 800000.0) contrast_factor_pos = 1 + (0.5 * iteration / 800000.0) contrast_factor_neg = 1 - (0.5 * iteration / 800000.0) #print ('iteration',iteration,'Augment Status ',frequency_factor,color_factor,dropout_factor,blur_factor,add_factor, # multiply_factor_pos,multiply_factor_neg,contrast_factor_pos,contrast_factor_neg) augmenter = iaa.Sequential( [ iaa.Sometimes(frequency_factor, iaa.GaussianBlur( (0, blur_factor))), # blur images with a sigma between 0 and 1.5 iaa.Sometimes( frequency_factor, iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, dropout_factor), per_channel=color_factor)), # add gaussian noise to images iaa.Sometimes( frequency_factor, iaa.CoarseDropout((0.0, dropout_factor), size_percent=(0.08, 0.2), per_channel=color_factor)), # randomly remove up to X% of the pixels iaa.Sometimes( frequency_factor, iaa.Dropout((0.0, dropout_factor), per_channel=color_factor)), # randomly remove up to X% of the pixels iaa.Sometimes( frequency_factor, iaa.Add((-add_factor, add_factor), per_channel=color_factor)), # change brightness of images (by -X to Y of original value) iaa.Sometimes( frequency_factor, iaa.Multiply((multiply_factor_neg, multiply_factor_pos), per_channel=color_factor)), # change brightness of images (X-Y% of original value) iaa.Sometimes( frequency_factor, iaa.ContrastNormalization( (contrast_factor_neg, contrast_factor_pos), per_channel=color_factor)), # improve or worsen the contrast iaa.Sometimes(frequency_factor, iaa.Grayscale( (0.0, 1))), # put grayscale ], random_order=True # do all of the above in random order ) return augmenter
def getSeq(self): self.sometimes = lambda aug: iaa.Sometimes(0.5, aug) self.seq = iaa.Sequential( [ # apply the following augmenters to most images # iaa.Fliplr(0.5), # horizontally flip 50% of all images # iaa.Flipud(0.5), # vertically flip 20% of all images iaa.SomeOf( (0, 5), [ #sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation iaa.OneOf([ iaa.GaussianBlur( (0, 3.0) ), # blur images with a sigma between 0 and 3.0 iaa.AverageBlur( k=(2, 7) ), # blur image using local means with kernel sizes between 2 and 7 iaa.MedianBlur( k=(3, 11) ), # blur image using local medians with kernel sizes between 2 and 7 ]), #iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images # search either for all edges or for directed edges, # blend the result with the original image using a blobby mask iaa.SimplexNoiseAlpha( iaa.OneOf([ iaa.EdgeDetect(alpha=(0.5, 1.0)), iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)), ])), iaa.AdditiveGaussianNoise( loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5), # add gaussian noise to images iaa.OneOf([ iaa.Dropout( (0.01, 0.1), per_channel=0.5 ), # randomly remove up to 10% of the pixels iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2), ]), iaa.Invert(0.05, per_channel=True), # invert color channels iaa.Add( (-10, 10), per_channel=0.5 ), # change brightness of images (by -10 to 10 of original value) iaa.AddToHueAndSaturation( (-20, 20)), # change hue and saturation # either change the brightness of the whole image (sometimes # per channel) or change the brightness of subareas iaa.OneOf([ iaa.Multiply((0.5, 1.5), per_channel=0.5), iaa.FrequencyNoiseAlpha( exponent=(-4, 0), first=iaa.Multiply( (0.5, 1.5), per_channel=True), second=iaa.ContrastNormalization((0.5, 2.0))) ]), iaa.ContrastNormalization( (0.5, 2.0), per_channel=0.5), # improve or worsen the contrast iaa.Grayscale(alpha=(0.0, 1.0)), ], random_order=True) ], random_order=True)
def data_aug(img, bboxs=None, keypoints=None, joint_nums=17): ''' :param img: 需要进行数据增强的图像 :param bboxs: list, [ [x1, y1, x2, y2], ..., [xn1, yn1, xn2, yn2] ] :param keypoints: 关键点, COCO format or Ai-challenger format, list of list, [ [num_joints x 3], [num_joints x 3], ..., ] :return: ''' is_flip = [random.randint(0, 1), random.randint(0, 1)] seq = iaa.Sequential([ iaa.Affine(rotate=(-15, 15), scale=(0.8, 1.2), mode='constant'), iaa.Multiply((0.7, 1.5)), iaa.Grayscale(iap.Choice(a=[0, 1], p=[0.8, 0.2]), from_colorspace='BGR'), iaa.Fliplr(is_flip[0]), iaa.Flipud(is_flip[1]), ]) seq_det = seq.to_deterministic() bbs = None kps = None bbs_aug = None kps_aug = None # joint_nums = 14 new_bboxs = [] new_keypoints = [] kps_ori = copy.copy(keypoints) kps_ori = np.reshape(np.asarray(kps_ori), newshape=(-1, joint_nums, 3)) if kps_ori is not None else None if bboxs is not None: assert type(bboxs) == type([]) bbs = ia.BoundingBoxesOnImage([], shape=img.shape) for box in bboxs: bbs.bounding_boxes.append( ia.BoundingBox(x1=box[0], y1=box[1], x2=box[2], y2=box[3])) if keypoints is not None: kps = ia.KeypointsOnImage([], shape=img.shape) assert type(keypoints) == type([]) for single_person_keypoints in keypoints: for i in range(joint_nums): joint = single_person_keypoints[i * 3:i * 3 + 3] kps.keypoints.append(ia.Keypoint(x=joint[0], y=joint[1])) img_aug = seq_det.augment_image(img) if bbs is not None: # print(bbs) bbs_aug = seq_det.augment_bounding_boxes([bbs]) # print(bbs_aug) for i in range(len(bbs_aug[0].bounding_boxes)): box_aug = bbs_aug[0].bounding_boxes[i] box = [box_aug.x1, box_aug.y1, box_aug.x2, box_aug.y2] new_bboxs.append(box) if kps is not None: # print(kps) kps_aug = seq_det.augment_keypoints([kps]) for i in range(len(kps_aug[0].keypoints)): point = kps_aug[0].keypoints[i] new_keypoints.append([point.x, point.y, 1]) new_keypoints = np.reshape(np.asarray(new_keypoints), newshape=(-1, joint_nums, 3)) # keep ori keypoint visiable attribute for i in range(kps_ori.shape[0]): for joint in range(kps_ori.shape[1]): new_keypoints[i][joint][2] = kps_ori[i][joint][2] if kps_ori[i][joint][0] == 0 or kps_ori[i][joint][1] == 0: new_keypoints[i][joint] = np.asarray([0, 0, 0]) # if flip, change keypoint order (left <-> right) # ai-format: [ 0-right_shoulder, 1-right_elbow, 2-right_wrist, # 3-left_shoulder, 4-left_elbow, 5-left_wrist, # 6-right_hip, 7-right_knee, 8-right_ankle, # 9-left_hip, 10-left_knee, 11-left_ankle, # 12-head, 13-neck ] # coco-format: TODO add coco-foramt change index # change_index = [[0, 3], [1, 4], [2, 5], [6, 9], [7, 10], [8, 11]] change_index = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12], [13, 14], [15, 16]] for flip in is_flip: if flip: for i in range(kps_ori.shape[0]): for index in change_index: right_point = copy.copy(new_keypoints[i][index[0]]) new_keypoints[i][index[0]] = new_keypoints[i][index[1]] new_keypoints[i][index[1]] = right_point new_keypoints = [ list(np.reshape(single_person_keypoints, (-1, ))) for single_person_keypoints in new_keypoints ] # test # if bbs is not None: # img_before = bbs.draw_on_image(img, color=(0, 255, 0), thickness=2) # img_after = bbs_aug.draw_on_image(img_aug, color=(0,0,255), thickness=2) # cv2.imshow('box ori', img_before) # cv2.imshow('box after', img_after) # cv2.waitKey(0) # if kps is not None: # img_before = kps.draw_on_image(img, color=(0, 255, 0), size=5) # img_after = kps_aug.draw_on_image(img_aug, color=(0, 0, 255), size=5) # for i in range(kps_ori.shape[0]): # for joint in range(kps_ori.shape[1]): # point = kps_ori[i][joint] # cv2.putText(img_before, str(joint), (int(point[0]), int(point[1])), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 250), 1) # point = new_keypoints[i][3*joint:3*joint+3] # # cv2.putText(img_after, str(point[2]), (int(point[0]), int(point[1])), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 250), 1) # cv2.putText(img_after, str(joint), (int(point[0]), int(point[1])), cv2.FONT_HERSHEY_COMPLEX, 1, # (0, 0, 250), 1) # cv2.imshow('kps ori', img_before) # cv2.imshow('kps after', img_after) # cv2.waitKey(0) return img_aug, new_bboxs, new_keypoints
def train(model): """Train the model.""" # Training dataset. dataset_train = WhaleDataset() dataset_train.load_whale(args.dataset, "train") dataset_train.prepare() # Validation dataset dataset_val = WhaleDataset() dataset_val.load_whale(args.dataset, "val") dataset_val.prepare() # *** This training schedule is an example. Update to your needs *** # Since we're using a very small dataset, and starting from # COCO trained weights, we don't need to train too long. Also, # no need to train all layers, just the heads should do it. # first run #layers_training='heads' # second run #layers_training='3+' # third run layers_training = 'all' epochs_to_train = 250 # adding image augmentation parameters augmentation = iaa.Sometimes( .667, iaa.Sequential( [ iaa.Fliplr(0.5), # horizontal flips iaa.Crop(percent=(0, 0.1)), # random crops # Small gaussian blur with random sigma between 0 and 0.25. # But we only blur about 50% of all images. iaa.Sometimes(0.5, iaa.GaussianBlur(sigma=(0, 0.25))), # Strengthen or weaken the contrast in each image. iaa.ContrastNormalization((0.75, 1.5)), # Add gaussian noise. # For 50% of all images, we sample the noise once per pixel. # For the other 50% of all images, we sample the noise per pixel AND # channel. This can change the color (not only brightness) of the # pixels. iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05 * 255)), # Make some images brighter and some darker. # In 20% of all cases, we sample the multiplier once per channel, # which can end up changing the color of the images. iaa.Multiply((0.8, 1.2)), # Apply affine transformations to each image. # Scale/zoom them, translate/move them, rotate them and shear them. iaa.Affine( scale={ "x": (0.8, 1.2), "y": (0.8, 1.2) }, #translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, rotate=(-180, 180), #shear=(-8, 8) ) ], random_order=True)) # apply augmenters in random order # old image aug parameters """ augmentation = iaa.Sometimes(0.9, [ iaa.Fliplr(0.5), iaa.Flipud(0.5), iaa.Multiply((0.8, 1.2)), iaa.GaussianBlur(sigma=(0.0, 5.0)) ]) """ print("Training 5+ layerss with augmentation.") print("*****Beginning training*****") print("config.LEARNING_RATE", config.LEARNING_RATE) print("layers_training:", layers_training) print("epochs_to_train:", epochs_to_train) print("augmentation: ", augmentation) print("---") print("Images: {}\nClasses: {}".format(len(dataset_train.image_ids), dataset_train.class_names)) #print("Training network in its entirety with augmentation") model.train(dataset_train, dataset_val, learning_rate=config.LEARNING_RATE, epochs=epochs_to_train, layers=layers_training, augmentation=augmentation)
def get_augs(): iaa_Blur = iaa.OneOf([ iaa.GaussianBlur((0, 1.5)), # blur images with a sigma between 0 and 1.5 iaa.AverageBlur(k=(2, 4)), # blur image using local means with kernel sizes between 2 and 7 iaa.MedianBlur(k=(3, 5)), # blur image using local medians with kernel sizes between 3 and 5 ]) iaa_Sharpen = iaa.OneOf([ iaa.Sharpen(alpha=(0, 0.5), lightness=(0.75, 1.5)), # sharpen images iaa.Emboss(alpha=(0, 0.5), strength=(0, 0.5)), # emboss images ]) iaa_Noise = iaa.OneOf([ iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.03*255), per_channel=0.1), # add gaussian noise to images iaa.Dropout((0, 0.02), per_channel=0.1), # randomly remove up to 10% of the pixels #iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2), ]) iaa_Affine = iaa.OneOf([ iaa.Affine( scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis) rotate=(-45, 45), # rotate by -45 to +45 degrees shear=(-16, 16), # shear by -16 to +16 degrees order=[0, 1], # use nearest neighbour or bilinear interpolation (fast) cval=(0, 255), # if mode is constant, use a cval between 0 and 255 mode='edge' # use zeros for padding ), iaa.PiecewiseAffine(scale=(0.01, 0.05)), # move parts of the image around # is it fast? iaa.PerspectiveTransform(scale=(0.01, 0.1)) # is it fast? #iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25), # move pixels locally around (with random strengths) # we need borders ]) iaa_HSV = iaa.OneOf([ iaa.Grayscale(alpha=(0.0, 0.5)), iaa.Add((-10, 10), per_channel=0.01), # change brightness of images (by -10 to 10 of original value) iaa.AddToHueAndSaturation((-20, 20)), # change hue and saturation iaa.Sequential([ iaa.ChangeColorspace(from_colorspace="RGB", to_colorspace="HSV"), iaa.WithChannels(0, iaa.Add((-100, 100))), iaa.ChangeColorspace(from_colorspace="HSV", to_colorspace="RGB") ]), iaa.ContrastNormalization((0.7, 1.6), per_channel=0), # improve or worsen the contrast iaa.Multiply((0.7, 1.2), per_channel=0.01), iaa.FrequencyNoiseAlpha( exponent=(-2, 2), first=iaa.Multiply((0.85, 1.2), per_channel=False), second=iaa.ContrastNormalization((0.5, 1.7))), ]) iaa_Simple = iaa.OneOf([ iaa.Fliplr(0.5), # horizontally flip 50% of all images iaa.Flipud(0.5), # vertically flip 20% of all images iaa.CropAndPad( percent=(-0.05, 0.1), pad_mode='edge', pad_cval=(0, 255) ), ]) augs = { 'simple': iaa_Simple, 'affine': iaa_Affine, 'hsv': iaa_HSV, 'sharpen': iaa_Sharpen, 'blur': iaa_Blur, 'noise': iaa_Noise, } return augs
def img_augment_run(input_path, output_path, coords_input, data_gen_args): flatten = lambda l: [obj for sublist in l for obj in sublist] img_list = glob.glob(os.path.join(input_path, "*.png")) print(img_list) coords = pd.read_csv(coords_input, header=[0, 1, 2], index_col=[0]) coords_aug = coords.copy() for img_num, img_name in enumerate(img_list): img = cv2.imread(img_name) coord_row_x = coords.iloc[img_num, 0::2] coord_row_y = coords.iloc[img_num, 1::2] coords_list = [ Keypoint(float(x), float(y)) for [x, y] in zip(coord_row_x, coord_row_y) ] keypoints = KeypointsOnImage(coords_list, shape=img.shape) seq = iaa.Sequential( [ iaa.Multiply( ( 0.5 - data_gen_args["brightness_range"], 0.5 + data_gen_args["brightness_range"], ) ), iaa.Affine( rotate=( -1 * data_gen_args["rotation_range"], data_gen_args["rotation_range"], ), scale=( 1 - data_gen_args["zoom_range"], 1 + data_gen_args["zoom_range"], ), shear=( -1 * data_gen_args["shear_range"], data_gen_args["shear_range"], ), translate_percent={ "x": ( -1 * data_gen_args["width_shift_range"], data_gen_args["width_shift_range"], ), "y": ( -1 * data_gen_args["height_shift_range"], data_gen_args["height_shift_range"], ), }, ), ] ) # Augment keypoints and images image_aug, kps_aug = seq(image=img, keypoints=keypoints) coords_aug.iloc[img_num, :] = flatten( [[kp.x, kp.y] for kp in kps_aug.keypoints] ) idx_name = coords_aug.index[img_num] idx_basename = os.path.basename(idx_name) coords_aug.rename( index={ idx_name: idx_name.replace( idx_basename, "{}_aug.png".format(idx_basename.split(".")[0]) ) }, inplace=True, ) print( idx_name.replace( idx_basename, "{}_aug.png".format(idx_basename.split(".")[0]) ) ) io.imsave(os.path.join(output_path, os.path.basename(img_name)), img) io.imsave( os.path.join( output_path, "{}_aug.png".format(os.path.basename(img_name).split(".")[0]), ), image_aug, ) # Adapted from DeepLabCut (to facilitate conversion to DLC-compatible format): # https://github.com/DeepLabCut/DeepLabCut/blob/master/deeplabcut/generate_training_dataset/labeling_toolbox.py coords_aug.sort_index(inplace=True) coords_aug = coords_aug.append(coords) coords_aug.to_csv( os.path.join( output_path, "{}.csv".format(os.path.basename(coords_input).split(".")[0]) ) ) coords_aug.to_hdf( os.path.join( output_path, "{}.h5".format(os.path.basename(coords_input).split(".")[0]) ), "df_with_missing", format="table", mode="w", )
def img_aug_fuc(img, kps): ''' function that used for data augmentation :param img: ori_image that was prepared to run aug, shape must be [h, w, c] :param kps: a ndarray that contains keypoints on this image. shape must be [person_num, joints_num, 3]. person_num means that there contains 'person_num' person on this image joints_num means that we want to detect how many joints. e.g. like 1 for single head point or 14 for all body points in ai-challenger format. 3 means [x, y, v], v is the visable attribute for one joint point. :return: img and kps after augmentation. ''' kps_ori = np.copy(kps) # 定义一个变换序列, 包括 # 1. 改变亮度 # 2. (-90, 90)内旋转并缩放 # 3. 随机变成灰度图 # 4. 随机水平翻转 # 5. 随机上下翻转 p_flip = np.random.randint(0, 2, (2)) # p_flip = [1, 0] seq = iaa.Sequential([ iaa.Fliplr(p_flip[0]), # 50% 水平flip iaa.Flipud(p_flip[1]), # 50% vertical flip iaa.Multiply((0.7, 1.5)), # 改变亮度,不影响关键点 (后面可逐步增大至4, 5, 6) iaa.Affine(rotate=(-45, 45), scale=(0.5, 0.7), mode='constant'), # 旋转然后缩放 iaa.Grayscale((0.0, 1.0)), # 随机变成灰度图 ]) seq_det = seq.to_deterministic() # 定义keypoints keypoints = ia.KeypointsOnImage([], shape=img.shape) person_num, joints_num, _ = kps.shape for person in range(person_num): for joint in range(joints_num): point = kps[person][joint] keypoints.keypoints.append(ia.Keypoint(x=point[0], y=point[1])) # 执行augmentation img_aug = seq_det.augment_image(img).copy() kps_aug = seq_det.augment_keypoints([keypoints])[0] # 返回augmentation之后的keypoints ret_kps = [] for i in range(len(keypoints.keypoints)): point_after = kps_aug.keypoints[i] ret_kps.append([point_after.x, point_after.y, 1]) ret_kps = np.reshape(np.asarray(ret_kps), newshape=(-1, joints_num, 3)) assert img_aug.shape == img.shape assert ret_kps.shape == kps_ori.shape # keep ori keypoint visiable attribute for person in range(ret_kps.shape[0]): for joint in range(ret_kps.shape[1]): ret_kps[person][joint][2] = int(kps_ori[person][joint][2]) # import cv2 # img_test = img_aug.copy() # for person in range(ret_kps.shape[0]): # for joint in range(ret_kps.shape[1]): # point = ret_kps[person][joint] # cv2.circle(img_test, (int(point[0]), int(point[1])), 2, (255, 0, 0), -1) # cv2.putText(img_test, str(joint), (int(point[0]), int(point[1])), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1) # cv2.imshow('after 0 ', img_test) # cv2.waitKey(0) if joints_num > 2: import copy # exchange left and right point # [ 0-right_shoulder, 1-right_elbow, 2-right_wrist, 3-left_shoulder, 4-left_elbow, 5-left_wrist, # 6-right_hip, 7-right_knee, 8-right_ankle, 9-left_hip, 10-left_knee, 11-left_ankle, 12-head, 13-neck ] change_index = [[0, 3], [1, 4], [2, 5], [6, 9], [7, 10], [8, 11]] for p in p_flip: if p == 1: for person in range(ret_kps.shape[0]): for index in change_index: left_point = copy.copy(ret_kps[person][index[0]]) ret_kps[person][index[0]] = ret_kps[person][index[1]] ret_kps[person][index[1]] = left_point # image_before = keypoints.draw_on_image(img, size=7) # # image_after = kps_aug.draw_on_image(img_aug, size=7) # for person in range(ret_kps.shape[0]): # for joint in range(ret_kps.shape[1]): # point = kps_ori[person][joint] # # print (point) # cv2.circle(img, (int(point[0]), int(point[1])), 2, (255, 0, 0), -1) # cv2.putText(img, str(joint) + str(point[2]), (int(point[0]), int(point[1])), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1) # # cv2.imshow('before', img) # cv2.waitKey(0) # for person in range(ret_kps.shape[0]): # for joint in range(ret_kps.shape[1]): # point = ret_kps[person][joint] # # print (point) # cv2.circle(img_aug, (int(point[0]), int(point[1])), 2, (255, 0, 0), -1) # cv2.putText(img_aug, str(point[2]), (int(point[0]), int(point[1])), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1) # cv2.imshow('after', img_aug) # cv2.waitKey(0) # print (ret_kps) return img_aug, ret_kps
# In[ ]: # Image augmentation augmentation = iaa.SomeOf((0, 1), [ iaa.Fliplr(0.5), iaa.Affine(scale={ "x": (0.8, 1.2), "y": (0.8, 1.2) }, translate_percent={ "x": (-0.2, 0.2), "y": (-0.2, 0.2) }, rotate=(-25, 25), shear=(-8, 8)), iaa.Multiply((0.9, 1.1)) ]) # ### Now it's time to train the model. Note that training even a basic model can take a few hours. # # Note: the following model is for demonstration purpose only. We have limited the training to one epoch, and have set nominal values for the Detector Configuration to reduce run-time. # # - dataset_train and dataset_val are derived from DetectorDataset # - DetectorDataset loads images from image filenames and masks from the annotation data # - model is Mask-RCNN # In[ ]: NUM_EPOCHS = 1 # Train Mask-RCNN Model
def main(): IN_JSON = 'dataset_descriptions/iamondb_generated_dates_10k.json' OUT_JSON = 'dataset_descriptions/iamondb_generated_dates_resized_aug2_10k.json' IMG_IN_DIR = 'datasets/tars' IMG_OUT_DIR = 'datasets/' SCALE_FACTOR = 2 # How much bigger the new dataset should be SAVE_ORIGINAL = False TARGET_DIMENSIONS = (216, 64) new_image_infos = [] with open(IN_JSON, 'r') as json_file: json_images = json.load(json_file) image_infos = copy.deepcopy(json_images) for idx in range(0, len(image_infos)): image_infos[idx]['path'] = os.path.join( IMG_IN_DIR, os.path.basename(image_infos[idx]['path'])) seq = iaa.Sequential( [ iaa.Sometimes(0.5, iaa.GaussianBlur( sigma=(0, 0.5))), # Small blur on half of the images iaa.ContrastNormalization((0.75, 1.5)), # Change in contrast iaa.AdditiveGaussianNoise( loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5), # Gaussian Noise can change colour iaa.Multiply((0.8, 1.2), per_channel=0.2) #, # brighter/darker # iaa.Affine( # rotate=(-10, 10), # shear=(-20, 20), # fit_output=True, # ), # iaa.PiecewiseAffine(scale=(0.01, 0.05), mode='edge') ], random_order=True) # apply augmenters in random order images = np.array( [np.array(Image.open(img['path'])) for img in image_infos]) images = np.repeat(images, SCALE_FACTOR, axis=0) image_infos = np.repeat(np.array(image_infos), SCALE_FACTOR, axis=0) # Make batches out of single images because processing multiple images at once leads to errors because the resulting # images have different shapes and therefore can't be organised in a numpy array -> leads to an internal error for idx, img in enumerate(images): old_filename = os.path.splitext( os.path.basename(image_infos[idx]['path']))[0] new_filename = old_filename + '-' + str(idx % SCALE_FACTOR + 1) + '.png' new_img_path = os.path.join(IMG_OUT_DIR, new_filename) if SAVE_ORIGINAL and idx % SCALE_FACTOR == 0: with open(os.path.join(IMG_OUT_DIR, old_filename + '.png'), 'wb+') as orig_img_file: resize_img(Image.fromarray(img), TARGET_DIMENSIONS).save(orig_img_file, format='PNG') padded_img = resize_img(Image.fromarray(img), TARGET_DIMENSIONS, padding_color=255) img_aug = seq(images=np.expand_dims(np.asarray(padded_img), axis=0)) pil_img = Image.fromarray(np.squeeze(img_aug)).convert('L') with open(new_img_path, 'wb+') as img_file: pil_img.save(img_file, format='PNG') new_image_infos.append({ "string": image_infos[idx]['string'], "type": image_infos[idx]['type'], "path": new_img_path }) with open(OUT_JSON, 'w') as out_json: if SAVE_ORIGINAL: assert os.path.split(json_images[0]['path'])[0] == os.path.split(new_image_infos[0]['path'])[0],\ "Original path differs from new path" json.dump(json_images + new_image_infos, out_json, ensure_ascii=False, indent=4) else: json.dump(new_image_infos, out_json, ensure_ascii=False, indent=4) print( f'Created {len(new_image_infos)} new images (original: {len(json_images)})' )
print("Image area <= {:4}**2: None".format(np.sqrt(image_area))) continue print( "Image area <= {:4.0f}**2: mean: {:.1f} median: {:.1f} min: {:.1f} max: {:.1f}" .format(np.sqrt(image_area), tumor_per_image.mean(), np.median(tumor_per_image), tumor_per_image.min(), tumor_per_image.max())) ax[i].set_title("Image Area <= {:4}**2".format(np.sqrt(image_area))) _ = ax[i].hist(tumor_per_image, bins=10) # List of augmentations # http://imgaug.readthedocs.io/en/latest/source/augmenters.html augmentation = iaa.Sometimes(0.9, [ iaa.Fliplr(0.5), iaa.Flipud(0.5), iaa.Multiply((0.8, 1.2)), iaa.GaussianBlur(sigma=(0.0, 5.0)) ]) # Load the image multiple times to show augmentations limit = 4 ax = get_ax(rows=2, cols=limit // 2) for i in range(limit): image, image_meta, class_ids, bbox, mask = modellib.load_image_gt( dataset, config, image_id, use_mini_mask=False, augment=True, augmentation=augmentation) visualize.display_instances(image,
def get_default_imgaug(): # Возвращает набор аугментаций imgaug'а, который по нашей договоренности является дефолтным (наиболее используемым) # Набор аугментаций уже непосредствено можно применять к данным # Вызывается конструктором DataGenerator, если в конструктор DataGenerator не передан свой набор аугментаций. # По большей части повторяет пример из документации: https://imgaug.readthedocs.io/en/latest/source/examples_basics.html#a-simple-and-common-augmentation-sequence sometimes = lambda aug: iaa.Sometimes(0.5, aug) return iaa.Sequential( [ iaa.Fliplr(0.5), # horizontally flip 50% of all images iaa.Flipud(0.2), # vertically flip 20% of all images sometimes(iaa.Crop(percent=(0, 0.1))), sometimes( iaa.Affine(scale={ "x": (0.8, 1.2), "y": (0.8, 1.2) }, translate_percent={ "x": (-0.2, 0.2), "y": (-0.2, 0.2) }, rotate=(-45, 45), shear=(-16, 16), order=[0, 1], cval=(0, 255), mode=ia.ALL)), iaa.SomeOf( (0, 5), [ sometimes( iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), iaa.OneOf([ iaa.GaussianBlur((0, 3.0)), iaa.AverageBlur(k=(2, 7)), iaa.MedianBlur(k=(3, 11)), ]), iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # Same as sharpen, but for an embossing effect. iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), sometimes( iaa.OneOf([ iaa.EdgeDetect(alpha=(0, 0.7)), iaa.DirectedEdgeDetect(alpha=(0, 0.7), direction=(0.0, 1.0)), ])), iaa.AdditiveGaussianNoise( loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5), iaa.OneOf([ iaa.Dropout((0.01, 0.1), per_channel=0.5), iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2), ]), iaa.Invert(0.05, per_channel=True), # invert color channels # Add a value of -10 to 10 to each pixel. iaa.Add((-10, 10), per_channel=0.5), # Change brightness of images (50-150% of original value). iaa.Multiply((0.5, 1.5), per_channel=0.5), # Improve or worsen the contrast of images. #iaa.LinearContrast((0.5, 2.0), per_channel=0.5), ПОЧЕМУ ТО НЕ РОБИТ # Convert each image to grayscale and then overlay the # result with the original with random alpha. I.e. remove # colors with varying strengths. iaa.Grayscale(alpha=(0.0, 1.0)), # In some images move pixels locally around (with random # strengths). sometimes( iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25) ), # In some images distort local areas with varying strength. sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) ], # do all of the above augmentations in random order random_order=True) ], # do all of the above augmentations in random order random_order=True)
iaa.BlendAlphaSimplexNoise(iaa.OneOf([ iaa.EdgeDetect(alpha=(0.5, 1.0)), iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)), ])), iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5), # add gaussian noise to images iaa.OneOf([ iaa.Dropout((0.01, 0.1), per_channel=0.5), # randomly remove up to 10% of the pixels iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2), ]), iaa.Invert(0.05, per_channel=True), # invert color channels iaa.Add((-10, 10), per_channel=0.5), # change brightness of images (by -10 to 10 of original value) iaa.AddToHueAndSaturation((-20, 20)), # change hue and saturation # either change the brightness of the whole image (sometimes # per channel) or change the brightness of subareas iaa.OneOf([ iaa.Multiply((0.5, 1.5), per_channel=0.5), iaa.BlendAlphaFrequencyNoise( exponent=(-4, 0), foreground=iaa.Multiply((0.5, 1.5), per_channel=True), background=iaa.LinearContrast((0.5, 2.0)) ) ]), iaa.LinearContrast((0.5, 2.0), per_channel=0.5), # improve or worsen the contrast iaa.Grayscale(alpha=(0.0, 1.0)), sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths) sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))), # sometimes move parts of the image around sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1))) ], random_order=True ) ],
def get_example(self, i): files_dir = self._files_dirs[i] image_file = osp.join(files_dir, 'image.png') image = skimage.io.imread(image_file) assert image.dtype == np.uint8 assert image.ndim == 3 depth_file = osp.join(files_dir, 'depth.npz') depth = np.load(depth_file)['arr_0'] depth[depth == 0] = np.nan if depth.dtype == np.uint16: depth = depth.astype(np.float32) depth /= 1000 depth_keep = ~np.isnan(depth) depth[depth_keep] = np.maximum(depth[depth_keep], self.min_value) depth[depth_keep] = np.minimum(depth[depth_keep], self.max_value) assert depth.dtype == np.float32 assert depth.ndim == 2 label_file = osp.join(files_dir, 'label.png') with open(label_file, 'r') as f: label = np.asarray(PIL.Image.open(f)).astype(np.int32) assert label.dtype == np.int32 assert label.ndim == 2 depth_gt_file = osp.join(files_dir, 'depth_gt.npz') depth_gt = np.load(depth_gt_file)['arr_0'] depth_gt[depth_gt == 0] = np.nan if depth_gt.dtype == np.uint16: depth_gt = depth_gt.astype(np.float32) depth /= 1000 depth_gt_keep = ~np.isnan(depth_gt) depth_gt[depth_gt_keep] = np.maximum(depth_gt[depth_gt_keep], self.min_value) depth_gt[depth_gt_keep] = np.minimum(depth_gt[depth_gt_keep], self.max_value) assert depth_gt.dtype == np.float32 assert depth_gt.ndim == 2 # Data augmentation if self.aug: # 1. Color augmentation obj_datum = dict(img=image) random_state = np.random.RandomState() def st(x): return iaa.Sometimes(0.3, x) augs = [ st(iaa.Add([-50, 50], per_channel=True)), st( iaa.InColorspace('HSV', children=iaa.WithChannels( [1, 2], iaa.Multiply([0.5, 2])))), st(iaa.GaussianBlur(sigma=[0.0, 1.0])), st( iaa.AdditiveGaussianNoise(scale=(0.0, 0.1 * 255), per_channel=True)), ] obj_datum = next( mvtk.aug.augment_object_data([obj_datum], random_state=random_state, augmentations=augs)) image = obj_datum['img'] # 2. Depth noise np.random.seed() if np.random.uniform() < 0.3: noise_rate = np.random.uniform() * 0.25 + 0.05 depth[np.random.rand(depth.shape[0], depth.shape[1]) < noise_rate] = np.nan # 3. Geometric augmentation if np.random.uniform() < 0.5: image = np.fliplr(image) depth = np.fliplr(depth) label = np.fliplr(label) depth_gt = np.fliplr(depth_gt) if np.random.uniform() < 0.5: image = np.flipud(image) depth = np.flipud(depth) label = np.flipud(label) depth_gt = np.flipud(depth_gt) if np.random.uniform() < 0.5: angle = (np.random.uniform() * 180) - 90 image = self.rotate_image(image, angle, cv2.INTER_LINEAR) depth = self.rotate_depth_image(depth, angle, cv2.INTER_LINEAR) label = self.rotate_image(label, angle, cv2.INTER_NEAREST) depth_gt = self.rotate_depth_image(depth_gt, angle, cv2.INTER_LINEAR) return image, depth, label, depth_gt
def train(model): """Train the model.""" # Training dataset. dataset_train = Deep_Lesion_Dataset() ############################## We need to separate the training and validation ### There is a variable in the dict, which is training_val_testing ### It's in image_info file, # under annotations, image #annotations['images'][0]['Train_Val_Test'] ### TRAIN = 1 ### VALIDATION = 2 ### TEST = 3 ###### NEED TO EXRACT THREE DIFFERENT DATASETS based on this. ## look at how "train"/"valid" is fed in below dataset_train.load_deep_lesion(args.dataset, "train") dataset_train.prepare() # Validation dataset dataset_val = Deep_Lesion_Dataset() dataset_val.load_deep_lesion(args.dataset, "val") dataset_val.prepare() augmentation = iaa.SomeOf( (0, 1), [ #iaa.Fliplr(0.5), iaa.Affine(scale={ "x": (0.9, 1.1), "y": (0.9, 1.1) }, translate_percent={ "x": (-0.1, 0.1), "y": (-0.1, 0.1) }, rotate=(-5, 5), shear=(-1, 1)), iaa.Multiply((0.9, 1.1)) ]) ######################################################################################## from keras.callbacks import ReduceLROnPlateau reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.3, patience=patience1, verbose=1, mode='min', cooldown=0, min_lr=0) callbacks = [reduce_lr] print("Training Images: {}\nClasses: {}".format( len(dataset_train.image_ids), dataset_train.class_names)) print("Validations Images: {}\nClasses: {}".format( len(dataset_val.image_ids), dataset_val.class_names)) # *** This training schedule is an example. Update to your needs *** # Since we're using a very small dataset, and starting from # COCO trained weights, we don't need to train too long. Also, # no need to train all layers, just the heads should do it. print("Training network heads") model.train( dataset_train, dataset_val, learning_rate=config.LEARNING_RATE, #epochs=30, #layers='heads') epochs=20, layers='all', custom_callbacks=callbacks, ############# augmentation=augmentation) ###################### print("finised training network")
iaa.Dropout((0.01, 0.1), per_channel=0.5 ), # randomly remove up to 10% of the pixels iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2), ]), iaa.Invert(0.05, per_channel=True), # invert color channels iaa.Add( (-10, 10), per_channel=0.5 ), # change brightness of images (by -10 to 10 of original value) iaa.AddToHueAndSaturation( (-20, 20)), # change hue and saturation # either change the brightness of the whole image (sometimes # per channel) or change the brightness of subareas iaa.OneOf([ iaa.Multiply((0.5, 1.5), per_channel=0.5), iaa.FrequencyNoiseAlpha(exponent=(-4, 0), first=iaa.Multiply( (0.5, 1.5), per_channel=True), second=iaa.LinearContrast( (0.5, 2.0))) ]), iaa.LinearContrast( (0.5, 2.0), per_channel=0.5), # improve or worsen the contrast iaa.Grayscale(alpha=(0.0, 1.0)), sometimes( iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25) ), # move pixels locally around (with random strengths) sometimes(iaa.PiecewiseAffine(scale=( 0.01, 0.05))), # sometimes move parts of the image around
def imgaug_img(img): # Sometimes(0.5, ...) applies the given augmenter in 50% of all cases, # e.g. Sometimes(0.5, GaussianBlur(0.3)) would blur roughly every second # image. sometimes = lambda aug: iaa.Sometimes(0.5, aug) # Define our sequence of augmentation steps that will be applied to every image. seq = iaa.Sequential( [ # # Apply the following augmenters to most images. # iaa.Fliplr(0.5), # horizontally flip 50% of all images # iaa.Flipud(0.2), # vertically flip 20% of all images # crop some of the images by 0-10% of their height/width sometimes(iaa.Crop(percent=(0, 0.1))), # Apply affine transformations to some of the images # - scale to 80-120% of image height/width (each axis independently) # - translate by -20 to +20 relative to height/width (per axis) # - rotate by -45 to +45 degrees # - shear by -16 to +16 degrees # - order: use nearest neighbour or bilinear interpolation (fast) # - mode: use any available mode to fill newly created pixels # see API or scikit-image for which modes are available # - cval: if the mode is constant, then use a random brightness # for the newly created pixels (e.g. sometimes black, # sometimes white) sometimes( iaa.Affine(scale={ "x": (0.8, 1.2), "y": (0.8, 1.2) }, translate_percent={ "x": (-0.2, 0.2), "y": (-0.2, 0.2) }, rotate=(-45, 45), shear=(-16, 16), order=[0, 1], cval=(0, 255), mode=ia.ALL)), # # Execute 0 to 5 of the following (less important) augmenters per # image. Don't execute all of them, as that would often be way too # strong. # iaa.SomeOf( (0, 5), [ # Convert some images into their superpixel representation, # sample between 20 and 200 superpixels per image, but do # not replace all superpixels with their average, only # some of them (p_replace). sometimes( iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # Blur each image with varying strength using # gaussian blur (sigma between 0 and 3.0), # average/uniform blur (kernel size between 2x2 and 7x7) # median blur (kernel size between 3x3 and 11x11). iaa.OneOf([ iaa.GaussianBlur((0, 3.0)), iaa.AverageBlur(k=(2, 7)), iaa.MedianBlur(k=(3, 11)), ]), # Sharpen each image, overlay the result with the original # image using an alpha between 0 (no sharpening) and 1 # (full sharpening effect). iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # Same as sharpen, but for an embossing effect. iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # Search in some images either for all edges or for # directed edges. These edges are then marked in a black # and white image and overlayed with the original image # using an alpha of 0 to 0.7. sometimes( iaa.OneOf([ iaa.EdgeDetect(alpha=(0, 0.7)), iaa.DirectedEdgeDetect(alpha=(0, 0.7), direction=(0.0, 1.0)), ])), # Add gaussian noise to some images. # In 50% of these cases, the noise is randomly sampled per # channel and pixel. # In the other 50% of all cases it is sampled once per # pixel (i.e. brightness change). iaa.AdditiveGaussianNoise( loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5), # Either drop randomly 1 to 10% of all pixels (i.e. set # them to black) or drop them on an image with 2-5% percent # of the original size, leading to large dropped # rectangles. iaa.OneOf([ iaa.Dropout((0.01, 0.1), per_channel=0.5), iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2), ]), # Invert each image's channel with 5% probability. # This sets each pixel value v to 255-v. iaa.Invert(0.05, per_channel=True), # invert color channels # Add a value of -10 to 10 to each pixel. iaa.Add((-10, 10), per_channel=0.5), # Change brightness of images (50-150% of original value). iaa.Multiply((0.5, 1.5), per_channel=0.5), # Improve or worsen the contrast of images. iaa.LinearContrast((0.5, 2.0), per_channel=0.5), # Convert each image to grayscale and then overlay the # result with the original with random alpha. I.e. remove # colors with varying strengths. iaa.Grayscale(alpha=(0.0, 1.0)), # In some images move pixels locally around (with random # strengths). sometimes( iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25) ), # In some images distort local areas with varying strength. sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) ], # do all of the above augmentations in random order random_order=True) ], # do all of the above augmentations in random order random_order=True) images_aug = seq.augment_images([img]) return images_aug[0]
meta = {'noop': iaa.Noop(), 'shuffle': iaa.ChannelShuffle(p=1.0)} arithmetic = { 'add-45': iaa.Add(value=-45), 'add-25': iaa.Add(value=-25), 'add+25': iaa.Add(value=25), 'add+45': iaa.Add(value=45), 'addp-': iaa.Add(value=(-35, -15), per_channel=True), 'addp+': iaa.Add(value=(15, 35), per_channel=True), 'addGN': iaa.AdditiveGaussianNoise(scale=0.10 * 255), 'addGNp': iaa.AdditiveGaussianNoise(scale=0.10 * 255, per_channel=True), 'addLN': iaa.AdditiveLaplaceNoise(scale=0.10 * 255), 'addLNp': iaa.AdditiveLaplaceNoise(scale=0.10 * 255, per_channel=True), 'addPN': iaa.AdditivePoissonNoise(lam=16.00), 'addPNp': iaa.AdditivePoissonNoise(lam=16.00, per_channel=True), 'mul-': iaa.Multiply(mul=0.50), 'mul+': iaa.Multiply(mul=1.50), 'mulp-': iaa.Multiply(mul=0.50, per_channel=True), 'mulp+': iaa.Multiply(mul=1.50, per_channel=True), 'jpeg': iaa.JpegCompression(compression=62), 'jpeg+': iaa.JpegCompression(compression=75), 'jpeg++': iaa.JpegCompression(compression=87) } blur = { 'GBlur': iaa.GaussianBlur(sigma=1.00), 'ABlur': iaa.AverageBlur(k=3), 'MBlur': iaa.MedianBlur(k=3), 'BBlur': iaa.BilateralBlur(sigma_color=250, sigma_space=250, d=5), 'MoBlur': iaa.MotionBlur(angle=0, k=7), 'MoBlurAng': iaa.MotionBlur(angle=144, k=5)
def main(): augmenters = [ iaa.Identity(name="Identity"), iaa.Crop(px=(0, 8), name="Crop-px"), iaa.Crop(percent=(0, 0.1), name="Crop-percent"), iaa.Fliplr(0.5, name="Fliplr"), iaa.Flipud(0.5, name="Flipud"), iaa.Grayscale((0.0, 1.0), name="Grayscale"), iaa.GaussianBlur((0, 3.0), name="GaussianBlur"), iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.1), name="AdditiveGaussianNoise"), iaa.Dropout((0.0, 0.1), name="Dropout"), iaa.Multiply((0.5, 1.5), name="Multiply"), iaa.ContrastNormalization(alpha=(0.5, 2.0), name="ContrastNormalization"), iaa.Grayscale(alpha=(0.0, 1.0), name="Grayscale"), iaa.ElasticTransformation(alpha=(0.5, 8.0), sigma=1.0, name="ElasticTransformation"), iaa.Affine(scale={ "x": (0.8, 1.2), "y": (0.8, 1.2) }, translate_px={ "x": (-16, 16), "y": (-16, 16) }, rotate=(-45, 45), shear=(-16, 16), order=0, cval=(0, 255), mode="constant", name="AffineOrder0ModeConstant") ] for order in [0, 1]: augmenters.append( iaa.Affine(scale={ "x": (0.8, 1.2), "y": (0.8, 1.2) }, translate_px={ "x": (-16, 16), "y": (-16, 16) }, rotate=(-45, 45), shear=(-16, 16), order=order, cval=(0, 255), mode=ia.ALL, name="AffineOrder%d" % (order, ))) augmenters.append( iaa.Affine(scale={ "x": (0.8, 1.2), "y": (0.8, 1.2) }, translate_px={ "x": (-16, 16), "y": (-16, 16) }, rotate=(-45, 45), shear=(-16, 16), order=ia.ALL, cval=(0, 255), mode=ia.ALL, name="AffineAll")) kps = [] for _ in sm.xrange(20): x = random.randint(0, 31) y = random.randint(0, 31) kps.append(ia.Keypoint(x=x, y=y)) kps = ia.KeypointsOnImage(kps, shape=(32, 32, 3)) small_keypoints_one = kps.on((4, 4, 3)) medium_keypoints_one = kps.on((32, 32, 3)) large_keypoints_one = kps.on((256, 256, 3)) small_keypoints = [small_keypoints_one.deepcopy() for _ in sm.xrange(16)] medium_keypoints = [medium_keypoints_one.deepcopy() for _ in sm.xrange(16)] large_keypoints = [large_keypoints_one.deepcopy() for _ in sm.xrange(16)] small_images = np.random.randint(0, 255, (16, 4, 4, 3)).astype(np.uint8) medium_images = np.random.randint(0, 255, (16, 32, 32, 3)).astype(np.uint8) large_images = np.random.randint(0, 255, (16, 256, 256, 3)).astype(np.uint8) print("---------------------------") print("Keypoints") print("---------------------------") for augmenter in augmenters: print("[Augmenter: %s]" % (augmenter.name, )) for keypoints in [small_keypoints, medium_keypoints, large_keypoints]: times = [] for i in sm.xrange(100): time_start = time.time() _img_aug = augmenter.augment_keypoints(keypoints) time_end = time.time() times.append(time_end - time_start) times = np.array(times) img_str = "{:20s}".format(keypoints[0].shape) print("%s | SUM %.5fs | PER ITER avg %.5fs, min %.5fs, max %.5fs" % (img_str, float(np.sum(times)), np.average(times), np.min(times), np.max(times))) print("---------------------------") print("Images") print("---------------------------") for augmenter in augmenters: print("[Augmenter: %s]" % (augmenter.name, )) for images in [small_images, medium_images, large_images]: times = [] for i in sm.xrange(100): time_start = time.time() _img_aug = augmenter.augment_images(images) time_end = time.time() times.append(time_end - time_start) times = np.array(times) img_str = "{:20s}".format(images.shape) print("%s | SUM %.5fs | PER ITER avg %.5fs, min %.5fs, max %.5fs" % (img_str, float(np.sum(times)), np.average(times), np.min(times), np.max(times)))
from imgaug import augmenters as iaa warnings.simplefilter('ignore', category=NumbaDeprecationWarning) warnings.simplefilter('ignore', category=NumbaPendingDeprecationWarning) seq = iaa.Sequential([ iaa.Resize(config.resize_wh), iaa.Sequential([ iaa.Fliplr(0.5), iaa.Flipud(0.5), iaa.Crop(percent=(0, 0.2)), iaa.Sometimes( 0.5, iaa.GaussianBlur(sigma=(0, 0.1)), iaa.CoarseDropout( (0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2)), iaa.ContrastNormalization((0.75, 1.5)), iaa.Multiply((0.8, 1.2), per_channel=0.2), iaa.Affine(scale={ "x": (0.9, 1.1), "y": (0.9, 1.1) }, translate_percent={ "x": (-0.1, 0.1), "y": (-0.1, 0.1) }, rotate=(-25, 25), shear=0) ], random_order=True) ], random_order=False)
def RandMultiply(img): return iaa.Multiply((0.9, 1.1)).augment_image(img)
def _load_augmentation_aug_all(): """ Load image augmentation model """ def sometimes(aug): return iaa.Sometimes(0.5, aug) return iaa.Sequential( [ # apply the following augmenters to most images iaa.Fliplr(0.5), # horizontally flip 50% of all images iaa.Flipud(0.2), # vertically flip 20% of all images # crop images by -5% to 10% of their height/width sometimes( iaa.CropAndPad(percent=(-0.05, 0.1), pad_mode='constant', pad_cval=(0, 255))), sometimes( iaa.Affine( # scale images to 80-120% of their size, individually per axis scale={ "x": (0.8, 1.2), "y": (0.8, 1.2) }, # translate by -20 to +20 percent (per axis) translate_percent={ "x": (-0.2, 0.2), "y": (-0.2, 0.2) }, rotate=(-45, 45), # rotate by -45 to +45 degrees shear=(-16, 16), # shear by -16 to +16 degrees # use nearest neighbour or bilinear interpolation (fast) order=[0, 1], # if mode is constant, use a cval between 0 and 255 cval=(0, 255), # use any of scikit-image's warping modes # (see 2nd image from the top for examples) mode='constant')), # execute 0 to 5 of the following (less important) augmenters per # image don't execute all of them, as that would often be way too # strong iaa.SomeOf( (0, 5), [ # convert images into their superpixel representation sometimes( iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), iaa.OneOf([ # blur images with a sigma between 0 and 3.0 iaa.GaussianBlur((0, 3.0)), # blur image using local means with kernel sizes # between 2 and 7 iaa.AverageBlur(k=(2, 7)), # blur image using local medians with kernel sizes # between 2 and 7 iaa.MedianBlur(k=(3, 11)), ]), iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images # search either for all edges or for directed edges, # blend the result with the original image using a blobby mask iaa.BlendAlphaSimplexNoise( iaa.OneOf([ iaa.EdgeDetect(alpha=(0.5, 1.0)), iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)), ])), # add gaussian noise to images iaa.AdditiveGaussianNoise( loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5), iaa.OneOf([ # randomly remove up to 10% of the pixels iaa.Dropout((0.01, 0.1), per_channel=0.5), iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2), ]), # invert color channels iaa.Invert(0.05, per_channel=True), # change brightness of images (by -10 to 10 of original value) iaa.Add((-10, 10), per_channel=0.5), # change hue and saturation iaa.AddToHueAndSaturation((-20, 20)), # either change the brightness of the whole image (sometimes # per channel) or change the brightness of subareas iaa.OneOf([ iaa.Multiply((0.5, 1.5), per_channel=0.5), iaa.BlendAlphaFrequencyNoise( exponent=(-4, 0), foreground=iaa.Multiply( (0.5, 1.5), per_channel=True), background=iaa.contrast.LinearContrast((0.5, 2.0))) ]), # improve or worsen the contrast iaa.contrast.LinearContrast((0.5, 2.0), per_channel=0.5), iaa.Grayscale(alpha=(0.0, 1.0)), # move pixels locally around (with random strengths) sometimes( iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25) ), # sometimes move parts of the image around sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))), sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1))) ], random_order=True) ], random_order=True)
shutil.rmtree(AUG_XML_DIR) except FileNotFoundError as e: a = 1 mkdir(AUG_XML_DIR) AUGLOOP = 15 # 每张影像增强的数量 boxes_img_aug_list = [] new_bndbox = [] new_bndbox_list = [] # 影像增强 seq = iaa.Sequential([ iaa.Flipud(0.5), # vertically flip 20% of all images iaa.Fliplr(0.5), # 镜像 iaa.Multiply((1.2, 1.5)), # change brightness, doesn't affect BBs iaa.GaussianBlur(sigma=(0, 3.0)), # iaa.GaussianBlur(0.5), iaa.Affine( translate_px={ "x": 15, "y": 15 }, scale=(0.8, 0.95), rotate=(-30, 30) ) # translate by 40/60px on x/y axis, and scale to 50-70%, affects BBs ]) for root, sub_folders, files in os.walk(XML_DIR): for name in files:
def img_random_brightness(image): brightness = iaa.Multiply((0.2, 1.2)) image = brightness.augment_image(image) return image
def __init__(self): configMain.__init__(self) st = lambda aug: iaa.Sometimes(0.4, aug) oc = lambda aug: iaa.Sometimes(0.3, aug) rl = lambda aug: iaa.Sometimes(0.09, aug) self.augment = iaa.Sequential([ rl(iaa.GaussianBlur((0, 1.5))), # blur images with a sigma between 0 and 1.5 rl(iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05), per_channel=0.5)), # add gaussian noise to images oc(iaa.Dropout((0.0, 0.10), per_channel=0.5)), # randomly remove up to X% of the pixels oc(iaa.CoarseDropout((0.0, 0.10), size_percent=(0.08, 0.2),per_channel=0.5)), # randomly remove up to X% of the pixels oc(iaa.Add((-40, 40), per_channel=0.5)), # change brightness of images (by -X to Y of original value) st(iaa.Multiply((0.10, 2.5), per_channel=0.2)), # change brightness of images (X-Y% of original value) rl(iaa.ContrastNormalization((0.5, 1.5), per_channel=0.5)), # improve or worsen the contrast rl(iaa.Grayscale((0.0, 1))), # put grayscale ], random_order=True # do all of the above in random order ) self.augment_labels = True self.augment_amount = 2 #3=max, 2=mid, 1=min self.labels_to_augment = {"road": True, "buildings": True, "grass": True, "sky_n_zebra": True } # there are files with data, 200 images each, and here we select which ones to use #5self.dataset_name = 'Carla' #with open(os.path.join(self.save_data_stats, 'path'),'r') as f: # path = f.read().strip() path = '../VirtualElektraData2_Double' train_path = os.path.join(path, 'SeqTrain') val_path = os.path.join(path, 'SeqVal') print train_path, val_path self.train_db_path = [os.path.join(train_path, f) for f in glob.glob1(train_path, "data_*.h5")] self.val_db_path = [os.path.join(val_path, f) for f in glob.glob1(val_path, "data_*.h5")] # When using data with noise, remove the recording during the first half of the noise impulse # TODO Felipe: change to noise percentage. self.remove_noise = False # Speed Divide Factor #TODO: FOr now is hardcooded, but eventually we should be able to calculate this from data at the loading time. self.speed_factor = 1.0 # In KM/H FOR GTA it should be maximun 30.0 # The division is made by three diferent data kinds # in every mini-batch there will be equal number of samples with labels from each group # e.g. for [[0,1],[2]] there will be 50% samples with labels 0 and 1, and 50% samples with label 2 self.labels_per_division = [[2],[2],[2]] self.dataset_names = ['targets'] self.queue_capacity = 20*self.batch_size # TODO NOT IMPLEMENTED Felipe: True/False switches to turn data balancing on or off self.balances_val = True self.balances_train = True self.augment_and_saturate_factor = True
def augmentation_generator(data, batch_size): ''' Performs data augmentation on training set. Practice data gen. ''' n = len(data) batches_to_make = n // batch_size # Containers for batch data and labels. batch_data = np.zeros((batch_size, 224, 224, 3), dtype=np.float32) batch_labels = np.zeros((batch_size, 2), dtype=np.float32) # Grab indices. indices = np.arange(n) # Define transformations. transformations = iaa.OneOf([ iaa.Fliplr(0.5), # Flip. iaa.ContrastNormalization((0.75, 1.5)), # Play around with contrast. iaa.GaussianBlur(sigma=(0, 0.5)), # Randomly blur. iaa.Multiply((0.8, 1.2)) # Change brightness. ]) # Start a counter. batches_made = 0 while True: # Make batch. np.random.shuffle(indices) count = 0 next_batch = indices[(batches_made * batch_size):(batches_made + 1) * batch_size] for foo, idx in enumerate(next_batch): img_name = data.iloc[idx]['image_path'] img_label = data.iloc[idx]['label'] # Encode label. encoded_label = to_categorical(img_label, num_classes=2) # Read image and resize. img = cv2.imread(str(img_name)) img = cv2.resize(img, (224, 224)) # Check for greyscale. if img.shape[2] == 1: img = np.dstack([img, img, img]) # Change from BGR to RGB and normalize. orig_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) orig_img = orig_img.astype(np.float32) / 255 batch_data[count] = orig_img batch_labels[count] = encoded_label # Data augment for under-sampled class. if img_label == 0 and count < batch_size - 2: # Image 1. aug_img1 = transformations.augment(images=img) aug_img1 = cv2.cvtColor(aug_img1, cv2.COLOR_BGR2RGB) aug_img1 = aug_img1.astype(np.float32) / 255 batch_data[count + 1] = aug_img1 batch_labels[count + 1] = encoded_label # Image 2. aug_img2 = transformations.augment(images=img) aug_img2 = cv2.cvtColor(aug_img2, cv2.COLOR_BGR2RGB) aug_img2 = aug_img2.astype(np.float32) / 255 batch_data[count + 2] = aug_img2 batch_labels[count + 2] = encoded_label else: count += 1 # Can't add anymore. if count == batch_size - 1: break batches_made += 1 yield batch_data, batch_labels # Reset for the next go. if batches_made >= batches_to_make: batches_made = 0
def __init__(self, images, config, shuffle=True, jitter=True, norm=None): self.generator = None self.images = images self.config = config self.shuffle = shuffle self.jitter = jitter self.norm = norm self.anchors = [ BoundBox(0, 0, config['ANCHORS'][2 * i], config['ANCHORS'][2 * i + 1]) for i in range(int(len(config['ANCHORS']) // 2)) ] ### augmentors by https://github.com/aleju/imgaug sometimes = lambda aug: iaa.Sometimes(0.5, aug) # Define our sequence of augmentation steps that will be applied to every image # All augmenters with per_channel=0.5 will sample one value _per image_ # in 50% of all cases. In all other cases they will sample new values # _per channel_. self.aug_pipe = iaa.Sequential( [ # apply the following augmenters to most images #iaa.Fliplr(0.5), # horizontally flip 50% of all images #iaa.Flipud(0.2), # vertically flip 20% of all images #sometimes(iaa.Crop(percent=(0, 0.1))), # crop images by 0-10% of their height/width sometimes( iaa.Affine( #scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis #translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis) #rotate=(-5, 5), # rotate by -45 to +45 degrees #shear=(-5, 5), # shear by -16 to +16 degrees #order=[0, 1], # use nearest neighbour or bilinear interpolation (fast) #cval=(0, 255), # if mode is constant, use a cval between 0 and 255 #mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples) )), # execute 0 to 5 of the following (less important) augmenters per image # don't execute all of them, as that would often be way too strong iaa.SomeOf( (0, 5), [ #sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation iaa.OneOf([ iaa.GaussianBlur( (0, 3.0) ), # blur images with a sigma between 0 and 3.0 iaa.AverageBlur( k=(2, 7) ), # blur image using local means with kernel sizes between 2 and 7 iaa.MedianBlur( k=(3, 11) ), # blur image using local medians with kernel sizes between 2 and 7 ]), iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images #iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images # search either for all edges or for directed edges #sometimes(iaa.OneOf([ # iaa.EdgeDetect(alpha=(0, 0.7)), # iaa.DirectedEdgeDetect(alpha=(0, 0.7), direction=(0.0, 1.0)), #])), iaa.AdditiveGaussianNoise( loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5), # add gaussian noise to images iaa.OneOf([ iaa.Dropout( (0.01, 0.1), per_channel=0.5 ), # randomly remove up to 10% of the pixels #iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2), ]), #iaa.Invert(0.05, per_channel=True), # invert color channels iaa.Add( (-10, 10), per_channel=0.5 ), # change brightness of images (by -10 to 10 of original value) iaa.Multiply( (0.5, 1.5), per_channel=0.5 ), # change brightness of images (50-150% of original value) iaa.ContrastNormalization( (0.5, 2.0), per_channel=0.5), # improve or worsen the contrast #iaa.Grayscale(alpha=(0.0, 1.0)), #sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths) #sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) # sometimes move parts of the image around ], random_order=True) ], random_order=True) if shuffle: np.random.shuffle(self.images)
def augmentor(self, images): 'Apply data augmentation' sometimes = lambda aug: iaa.Sometimes(0.5, aug) seq = iaa.Sequential( [ # apply the following augmenters to most images iaa.Fliplr(0.5), # horizontally flip 50% of all images iaa.Flipud(0.2), # vertically flip 20% of all images sometimes( iaa.Affine( scale={ "x": (0.9, 1.1), "y": (0.9, 1.1) }, # scale images to 80-120% of their size, individually per axis translate_percent={ "x": (-0.1, 0.1), "y": (-0.1, 0.1) }, # translate by -20 to +20 percent (per axis) rotate=(-10, 10), # rotate by -45 to +45 degrees shear=(-5, 5), # shear by -16 to +16 degrees order=[0, 1], # use nearest neighbour or bilinear interpolation (fast) cval=( 0, 255 ), # if mode is constant, use a cval between 0 and 255 mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples) )), # execute 0 to 5 of the following (less important) augmenters per image # don't execute all of them, as that would often be way too strong iaa.SomeOf( (0, 5), [ sometimes( iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation iaa.OneOf([ iaa.GaussianBlur((0, 1.0)), # blur images with a sigma between 0 and 3.0 iaa.AverageBlur(k=(3, 5)), # blur image using local means with kernel sizes between 2 and 7 iaa.MedianBlur(k=(3, 5)), # blur image using local medians with kernel sizes between 2 and 7 ]), iaa.Sharpen(alpha=(0, 1.0), lightness=(0.9, 1.1)), # sharpen images iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images # search either for all edges or for directed edges, # blend the result with the original image using a blobby mask iaa.SimplexNoiseAlpha( iaa.OneOf([ iaa.EdgeDetect(alpha=(0.5, 1.0)), iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)), ])), iaa.AdditiveGaussianNoise( loc=0, scale=(0.0, 0.01 * 255), per_channel=0.5), # add gaussian noise to images iaa.OneOf([ iaa.Dropout((0.01, 0.05), per_channel=0.5), # randomly remove up to 10% of the pixels iaa.CoarseDropout((0.01, 0.03), size_percent=(0.01, 0.02), per_channel=0.2), ]), iaa.Invert(0.01, per_channel=True), # invert color channels iaa.Add((-2, 2), per_channel=0.5), # change brightness of images (by -10 to 10 of original value) iaa.AddToHueAndSaturation((-1, 1)), # change hue and saturation # either change the brightness of the whole image (sometimes # per channel) or change the brightness of subareas iaa.OneOf([ iaa.Multiply((0.9, 1.1), per_channel=0.5), iaa.FrequencyNoiseAlpha( exponent=(-1, 0), first=iaa.Multiply( (0.9, 1.1), per_channel=True), second=iaa.ContrastNormalization((0.9, 1.1))) ]), sometimes( iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths) sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))), # sometimes move parts of the image around sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1))) ], random_order=True) ], random_order=True) return seq.augment_images(images)
def __call__(self, sample): image, pose = sample['image'], sample['pose'].reshape([-1, 2]) sometimes = lambda aug: iaa.Sometimes(0.3, aug) seq = iaa.Sequential( [ # Apply the following augmenters to most images. sometimes( iaa.CropAndPad(percent=(-0.25, 0.25), pad_mode=["edge"], keep_size=False)), sometimes( iaa.Affine(scale={ "x": (0.75, 1.25), "y": (0.75, 1.25) }, translate_percent={ "x": (-0.25, 0.25), "y": (-0.25, 0.25) }, rotate=(-45, 45), shear=(-5, 5), order=[0, 1], cval=(0, 255), mode=ia.ALL)), iaa.SomeOf( (0, 3), [ iaa.OneOf([ iaa.GaussianBlur((0, 3.0)), # iaa.AverageBlur(k=(2, 7)), iaa.MedianBlur(k=(3, 11)), iaa.MotionBlur(k=5, angle=[-45, 45]) ]), iaa.OneOf([ iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5), iaa.AdditivePoissonNoise(lam=(0, 8), per_channel=True), ]), iaa.OneOf([ iaa.Add((-10, 10), per_channel=0.5), iaa.Multiply((0.2, 1.2), per_channel=0.5), iaa.ContrastNormalization( (0.5, 2.0), per_channel=0.5), ]), ], # do all of the above augmentations in random order random_order=True) ], # do all of the above augmentations in random order random_order=True) # augmentation choices seq_det = seq.to_deterministic() image_aug = seq_det.augment_images([image])[0] keypoints_aug = seq_det.augment_keypoints( [self.pose2keypoints(image, pose)])[0] return {'image': image_aug, 'pose': self.keypoints2pose(keypoints_aug)}
def train(model): """Train the model.""" # Training dataset. dataset_train = BalloonDataset() dataset_train.load_balloon(args.dataset, "train") dataset_train.prepare() # Validation dataset dataset_val = BalloonDataset() dataset_val.load_balloon(args.dataset, "val") dataset_val.prepare() # Image augmentation # http://imgaug.readthedocs.io/en/latest/source/augmenters.html augmentation = iaa.SomeOf( (1, 5), [ iaa.Fliplr(0.5), iaa.Flipud(0.5), iaa.Affine(rotate=90), # new to force more horizontal lines iaa.Affine( rotate=(-90, 90), mode="edge" ), # mode= "edge" ads straight lines in created empty space #for this do one of iaa.OneOf([ iaa.CropAndPad(percent=(-0.3, 0.05), sample_independently=False, pad_mode="edge"), iaa.Crop(percent=(0, 0.05)) ]), iaa.GaussianBlur(sigma=(0.0, 1.0)), iaa.Multiply((0.5, 1.2)), #changeing brightness iaa.Grayscale(alpha=(0.0, 0.9)) ]) # *** This training schedule is an example. Update to your needs *** # Since we're using a very small dataset, and starting from # COCO trained weights, we don't need to train too long. Also, # no need to train all layers, just the heads should do it. print("Training network heads") model.train(dataset_train, dataset_val, learning_rate=config.LEARNING_RATE, epochs=15, augmentation=augmentation, layers='heads') # 'heads' or 'all' print("Training 4+") model.train(dataset_train, dataset_val, learning_rate=config.LEARNING_RATE, epochs=60, augmentation=augmentation, layers='4+') # 'heads' or 'all' print("Train all") model.train(dataset_train, dataset_val, learning_rate=config.LEARNING_RATE, epochs=200, augmentation=augmentation, layers='all') # 'heads' or 'all' print("Train all lower learning rate") model.train(dataset_train, dataset_val, learning_rate=config.LEARNING_RATE / 10, epochs=400, augmentation=augmentation, layers='all') # 'heads' or 'all'
# sometimes(iaa.OneOf([ # iaa.EdgeDetect(alpha=(0, 0.7)), # iaa.DirectedEdgeDetect(alpha=(0, 0.7), direction=(0.0, 1.0)), # ])), iaa.AdditiveGaussianNoise( loc=0, scale=(0.0, 0.05 * 255), per_channel=0.5), # add gaussian noise to images iaa.OneOf([ iaa.Dropout((0.01, 0.1), per_channel=0.5 ), # randomly remove up to 10% of the pixels # iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2), ]), # iaa.Invert(0.05, per_channel=True), # invert color channels iaa.Add((-10, 10), per_channel=0.5), # change brightness of images (by -10 to 10 of original value) iaa.Multiply((0.5, 1.5), per_channel=0.5), # change brightness of images (50-150% of original value) iaa.ContrastNormalization( (0.5, 2.0), per_channel=0.5), # improve or worsen the contrast # iaa.Grayscale(alpha=(0.0, 1.0)), # sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths) # sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) # sometimes move parts of the image around ], random_order=True) ], random_order=True) def aug_image(image, annos, jitter): orig_h, orig_w = image.shape[:2]