def val_pipeline(cfg: ValLoaderConfig): jpeg, label = fn.readers.file( file_root=ROOT_DATA_DIR + "/val/", shard_id=env_rank(), num_shards=env_world_size(), name="Reader", ) image = fn.decoders.image(jpeg, device="mixed", output_type=types.RGB) crop_size = cfg.image_size if cfg.full_crop else math.ceil( (cfg.image_size * 1.14 + 8) // 16 * 16) image = fn.resize(image, device="gpu", interp_type=types.INTERP_TRIANGULAR, resize_shorter=crop_size) image = fn.crop_mirror_normalize( image, device="gpu", crop=(cfg.image_size, cfg.image_size), mean=DATA_MEAN, std=DATA_STD, dtype=types.FLOAT, output_layout=types.NCHW, ) label = fn.one_hot(label, num_classes=cfg.num_classes).gpu() return image, label
def test_one_hot_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) test_data_shape = [200] def get_data(): out = [ np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8) for _ in range(batch_size) ] return out data = fn.external_source(source=get_data) processed = fn.one_hot(data, num_classes=256) pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def train_pipeline(cfg: TrainLoaderConfig): jpeg, label = fn.readers.file( file_root=ROOT_DATA_DIR + "/train/", random_shuffle=True, shard_id=env_rank(), num_shards=env_world_size(), name="Reader", ) image = fn.decoders.image_random_crop( jpeg, device="mixed", random_aspect_ratio=[0.75, 1.25], random_area=[cfg.min_area, 1.0], num_attempts=100, output_type=types.RGB, ) image_tr = fn.resize(image, device="gpu", size=cfg.image_size, interp_type=types.INTERP_TRIANGULAR) if cfg.random_interpolation: image_cub = fn.resize(image, device="gpu", size=cfg.image_size, interp_type=types.INTERP_CUBIC) image = mix(fn.random.coin_flip(probability=0.5), image_cub, image_tr) else: image = image_tr if cfg.blur_prob > 0: blur_image = fn.gaussian_blur( image, device="gpu", window_size=11, sigma=fn.random.uniform(range=[0.5, 1.1])) image = mix( fn.random.coin_flip(probability=cfg.blur_prob, dtype=types.BOOL), blur_image, image) if cfg.color_twist_prob > 0: image_ct = fn.color_twist( image, device="gpu", contrast=fn.random.uniform(range=[0.7, 1.3]), brightness=fn.random.uniform(range=[0.7, 1.3]), hue=fn.random.uniform(range=[-20, 20]), # in degrees saturation=fn.random.uniform(range=[0.7, 1.3]), ) image = mix( fn.random.coin_flip(probability=cfg.color_twist_prob, dtype=types.BOOL), image_ct, image) if cfg.gray_prob > 0: grayscale_coin = fn.cast( fn.random.coin_flip(probability=cfg.gray_prob), dtype=types.FLOAT) image = fn.hsv(image, device="gpu", saturation=grayscale_coin) if cfg.re_prob: # random erasing image_re = fn.erase( image, device="gpu", anchor=fn.random.uniform(range=(0.0, 1), shape=cfg.re_count * 2), shape=fn.random.uniform(range=(0.05, 0.25), shape=cfg.re_count * 2), axis_names="HW", fill_value=DATA_MEAN, normalized_anchor=True, normalized_shape=True, ) image = mix( fn.random.coin_flip(probability=cfg.re_prob, dtype=types.BOOL), image_re, image) image = fn.crop_mirror_normalize( image, device="gpu", crop=(cfg.image_size, cfg.image_size), mirror=fn.random.coin_flip(probability=0.5), mean=DATA_MEAN, std=DATA_STD, dtype=types.FLOAT, output_layout=types.NCHW, ) label = fn.one_hot(label, num_classes=cfg.num_classes).gpu() return image, label