def val_pipeline(cfg: ValLoaderConfig):
jpeg, label = fn.readers.file(
file_root=ROOT_DATA_DIR + "/val/",
shard_id=env_rank(),
num_shards=env_world_size(),
name="Reader",
)
image = fn.decoders.image(jpeg, device="mixed", output_type=types.RGB)
crop_size = cfg.image_size if cfg.full_crop else math.ceil(
(cfg.image_size * 1.14 + 8) // 16 * 16)
image = fn.resize(image,
device="gpu",
interp_type=types.INTERP_TRIANGULAR,
resize_shorter=crop_size)
image = fn.crop_mirror_normalize(
image,
device="gpu",
crop=(cfg.image_size, cfg.image_size),
mean=DATA_MEAN,
std=DATA_STD,
dtype=types.FLOAT,
output_layout=types.NCHW,
)
label = fn.one_hot(label, num_classes=cfg.num_classes).gpu()
return image, label
def test_one_hot_cpu():
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None)
test_data_shape = [200]
def get_data():
out = [
np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8)
for _ in range(batch_size)
]
return out
data = fn.external_source(source=get_data)
processed = fn.one_hot(data, num_classes=256)
pipe.set_outputs(processed)
pipe.build()
for _ in range(3):
pipe.run()
def train_pipeline(cfg: TrainLoaderConfig):
jpeg, label = fn.readers.file(
file_root=ROOT_DATA_DIR + "/train/",
random_shuffle=True,
shard_id=env_rank(),
num_shards=env_world_size(),
name="Reader",
)
image = fn.decoders.image_random_crop(
jpeg,
device="mixed",
random_aspect_ratio=[0.75, 1.25],
random_area=[cfg.min_area, 1.0],
num_attempts=100,
output_type=types.RGB,
)
image_tr = fn.resize(image,
device="gpu",
size=cfg.image_size,
interp_type=types.INTERP_TRIANGULAR)
if cfg.random_interpolation:
image_cub = fn.resize(image,
device="gpu",
size=cfg.image_size,
interp_type=types.INTERP_CUBIC)
image = mix(fn.random.coin_flip(probability=0.5), image_cub, image_tr)
else:
image = image_tr
if cfg.blur_prob > 0:
blur_image = fn.gaussian_blur(
image,
device="gpu",
window_size=11,
sigma=fn.random.uniform(range=[0.5, 1.1]))
image = mix(
fn.random.coin_flip(probability=cfg.blur_prob, dtype=types.BOOL),
blur_image, image)
if cfg.color_twist_prob > 0:
image_ct = fn.color_twist(
image,
device="gpu",
contrast=fn.random.uniform(range=[0.7, 1.3]),
brightness=fn.random.uniform(range=[0.7, 1.3]),
hue=fn.random.uniform(range=[-20, 20]), # in degrees
saturation=fn.random.uniform(range=[0.7, 1.3]),
)
image = mix(
fn.random.coin_flip(probability=cfg.color_twist_prob,
dtype=types.BOOL), image_ct, image)
if cfg.gray_prob > 0:
grayscale_coin = fn.cast(
fn.random.coin_flip(probability=cfg.gray_prob), dtype=types.FLOAT)
image = fn.hsv(image, device="gpu", saturation=grayscale_coin)
if cfg.re_prob: # random erasing
image_re = fn.erase(
image,
device="gpu",
anchor=fn.random.uniform(range=(0.0, 1), shape=cfg.re_count * 2),
shape=fn.random.uniform(range=(0.05, 0.25),
shape=cfg.re_count * 2),
axis_names="HW",
fill_value=DATA_MEAN,
normalized_anchor=True,
normalized_shape=True,
)
image = mix(
fn.random.coin_flip(probability=cfg.re_prob, dtype=types.BOOL),
image_re, image)
image = fn.crop_mirror_normalize(
image,
device="gpu",
crop=(cfg.image_size, cfg.image_size),
mirror=fn.random.coin_flip(probability=0.5),
mean=DATA_MEAN,
std=DATA_STD,
dtype=types.FLOAT,
output_layout=types.NCHW,
)
label = fn.one_hot(label, num_classes=cfg.num_classes).gpu()
return image, label