def make_voc_dataset_sub(n_train,
n_val,
batch_size,
eval_batch_size,
transform,
data_dir=None,
prefetch=True):
steps_per_epoch, val_steps = n_train // batch_size, n_val // eval_batch_size
ds_train = tfds.load("voc/2012",
split=f"train[:{n_train}]",
data_dir=data_dir,
shuffle_files=True,
read_config=tfds.ReadConfig(try_autocache=False,
skip_prefetch=True))
ds_val = tfds.load("voc/2012",
split=f"train[:{n_val}]",
data_dir=data_dir,
shuffle_files=False,
read_config=tfds.ReadConfig(try_autocache=False,
skip_prefetch=True))
ds_train = prepare(ds_train,
batch_size,
transform(training=True),
training=True,
repeat=False,
prefetch=prefetch)
ds_val = prepare(ds_val,
eval_batch_size,
transform(training=False),
training=False,
repeat=False,
drop_remainder=False,
prefetch=prefetch)
return ds_train, ds_val, steps_per_epoch, val_steps
def make_dataset(batch_size,
eval_batch_size,
transform,
data_dir=None,
drop_remainder=None):
n_train, n_val = NUM_EXAMPLES['train'], NUM_EXAMPLES['validation']
steps_per_epoch = n_train // batch_size
if drop_remainder:
val_steps = n_val // eval_batch_size
else:
val_steps = math.ceil(n_val / eval_batch_size)
read_config = tfds.ReadConfig(try_autocache=False, skip_prefetch=True)
ds_train = tfds.load("coco/2017",
split=f"train",
data_dir=data_dir,
shuffle_files=True,
read_config=read_config)
ds_val = tfds.load("coco/2017",
split=f"validation",
data_dir=data_dir,
shuffle_files=False,
read_config=read_config)
ds_train = prepare(ds_train,
batch_size,
transform(training=True),
training=True,
repeat=False)
ds_val = prepare(ds_val,
eval_batch_size,
transform(training=False),
training=False,
repeat=False,
drop_remainder=drop_remainder)
return ds_train, ds_val, steps_per_epoch, val_steps
def make_cifar_dataset(load_fn,
batch_size,
eval_batch_size,
transform,
drop_remainder=None,
sub_ratio=None):
if drop_remainder is None:
drop_remainder = False
(x_train, y_train), (_x_test, _y_test) = load_fn()
if sub_ratio:
x_train, y_train = subsample(x_train, y_train, ratio=sub_ratio)
n_val = len(x_train) // 2
x_val, y_val = x_train[n_val:], y_train[n_val:]
x_train, y_train = x_train[:n_val], y_train[:n_val]
n_train = len(x_train)
steps_per_epoch = n_train // batch_size
eval_steps = math.ceil(n_val / eval_batch_size)
ds_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
ds_val = tf.data.Dataset.from_tensor_slices((x_val, y_val))
ds_train = prepare(ds_train,
batch_size,
transform(training=True),
training=True,
buffer_size=n_train,
prefetch=False)
ds_search = prepare(ds_val,
batch_size,
transform(training=False),
training=True,
buffer_size=n_val,
prefetch=False)
ds_train = tf.data.Dataset.zip((ds_train, ds_search))
ds_train = ds_train.prefetch(tf.data.experimental.AUTOTUNE)
ds_val = prepare(ds_val,
eval_batch_size,
transform(training=False),
training=False,
buffer_size=n_val,
drop_remainder=drop_remainder)
return ds_train, ds_val, steps_per_epoch, eval_steps
def make_imagenet_dataset_split(batch_size,
transform,
filenames,
split,
training=None,
cache_decoded_image=False,
cache_parsed=False,
**kwargs):
assert split in NUM_IMAGES.keys()
if training is None:
training = split == 'train'
drop_remainder = training
dataset = tf.data.Dataset.from_tensor_slices(filenames)
if training:
dataset = dataset.shuffle(buffer_size=len(filenames))
dataset = dataset.interleave(
tf.data.TFRecordDataset,
cycle_length=16,
num_parallel_calls=tf.data.experimental.AUTOTUNE,
deterministic=False)
if cache_parsed:
dataset = dataset.map(parse_example_proto,
num_parallel_calls=tf.data.experimental.AUTOTUNE)
transform = functools.partial(transform, training=training)
else:
transform = parse_and_transform(transform, training)
ds = prepare(dataset,
batch_size,
transform,
training=training,
buffer_size=_SHUFFLE_BUFFER,
cache=True,
prefetch=True,
repeat=training,
**kwargs)
n = NUM_IMAGES[split]
chunksize = math.ceil(n / NUM_FILES[split])
n = min(len(filenames) * chunksize, n)
if 'repeat' in kwargs and type(kwargs['repeat']) == int:
n *= kwargs['repeat']
if 'aug_repeats' in kwargs and type(kwargs['aug_repeats']) == int:
n *= kwargs['aug_repeats']
if drop_remainder:
steps = n // batch_size
else:
steps = math.ceil(n / batch_size)
return ds, steps
def make_dataset_sub(n_train,
n_val,
batch_size,
eval_batch_size,
transform,
data_dir=None,
drop_remainder=None):
steps_per_epoch = n_train // batch_size
if drop_remainder:
val_steps = n_val // eval_batch_size
else:
val_steps = math.ceil(n_val / eval_batch_size)
read_config = tfds.ReadConfig(try_autocache=False, skip_prefetch=True)
train_split = f"validation[:{n_train}]" if n_train != 5000 else 'validation'
val_split = f"validation[:{n_val}]" if n_val != 5000 else 'validation'
ds_train = tfds.load("coco/2017",
split=train_split,
data_dir=data_dir,
shuffle_files=False,
read_config=read_config)
ds_val = tfds.load("coco/2017",
split=val_split,
data_dir=data_dir,
shuffle_files=False,
read_config=read_config)
ds_train = prepare(ds_train,
batch_size,
transform(training=True),
training=True,
repeat=False)
ds_val = prepare(ds_val,
eval_batch_size,
transform(training=False),
training=False,
repeat=False,
drop_remainder=drop_remainder)
return ds_train, ds_val, steps_per_epoch, val_steps
shuffle=True,
random_state=42,
stratify=y)
batch_size = 32
eval_batch_size = batch_size * 2
n_train, n_test = len(X_train), len(X_test)
steps_per_epoch, test_steps = n_train // batch_size, math.ceil(n_test /
eval_batch_size)
ds = tf.data.Dataset.from_tensor_slices((X_train, y_train))
ds_test = tf.data.Dataset.from_tensor_slices((X_test, y_test))
ds_train = prepare(ds,
batch_size,
transform=transform(training=True),
training=True,
buffer_size=n_train)
ds_test = prepare(ds_test,
eval_batch_size,
transform=transform(training=False),
training=False)
setup_runtime()
ds_train, ds_test = distribute_datasets(ds_train, ds_test)
class MixOp(tf.keras.layers.Layer):
def __init__(self):
super().__init__()
self.ops = [
label = tf.squeeze(label, -1)
image = tf.cast(image, tf.bfloat16)
return image, label
train_files = [
"/Users/hrvvi/Downloads/datasets/tfrecords/Cityscapes_sub/train-%05d-of-00004.tfrecord" % i for i in range(4)]
val_files = [
"/Users/hrvvi/Downloads/datasets/tfrecords/Cityscapes_sub/val-%05d-of-00002.tfrecord" % i for i in range(2)]
mul = 8
n_train, n_val = 2975, 500
batch_size, eval_batch_size = 2 * mul, 2 * mul
steps_per_epoch, val_steps = n_train // batch_size, n_val // eval_batch_size
ds_train = prepare(tf.data.TFRecordDataset(train_files), batch_size, preprocess(training=True),
training=True, repeat=False)
ds_val = prepare(tf.data.TFRecordDataset(val_files), eval_batch_size, preprocess(training=False),
training=False, repeat=False)
set_defaults({
'bn': {
'sync': True,
}
})
backbone = resnet50(output_stride=16, multi_grad=(1, 2, 4))
model = DeepLabV3P(backbone, aspp_ratios=(1, 6, 12, 18), aspp_channels=256, num_classes=19)
model.build((None, HEIGHT, WIDTH, 3))
import tensorflow as tf
import tensorflow_datasets as tfds
from hanser.transform import pad, to_tensor, normalize
@curry
def transform(image, label, training):
image = pad(image, 2)
image, label = to_tensor(image, label)
image = normalize(image, [0.1307], [0.3081])
# label = tf.one_hot(label, 10)
return image, label
ds = tfds.load("mnist",
split='train',
try_gcs=False,
download=True,
shuffle_files=True,
as_supervised=True)
ds_train = prepare(ds,
4,
transform(training=True),
training=False,
buffer_size=8)
it = iter(ds_train)
next(it)[1]