def make_voc_dataset_sub(n_train, n_val, batch_size, eval_batch_size, transform, data_dir=None, prefetch=True): steps_per_epoch, val_steps = n_train // batch_size, n_val // eval_batch_size ds_train = tfds.load("voc/2012", split=f"train[:{n_train}]", data_dir=data_dir, shuffle_files=True, read_config=tfds.ReadConfig(try_autocache=False, skip_prefetch=True)) ds_val = tfds.load("voc/2012", split=f"train[:{n_val}]", data_dir=data_dir, shuffle_files=False, read_config=tfds.ReadConfig(try_autocache=False, skip_prefetch=True)) ds_train = prepare(ds_train, batch_size, transform(training=True), training=True, repeat=False, prefetch=prefetch) ds_val = prepare(ds_val, eval_batch_size, transform(training=False), training=False, repeat=False, drop_remainder=False, prefetch=prefetch) return ds_train, ds_val, steps_per_epoch, val_steps
def make_dataset(batch_size, eval_batch_size, transform, data_dir=None, drop_remainder=None): n_train, n_val = NUM_EXAMPLES['train'], NUM_EXAMPLES['validation'] steps_per_epoch = n_train // batch_size if drop_remainder: val_steps = n_val // eval_batch_size else: val_steps = math.ceil(n_val / eval_batch_size) read_config = tfds.ReadConfig(try_autocache=False, skip_prefetch=True) ds_train = tfds.load("coco/2017", split=f"train", data_dir=data_dir, shuffle_files=True, read_config=read_config) ds_val = tfds.load("coco/2017", split=f"validation", data_dir=data_dir, shuffle_files=False, read_config=read_config) ds_train = prepare(ds_train, batch_size, transform(training=True), training=True, repeat=False) ds_val = prepare(ds_val, eval_batch_size, transform(training=False), training=False, repeat=False, drop_remainder=drop_remainder) return ds_train, ds_val, steps_per_epoch, val_steps
def make_cifar_dataset(load_fn, batch_size, eval_batch_size, transform, drop_remainder=None, sub_ratio=None): if drop_remainder is None: drop_remainder = False (x_train, y_train), (_x_test, _y_test) = load_fn() if sub_ratio: x_train, y_train = subsample(x_train, y_train, ratio=sub_ratio) n_val = len(x_train) // 2 x_val, y_val = x_train[n_val:], y_train[n_val:] x_train, y_train = x_train[:n_val], y_train[:n_val] n_train = len(x_train) steps_per_epoch = n_train // batch_size eval_steps = math.ceil(n_val / eval_batch_size) ds_train = tf.data.Dataset.from_tensor_slices((x_train, y_train)) ds_val = tf.data.Dataset.from_tensor_slices((x_val, y_val)) ds_train = prepare(ds_train, batch_size, transform(training=True), training=True, buffer_size=n_train, prefetch=False) ds_search = prepare(ds_val, batch_size, transform(training=False), training=True, buffer_size=n_val, prefetch=False) ds_train = tf.data.Dataset.zip((ds_train, ds_search)) ds_train = ds_train.prefetch(tf.data.experimental.AUTOTUNE) ds_val = prepare(ds_val, eval_batch_size, transform(training=False), training=False, buffer_size=n_val, drop_remainder=drop_remainder) return ds_train, ds_val, steps_per_epoch, eval_steps
def make_imagenet_dataset_split(batch_size, transform, filenames, split, training=None, cache_decoded_image=False, cache_parsed=False, **kwargs): assert split in NUM_IMAGES.keys() if training is None: training = split == 'train' drop_remainder = training dataset = tf.data.Dataset.from_tensor_slices(filenames) if training: dataset = dataset.shuffle(buffer_size=len(filenames)) dataset = dataset.interleave( tf.data.TFRecordDataset, cycle_length=16, num_parallel_calls=tf.data.experimental.AUTOTUNE, deterministic=False) if cache_parsed: dataset = dataset.map(parse_example_proto, num_parallel_calls=tf.data.experimental.AUTOTUNE) transform = functools.partial(transform, training=training) else: transform = parse_and_transform(transform, training) ds = prepare(dataset, batch_size, transform, training=training, buffer_size=_SHUFFLE_BUFFER, cache=True, prefetch=True, repeat=training, **kwargs) n = NUM_IMAGES[split] chunksize = math.ceil(n / NUM_FILES[split]) n = min(len(filenames) * chunksize, n) if 'repeat' in kwargs and type(kwargs['repeat']) == int: n *= kwargs['repeat'] if 'aug_repeats' in kwargs and type(kwargs['aug_repeats']) == int: n *= kwargs['aug_repeats'] if drop_remainder: steps = n // batch_size else: steps = math.ceil(n / batch_size) return ds, steps
def make_dataset_sub(n_train, n_val, batch_size, eval_batch_size, transform, data_dir=None, drop_remainder=None): steps_per_epoch = n_train // batch_size if drop_remainder: val_steps = n_val // eval_batch_size else: val_steps = math.ceil(n_val / eval_batch_size) read_config = tfds.ReadConfig(try_autocache=False, skip_prefetch=True) train_split = f"validation[:{n_train}]" if n_train != 5000 else 'validation' val_split = f"validation[:{n_val}]" if n_val != 5000 else 'validation' ds_train = tfds.load("coco/2017", split=train_split, data_dir=data_dir, shuffle_files=False, read_config=read_config) ds_val = tfds.load("coco/2017", split=val_split, data_dir=data_dir, shuffle_files=False, read_config=read_config) ds_train = prepare(ds_train, batch_size, transform(training=True), training=True, repeat=False) ds_val = prepare(ds_val, eval_batch_size, transform(training=False), training=False, repeat=False, drop_remainder=drop_remainder) return ds_train, ds_val, steps_per_epoch, val_steps
shuffle=True, random_state=42, stratify=y) batch_size = 32 eval_batch_size = batch_size * 2 n_train, n_test = len(X_train), len(X_test) steps_per_epoch, test_steps = n_train // batch_size, math.ceil(n_test / eval_batch_size) ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)) ds_test = tf.data.Dataset.from_tensor_slices((X_test, y_test)) ds_train = prepare(ds, batch_size, transform=transform(training=True), training=True, buffer_size=n_train) ds_test = prepare(ds_test, eval_batch_size, transform=transform(training=False), training=False) setup_runtime() ds_train, ds_test = distribute_datasets(ds_train, ds_test) class MixOp(tf.keras.layers.Layer): def __init__(self): super().__init__() self.ops = [
label = tf.squeeze(label, -1) image = tf.cast(image, tf.bfloat16) return image, label train_files = [ "/Users/hrvvi/Downloads/datasets/tfrecords/Cityscapes_sub/train-%05d-of-00004.tfrecord" % i for i in range(4)] val_files = [ "/Users/hrvvi/Downloads/datasets/tfrecords/Cityscapes_sub/val-%05d-of-00002.tfrecord" % i for i in range(2)] mul = 8 n_train, n_val = 2975, 500 batch_size, eval_batch_size = 2 * mul, 2 * mul steps_per_epoch, val_steps = n_train // batch_size, n_val // eval_batch_size ds_train = prepare(tf.data.TFRecordDataset(train_files), batch_size, preprocess(training=True), training=True, repeat=False) ds_val = prepare(tf.data.TFRecordDataset(val_files), eval_batch_size, preprocess(training=False), training=False, repeat=False) set_defaults({ 'bn': { 'sync': True, } }) backbone = resnet50(output_stride=16, multi_grad=(1, 2, 4)) model = DeepLabV3P(backbone, aspp_ratios=(1, 6, 12, 18), aspp_channels=256, num_classes=19) model.build((None, HEIGHT, WIDTH, 3))
import tensorflow as tf import tensorflow_datasets as tfds from hanser.transform import pad, to_tensor, normalize @curry def transform(image, label, training): image = pad(image, 2) image, label = to_tensor(image, label) image = normalize(image, [0.1307], [0.3081]) # label = tf.one_hot(label, 10) return image, label ds = tfds.load("mnist", split='train', try_gcs=False, download=True, shuffle_files=True, as_supervised=True) ds_train = prepare(ds, 4, transform(training=True), training=False, buffer_size=8) it = iter(ds_train) next(it)[1]