def process_input(epochs, batch_size, channel, channel_name, data_config):
mode = data_config[channel_name]['TrainingInputMode']
filenames = _get_filenames(channel_name, channel)
# Repeat infinitely.
logging.info("Running {} in {} mode".format(channel_name, mode))
if mode == 'Pipe':
from sagemaker_tensorflow import PipeModeDataset
dataset = PipeModeDataset(channel=channel_name,
record_format='TFRecord')
else:
dataset = tf.data.TFRecordDataset(filenames)
dataset = dataset.repeat(epochs)
dataset = dataset.prefetch(10)
dataset = dataset.map(_dataset_parser, num_parallel_calls=10)
if channel_name == 'train':
# Ensure that the capacity is sufficiently large to provide good random
# shuffling.
#buffer_size = int(NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN * 0.4) + 3 * batch_size
buffer_size = 10000
dataset = dataset.shuffle(buffer_size=buffer_size)
if channel_name == 'validation':
buffer_size = 10000
dataset = dataset.shuffle(buffer_size=buffer_size)
dataset = dataset.batch(batch_size, drop_remainder=True)
return dataset
def _input(epochs, batch_size, channel, channel_name):
"""Uses the tf.data input pipeline for CIFAR-10 dataset."""
mode = args.data_config[channel_name]['TrainingInputMode']
logging.info("Running {} in {} mode".format(channel_name, mode))
if mode == 'Pipe':
from sagemaker_tensorflow import PipeModeDataset
dataset = PipeModeDataset(channel=channel_name,
record_format='TFRecord')
else:
filenames = _get_filenames(channel_name, channel)
dataset = tf.data.TFRecordDataset(filenames)
# Repeat infinitely.
dataset = dataset.repeat()
dataset = dataset.prefetch(10)
# Parse records.
dataset = dataset.map(_dataset_parser, num_parallel_calls=10)
# Potentially shuffle records.
if channel_name == 'train':
# Ensure that the capacity is sufficiently large to provide good random shuffling.
buffer_size = int(
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN * 0.4) + 3 * batch_size
dataset = dataset.shuffle(buffer_size=buffer_size)
# Batch it up.
dataset = dataset.batch(batch_size, drop_remainder=True)
iterator = tf.compat.v1.data.make_one_shot_iterator(dataset)
image_batch, label_batch = iterator.get_next()
return {INPUT_TENSOR_NAME: image_batch}, label_batch
def input_fn(params):
"""The actual input function."""
batch_size = params['batch_size']
# For training, we want a lot of parallel reading and shuffling.
# For eval, we want no shuffling and parallel reading doesn't matter.
if is_training:
channel = 'train'
else:
channel = 'validation'
if pipe_mode:
print('***** Using pipe_mode!!!!')
ds = PipeModeDataset(channel=channel, record_format='TFRecord')
else:
ds = tf.data.TFRecordDataset(input_files)
if is_training:
ds = ds.repeat()
ds = ds.shuffle(buffer_size=100)
ds = ds.apply(
tf.contrib.data.map_and_batch(
lambda record: _decode_record(record, name_to_features),
batch_size=batch_size,
drop_remainder=drop_remainder))
return ds
def _input_fn():
def _read_and_decode(record):
features = tf.parse_single_example(
record,
features={
'image_raw': tf.FixedLenFeature([], tf.string),
'label': tf.FixedLenFeature([], tf.int64),
})
image = tf.decode_raw(features['image_raw'], tf.uint8)
image.set_shape([HEIGHT * WIDTH * DEPTH])
image = tf.cast(image, tf.float32) * (1. / 255)
label = tf.cast(features['label'], tf.int32)
return {INPUT_TENSOR_NAME: image}, label
ds = PipeModeDataset(channel, record_format='TFRecord')
ds = ds.repeat()
ds = ds.prefetch(batch_size)
ds = ds.map(_read_and_decode, num_parallel_calls=NUM_PARALLEL_BATCHES)
if channel == 'train':
ds = ds.shuffle(buffer_size=batch_size)
ds = ds.batch(batch_size, drop_remainder=True)
ds = ds.make_one_shot_iterator().get_next()
return ds
def _input(epochs, batch_size, channel, channel_name):
filenames = get_filenames(channel_name, channel)
# ----- 추가 부분 (PipeModeDataSet) -----
#dataset = tf.data.TFRecordDataset(filenames)
dataset = PipeModeDataset(channel=channel_name, record_format='TFRecord')
dataset = dataset.repeat(epochs)
dataset = dataset.prefetch(10)
# Parse records.
dataset = dataset.map(
_dataset_parser, num_parallel_calls=10)
# Potentially shuffle records.
if channel_name == 'train':
# Ensure that the capacity is sufficiently large to provide good random
# shuffling.
buffer_size = int(NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN * 0.4) + 3 * batch_size
dataset = dataset.shuffle(buffer_size=buffer_size)
# Batch it up.
dataset = dataset.batch(batch_size, drop_remainder=True)
iterator = dataset.make_one_shot_iterator()
image_batch, label_batch = iterator.get_next()
return {INPUT_TENSOR_NAME: image_batch}, label_batch
def _input_fn(channel):
"""Returns a Dataset for reading from a SageMaker PipeMode channel."""
features = {
'label': tf.FixedLenFeature([], tf.int64),
'feature': tf.FixedLenFeature([480], tf.int64)
}
def parse(record):
parsed = tf.parse_single_example(record, features)
data = tf.reshape(parsed['feature'], [160, 3])
data = tf.cast(data, tf.float32)
label = tf.cast(parsed['label'], tf.int32)
return ({INPUT_TENSOR_NAME: data}, label)
ds = PipeModeDataset(channel=channel, record_format='TFRecord')
ds = ds.shuffle(SHUFFLE_BUFFER_SIZE)
ds = ds.repeat(MAX_EPOCHS)
ds = ds.prefetch(PREFETCH_SIZE)
ds = ds.map(parse, num_parallel_calls=NUM_PARALLEL_BATCHES)
ds = ds.batch(BATCH_SIZE)
return ds
def process_input(epochs, batch_size, channel, channel_name, data_config):
mode = data_config[channel_name]["TrainingInputMode"]
filenames = _get_filenames(channel_name, channel)
# Repeat infinitely.
logging.info("Running {} in {} mode".format(channel_name, mode))
if mode == "Pipe":
from sagemaker_tensorflow import PipeModeDataset
dataset = PipeModeDataset(channel=channel_name,
record_format="TFRecord")
else:
dataset = tf.data.TFRecordDataset(filenames)
dataset = dataset.repeat(epochs)
dataset = dataset.prefetch(10)
# Parse records.
dataset = dataset.map(_dataset_parser, num_parallel_calls=10)
# Potentially shuffle records.
if channel_name == "train":
# Ensure that the capacity is sufficiently large to provide good random
# shuffling.
buffer_size = int(
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN * 0.4) + 3 * batch_size
dataset = dataset.shuffle(buffer_size=buffer_size)
# Batch it up.
dataset = dataset.batch(batch_size, drop_remainder=True)
iterator = dataset.make_one_shot_iterator()
image_batch, label_batch = iterator.get_next()
return image_batch, label_batch
def _input(epochs, batch_size, channel, channel_name, hvd=None):
if hvd != None:
channel_name = '{}_{}'.format(channel_name, hvd.rank() % 4)
print("The channel name is ", channel_name)
channel_input_dir = args.training_env['channel_input_dirs'][channel_name]
print("The corresponding input directory is ", channel_input_dir)
mode = args.data_config[channel_name]['TrainingInputMode']
if mode == 'Pipe':
from sagemaker_tensorflow import PipeModeDataset
dataset = PipeModeDataset(channel=channel_name,
record_format='TFRecord')
else:
filenames = get_filenames(channel_input_dir, hvd)
print("The correpsonding filenames are", filenames)
dataset = tf.data.TFRecordDataset(filenames)
if 'test' in channel_name:
dataset = dataset.map(_dataset_parser_with_slide)
else:
dataset = dataset.repeat(epochs)
dataset = dataset.map(_dataset_parser)
if 'train' in channel_name:
# Ensure that the capacity is sufficiently large to provide good random shuffling.
buffer_size = int(
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN * 0.4) + 3 * batch_size
dataset = dataset.shuffle(buffer_size=buffer_size)
# Batch it up (only for train and valid)
if 'test' not in channel_name:
dataset = dataset.batch(batch_size, drop_remainder=True)
dataset = dataset.prefetch(10)
return dataset
def file_based_input_dataset_builder(
channel,
input_filenames,
pipe_mode,
is_training,
drop_remainder,
batch_size,
epochs,
steps_per_epoch,
max_seq_length,
):
# For training, we want a lot of parallel reading and shuffling.
# For eval, we want no shuffling and parallel reading doesn't matter.
if pipe_mode:
print("***** Using pipe_mode with channel {}".format(channel))
from sagemaker_tensorflow import PipeModeDataset
dataset = PipeModeDataset(channel=channel, record_format="TFRecord")
else:
print("***** Using input_filenames {}".format(input_filenames))
dataset = tf.data.TFRecordDataset(input_filenames)
dataset = dataset.repeat(epochs * steps_per_epoch * 100)
name_to_features = {
"input_ids": tf.io.FixedLenFeature([max_seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([max_seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([max_seq_length], tf.int64),
"label_ids": tf.io.FixedLenFeature([], tf.int64),
}
def _decode_record(record, name_to_features):
"""Decodes a record to a TensorFlow example."""
record = tf.io.parse_single_example(record, name_to_features)
return record
dataset = dataset.apply(
tf.data.experimental.map_and_batch(
lambda record: _decode_record(record, name_to_features),
batch_size=batch_size,
drop_remainder=drop_remainder,
num_parallel_calls=tf.data.experimental.AUTOTUNE,
)
)
dataset = dataset.shuffle(buffer_size=1000, reshuffle_each_iteration=True)
row_count = 0
print("**************** {} *****************".format(channel))
for row in dataset.as_numpy_iterator():
print(row)
if row_count == 5:
break
row_count = row_count + 1
return dataset
def file_based_input_dataset_builder(channel,
input_filenames,
pipe_mode,
is_training,
drop_remainder,
batch_size,
epochs,
steps_per_epoch,
max_seq_length):
# For training, we want a lot of parallel reading and shuffling.
# For eval, we want no shuffling and parallel reading doesn't matter.
if pipe_mode:
print('***** Using pipe_mode with channel {}'.format(channel))
from sagemaker_tensorflow import PipeModeDataset
dataset = PipeModeDataset(channel=channel,
record_format='TFRecord')
else:
print('***** Using input_filenames {}'.format(input_filenames))
dataset = tf.data.TFRecordDataset(input_filenames)
dataset = dataset.repeat(epochs * steps_per_epoch * 100)
dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
name_to_features = {
"input_ids": tf.io.FixedLenFeature([max_seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([max_seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([max_seq_length], tf.int64),
"label_ids": tf.io.FixedLenFeature([], tf.int64),
}
def _decode_record(record, name_to_features):
"""Decodes a record to a TensorFlow example."""
record = tf.io.parse_single_example(record, name_to_features)
# TODO: wip/bert/bert_attention_head_view/train.py
# Convert input_ids into input_tokens with DistilBert vocabulary
# if hook.get_collections()['all'].save_config.should_save_step(modes.EVAL, hook.mode_steps[modes.EVAL]):
# hook._write_raw_tensor_simple("input_tokens", input_tokens)
return record
dataset = dataset.apply(
tf.data.experimental.map_and_batch(
lambda record: _decode_record(record, name_to_features),
batch_size=batch_size,
drop_remainder=drop_remainder,
num_parallel_calls=tf.data.experimental.AUTOTUNE))
dataset.cache()
if is_training:
dataset = dataset.shuffle(seed=42,
buffer_size=100,
reshuffle_each_iteration=True)
return dataset
def _input(epochs, batch_size, channel, channel_name):
mode = args.data_config[channel_name]['TrainingInputMode']
"""Uses the tf.data input pipeline for our dataset.
Args:
mode: Standard names for model modes (tf.estimators.ModeKeys).
batch_size: The number of samples per batch of input requested.
"""
logging.info("Running {} in {} mode for {} epochs".format(channel_name, mode, epochs))
filenames = get_filenames(channel_name, channel)
# Repeat infinitely.
if mode == 'Pipe':
from sagemaker_tensorflow import PipeModeDataset
dataset = PipeModeDataset(channel=channel_name, record_format='TFRecord')
else:
dataset = tf.data.TFRecordDataset(filenames)
dataset = dataset.repeat(epochs)
dataset = dataset.prefetch(batch_size)
# Parse records.
dataset = dataset.map(_dataset_parser, num_parallel_calls=10)
# Shuffle training records.
if channel_name == 'train':
# Ensure that the capacity is sufficiently large to provide good random
# shuffling.
buffer_size = args.num_train_samples // args.batch_size
dataset = dataset.shuffle(buffer_size=buffer_size)
# Batch it up.
dataset = dataset.batch(batch_size, drop_remainder=True)
if tf.version.VERSION[0] == '2':
iterator = tf.compat.v1.data.make_one_shot_iterator(dataset)
else:
iterator = dataset.make_one_shot_iterator()
features_batch, label_batch = iterator.get_next()
if tf.version.VERSION[0] == '2':
with tf.compat.v1.Session() as sess:
logging.info('type of features_batch: {}, type of values: {}'.format(type(features_batch),
type(features_batch)))
logging.info('label_batch: {}'.format(label_batch))
logging.info('type of label_batch: {}'.format(type(label_batch)))
else:
with tf.Session() as sess:
logging.info('type of features_batch: {}, type of values: {}'.format(type(features_batch),
type(features_batch)))
logging.info('label_batch: {}'.format(label_batch))
logging.info('type of label_batch: {}'.format(type(label_batch)))
return {INPUT_TENSOR_NAME: features_batch}, label_batch
def _input(epochs, batch_size, channel, channel_name, hvd=None):
# If Horovod, assign channel name using the horovod rank
if hvd != None:
channel_name = '{}_{}'.format(channel_name, hvd.local_rank())
channel_input_dir = args.training_env['channel_input_dirs'][channel_name]
mode = args.data_config[channel_name]['TrainingInputMode']
"""Uses the tf.data input pipeline for CIFAR-10 dataset.
Args:
mode: Standard names for model modes (tf.estimators.ModeKeys).
batch_size: The number of samples per batch of input requested.
"""
if mode == 'Pipe':
from sagemaker_tensorflow import PipeModeDataset
dataset = PipeModeDataset(channel=channel_name,
record_format='TFRecord') #, benchmark=True)
else:
filenames = get_filenames(channel_input_dir)
print(f'DEBUG tfrecords : {filenames}')
dataset = tf.data.TFRecordDataset(filenames)
if 'train' in channel_name:
dataset = dataset.repeat(epochs)
else:
dataset = dataset.repeat(20)
# Parse records.
dataset = dataset.map(_dataset_parser, num_parallel_calls=10)
# Potentially shuffle records.
# if hvd == None and 'train' in channel_name:
if 'train' in channel_name:
# Ensure that the capacity is sufficiently large to provide good random
# shuffling.
buffer_size = int(
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN * 0.4) + 3 * batch_size
dataset = dataset.shuffle(buffer_size=buffer_size)
# Batch it up.
dataset = dataset.batch(batch_size, drop_remainder=True)
dataset = dataset.prefetch(10)
return dataset
def file_based_input_dataset_builder(channel, input_filenames, pipe_mode,
is_training, drop_remainder, batch_size,
epochs, steps_per_epoch, max_seq_length):
# For training, we want a lot of parallel reading and shuffling.
# For eval, we want no shuffling and parallel reading doesn't matter.
if pipe_mode:
print('***** Using pipe_mode with channel {}'.format(channel))
from sagemaker_tensorflow import PipeModeDataset
dataset = PipeModeDataset(channel=channel, record_format='TFRecord')
else:
print('***** Using input_filenames {}'.format(input_filenames))
dataset = tf.data.TFRecordDataset(input_filenames)
dataset = dataset.repeat(epochs * steps_per_epoch)
dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
name_to_features = {
"input_ids": tf.io.FixedLenFeature([max_seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([max_seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([max_seq_length], tf.int64),
"label_ids": tf.io.FixedLenFeature([], tf.int64),
# "is_real_example": tf.io.FixedLenFeature([], tf.int64),
}
def _decode_record(record, name_to_features):
"""Decodes a record to a TensorFlow example."""
return tf.io.parse_single_example(record, name_to_features)
dataset = dataset.apply(
tf.data.experimental.map_and_batch(
lambda record: _decode_record(record, name_to_features),
batch_size=batch_size,
drop_remainder=drop_remainder,
num_parallel_calls=tf.data.experimental.AUTOTUNE))
dataset.cache()
if is_training:
dataset = dataset.shuffle(seed=42,
buffer_size=1000,
reshuffle_each_iteration=True)
return dataset
def read_dataset(epochs, batch_size, channel, channel_name):
mode = args.data_config[channel_name]["TrainingInputMode"]
logging.info("Running {} in {} mode".format(channel_name, mode))
if mode == "Pipe":
from sagemaker_tensorflow import PipeModeDataset
dataset = PipeModeDataset(channel=channel_name,
record_format="TFRecord")
else:
filenames = [os.path.join(channel, channel_name + ".tfrecords")]
dataset = tf.data.TFRecordDataset(filenames)
image_feature_description = {
"image": tf.io.FixedLenFeature([], tf.string),
"label": tf.io.FixedLenFeature([], tf.int64),
}
def _parse_image_function(example_proto):
# Parse the input tf.Example proto using the dictionary above.
features = tf.io.parse_single_example(example_proto,
image_feature_description)
image = tf.io.decode_raw(features["image"], tf.uint8)
image.set_shape([3 * 32 * 32])
image = tf.reshape(image, [32, 32, 3])
label = tf.cast(features["label"], tf.int32)
label = tf.one_hot(label, 10)
return image, label
dataset = dataset.map(_parse_image_function, num_parallel_calls=10)
dataset = dataset.prefetch(10)
dataset = dataset.repeat(epochs)
dataset = dataset.shuffle(buffer_size=10 * batch_size)
dataset = dataset.batch(batch_size, drop_remainder=True)
return dataset
def parse(record):
parsed = tf.parse_single_example(record, features)
image = tf.decode_raw(parsed['image'], tf.uint8)
image.set_shape([DEPTH * HEIGHT * WIDTH])
image = tf.cast(image, tf.float32)/255.0
label = tf.cast(parsed['label'], tf.int32)
return image, label
ds = PipeModeDataset(channel='train', record_format='TFRecord')
num_epochs = 10
# This yields 40000 (training images)/64 (batch_size) * 10 (epoch) = 6250 batches (steps)
# Tensorflow dataset raises tf.errors.OutOfRangeError when all the batches are fed as described in training-loop
ds = ds.repeat(num_epochs)
ds = ds.prefetch(10)
ds = ds.map(parse, num_parallel_calls=10)
ds = ds.shuffle(buffer_size = 64) #larger than batch_size
ds = ds.batch(batch_size = 64)
iterator = ds.make_one_shot_iterator()
itr_initializer = iterator.make_initializer(ds)
image_batch, label_batch = iterator.get_next()
# Set up PyTorch Neural Network and optimizer
net = MLP().to(device)
if is_distributed and use_cuda:
# multi-machine multi-gpu case
net = torch.nn.parallel.DistributedDataParallel(net)
else:
# single-machine multi-gpu case or single-machine or multi-machine cpu case
net = torch.nn.DataParallel(net)
optimizer = optim.SGD(net.parameters(), lr=0.01, momentum=0.9)