def test_loader_tf_data_set_should_be_ok(self):
builder = loader.DatasetBuilder('data', shape=(256, 256))
dataset = TFRecordDataset('data/test.records')
dataset = dataset.map(builder.decode)
dataset = dataset.map(builder.augmentation())
dataset = dataset.shuffle(4000)
dataset = dataset.batch(batch_size=60)
def main():
parser = argparse.ArgumentParser(description='Running Settings')
parser.add_argument('--model', help='valid options Vgg, ResNet and '
'Squeeze Excitation Models',
required=True)
parser.add_argument('--batch', help='# of batches', type=int,
default=32)
parser.add_argument('--data', help='path where the data is stored',
default='data')
args = parser.parse_args()
if MODELS.get(args.model) is None:
raise ValueError("Model Does not Exist")
builder = DatasetBuilder(args.data, shape=(256, 256))
builder()
data_train = TFRecordDataset(join(args.data, 'train.records'))
data_train = data_train.map(builder.decode)
data_train = data_train.map(builder.augmentation)
data_train = data_train.shuffle(7000)
data_train = data_train.batch(batch_size=args.batch)
data_test = TFRecordDataset(join(args.data, 'test.records'))
data_test = data_test.map(builder.decode)
data_test = data_test.batch(batch_size=args.batch)
model = MODELS.get(args.model)()
model.build((1, 256, 256, 3))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
log_dir = join('logs', args.model)
tensor_board_callback = callbacks.TensorBoard(log_dir=log_dir)
model_checkpoint = callbacks.ModelCheckpoint('models/{}.h5'.format(args.model),
save_best_only=True)
reduce_lr = callbacks.ReduceLROnPlateau(factor=0.2, patience=5,
min_lr=1e-6)
early_stop = callbacks.EarlyStopping(patience=10)
_callbacks = [model_checkpoint, reduce_lr, early_stop,
tensor_board_callback]
model.fit(data_train, epochs=100, validation_data=data_test,
callbacks=_callbacks)
def __init__(self, data_files, sampler_config):
""" Create a new ImageDataGenerator.
Receives a configure dictionary, which specify how to load the data
"""
self.config = sampler_config
self.__check_image_patch_shape()
batch_size = self.config['batch_size']
self.label_convert_source = self.config.get('label_convert_source',
None)
self.label_convert_target = self.config.get('label_convert_target',
None)
data = TFRecordDataset(data_files, "ZLIB")
data = data.map(self._parse_function, num_parallel_calls=5)
if (self.config.get('data_shuffle', False)):
data = data.shuffle(buffer_size=20 * batch_size)
data = data.batch(batch_size)
self.data = data