def __init__(self, mode: str, params:Cifar10Params=Cifar10Params(), batch_size: int = None, noise_batch_size:int = None):
"""
Args:
mode: can be either `'TRAIN'`,`'VALIDATION'`,`'TEST'`. controls which data set to use.
params: a `Cifar10Params` object containing parameters of the images
batch_size: number of images per batch
noise_batch_size: if `None`, the batch will be a four dimensional tensor, whose first dimension is the batch dimension with size `batch_size`. otherwise, the batch will be a five dimensional ternsor, whose first dimension is the noise batch dimension, with size `noise_batch_size`, and the second dimension is the image batch dimension, with size `batch_size`.
order: either `'NCHW'` or `'NHWC'`. (currently only `'NHWC'` is supported for image augmentation preprocessing)
"""
# choose correct path for images, between training, validation and test
if mode.upper() in ['TRAIN','TRAINING']:
mode = 'TRAIN'
path = os.path.join(params.path,'train.tfrecords')
is_training = True
elif mode.upper() in ['VALIDATE', 'VALIDATION', 'VALIDATING']:
mode = 'VALIDATION'
path = os.path.join(params.path, 'validation.tfrecords')
is_training = False
elif mode.upper() in ['EVALUATE', 'EVAL', 'EVALUATION', 'EVALUATING', 'TEST', 'TESTING']:
mode = 'TEST'
path = os.path.join(params.path, 'test.tfrecords')
is_training = False
else:
raise ValueError('wrong value for `mode`')
order = params.order
self._graph = tf.get_default_graph()
# get list of filenames of tfrecords
if os.path.isdir(path):
file_names = glob(path + "/**/*", recursive=True)
if is_training:
random.shuffle(file_names)
else:
file_names = [path]
with self._graph.as_default():
dataset = tf.data.TFRecordDataset(file_names)
dataset = dataset.repeat()
def preprocess_image(image, is_training):
##TODO: make this work for NCHW as well
if is_training:
image = tf.image.resize_image_with_crop_or_pad(image, params.image_size + 8, params.image_size+ 8)
image = tf.random_crop(image, [params.image_size, params.image_size, params.number_of_channels])
image = tf.image.random_flip_left_right(image)
return tf.image.per_image_standardization(image)
def parse_record(raw_record, is_training):
features = tf.parse_single_example(raw_record,features={
'image': tf.FixedLenFeature([], tf.string),
'label': tf.FixedLenFeature([], tf.int64),
})
image = tf.decode_raw(features['image'], tf.uint8)
image = tf.reshape(image, [params.number_of_channels,params.image_size,params.image_size])
image = tf.transpose(image, [1, 2, 0]) # to NHWC
image = tf.cast(image,tf.float32)
image = preprocess_image(image, is_training)
if order=='NCHW':
image = tf.transpose(image, [2, 0, 1]) # to NCHW
label = tf.one_hot(tf.cast(features['label'], tf.int32),params.num_classes)
return image, label
dataset = dataset.map(lambda value: parse_record(value, is_training))
if (batch_size is None):
batch_size = 1
if is_training:
# TODO: I just took this formula from somewhere, it should be empirical justified...
dataset = dataset.shuffle(int(params.training_set_size*0.4+3*batch_size))
if noise_batch_size is None:
total_batch_size = batch_size
else:
total_batch_size = batch_size*noise_batch_size
if mode == 'VALIDATION':
set_size = params.validation_set_size
else:
set_size = params.test_set_size
if (set_size % total_batch_size) != 0:
raise ValueError("in modes 'VALIDATION' and 'TEST', total batch size should divide set size")
dataset = dataset.apply(tf.contrib.data.batch_and_drop_remainder(total_batch_size))
if noise_batch_size is not None:
def reshape(images, labels):
if order == 'NHWC':
images = tf.reshape(images, [noise_batch_size, batch_size, params.image_size, params.image_size, params.number_of_channels])
if order == 'NCHW':
images = tf.reshape(images, [noise_batch_size, batch_size, params.number_of_channels, params.image_size, params.image_size])
labels = tf.reshape(labels, [noise_batch_size, batch_size, params.num_classes])
return images, labels
dataset = dataset.map(lambda images,labels: reshape(images,labels))
self._next = dataset.make_one_shot_iterator().get_next()
self.image, self.label = self._next
from cifar10_data_fetcher import Cifar10ConvertToTFRecords from params import Cifar10Params Cifar10ConvertToTFRecords(Cifar10Params())
from utils import CreateOutputDir, GetLogger
from hypernetwork import Hypernetwork
from cifar10_data_fetcher import Cifar10DataFetcher
from params import GeneralParameters, HypernetworkHyperParameters, Cifar10Params, ResNetCifar10HyperParameters
initialize_from_checkpoint = False
output_dir = CreateOutputDir('output/', __file__)
logger = GetLogger(log_file_mode='a' if initialize_from_checkpoint else 'w',
log_file_path=os.path.join(output_dir, 'log.txt'))
# create parameter objects
general_params = GeneralParameters()
hparams = HypernetworkHyperParameters()
target_hparams = ResNetCifar10HyperParameters()
image_params = Cifar10Params()
# override
hparams.initialization_std = 1e-3
hparams.learning_rate = 1e-6
hparams.learning_rate_rate = 0.9999
hparams.lamBda = 1e8
target_hparams.batch_type = 'BATCH_TYPE5'
# set random seeds
np.random.seed(general_params.seed)
tf.set_random_seed(general_params.seed)
# get training data pipeline. TODO: get also validation data
training_data = Cifar10DataFetcher('TRAIN',
def __init__(self,
x,
y,
mode: str = 'EVAL',
general_params=GeneralParameters(),
hnet_hparams=HypernetworkHyperParameters(),
target_hparams:
ResNetHyperParameters = ResNetCifar10HyperParameters(),
image_params: DataParams = Cifar10Params(),
devices=None,
graph=None):
"""
Args:
x: image batch
y: labels (one hot)
mode: either `'TRAIN'` or `'EVAL'`. determines whether to add optimization ops
general_params:
hnet_hparams:
target_hparams:
image_params:
devices: (optional) a dictionary, with keys `'cpu'` and `'gpus'`, where the former is the cpu device and the latter is a list of gpu devices.
graph:
"""
if mode.upper() in ['TRAIN', 'TRAINING']:
mode = 'TRAIN'
elif mode.upper() in [
'VALIDATE', 'VALIDATION', 'VALIDATING', 'EVALUATE', 'EVAL',
'EVALUATION', 'EVALUATING', 'TEST', 'TESTING'
]:
mode = 'EVAL'
else:
raise ValueError(
"mode needs to be one of 'TRAIN','VALIDATE','INFER'")
self.general_params = general_params
self.hnet_hparams = hnet_hparams
self.target_hparams = target_hparams
self.image_params = image_params
self.graph = graph
if devices is None:
devices = GetDevices()
self.devices = devices
self.cpu = devices['cpu']
# we will cycle over the GPU list, so that our code is agnostic to the number of GPUs
self.gpus = cycle(
devices['gpus']) if len(devices['gpus']) > 0 else cycle(
[devices['cpu']])
self.x = x
self.y = y
if graph is None:
graph = tf.get_default_graph()
self.graph = graph
with self.graph.as_default():
self.z = tf.placeholder(tf.float32,
[None, hnet_hparams.input_noise_size])
self.is_training = tf.Variable(False,
trainable=False,
name='is_training')
with tf.variable_scope('generator', reuse=tf.AUTO_REUSE):
self.__AddGeneratorOps()
self.__AddEvaluationOps()
if mode == 'TRAIN':
self.__AddOptimizationOps()
self.initializer = tf.variables_initializer(
self.graph.get_collection('variables'), name='initializer')