def __init__(self, mode: str, params:Cifar10Params=Cifar10Params(), batch_size: int = None, noise_batch_size:int = None): """ Args: mode: can be either `'TRAIN'`,`'VALIDATION'`,`'TEST'`. controls which data set to use. params: a `Cifar10Params` object containing parameters of the images batch_size: number of images per batch noise_batch_size: if `None`, the batch will be a four dimensional tensor, whose first dimension is the batch dimension with size `batch_size`. otherwise, the batch will be a five dimensional ternsor, whose first dimension is the noise batch dimension, with size `noise_batch_size`, and the second dimension is the image batch dimension, with size `batch_size`. order: either `'NCHW'` or `'NHWC'`. (currently only `'NHWC'` is supported for image augmentation preprocessing) """ # choose correct path for images, between training, validation and test if mode.upper() in ['TRAIN','TRAINING']: mode = 'TRAIN' path = os.path.join(params.path,'train.tfrecords') is_training = True elif mode.upper() in ['VALIDATE', 'VALIDATION', 'VALIDATING']: mode = 'VALIDATION' path = os.path.join(params.path, 'validation.tfrecords') is_training = False elif mode.upper() in ['EVALUATE', 'EVAL', 'EVALUATION', 'EVALUATING', 'TEST', 'TESTING']: mode = 'TEST' path = os.path.join(params.path, 'test.tfrecords') is_training = False else: raise ValueError('wrong value for `mode`') order = params.order self._graph = tf.get_default_graph() # get list of filenames of tfrecords if os.path.isdir(path): file_names = glob(path + "/**/*", recursive=True) if is_training: random.shuffle(file_names) else: file_names = [path] with self._graph.as_default(): dataset = tf.data.TFRecordDataset(file_names) dataset = dataset.repeat() def preprocess_image(image, is_training): ##TODO: make this work for NCHW as well if is_training: image = tf.image.resize_image_with_crop_or_pad(image, params.image_size + 8, params.image_size+ 8) image = tf.random_crop(image, [params.image_size, params.image_size, params.number_of_channels]) image = tf.image.random_flip_left_right(image) return tf.image.per_image_standardization(image) def parse_record(raw_record, is_training): features = tf.parse_single_example(raw_record,features={ 'image': tf.FixedLenFeature([], tf.string), 'label': tf.FixedLenFeature([], tf.int64), }) image = tf.decode_raw(features['image'], tf.uint8) image = tf.reshape(image, [params.number_of_channels,params.image_size,params.image_size]) image = tf.transpose(image, [1, 2, 0]) # to NHWC image = tf.cast(image,tf.float32) image = preprocess_image(image, is_training) if order=='NCHW': image = tf.transpose(image, [2, 0, 1]) # to NCHW label = tf.one_hot(tf.cast(features['label'], tf.int32),params.num_classes) return image, label dataset = dataset.map(lambda value: parse_record(value, is_training)) if (batch_size is None): batch_size = 1 if is_training: # TODO: I just took this formula from somewhere, it should be empirical justified... dataset = dataset.shuffle(int(params.training_set_size*0.4+3*batch_size)) if noise_batch_size is None: total_batch_size = batch_size else: total_batch_size = batch_size*noise_batch_size if mode == 'VALIDATION': set_size = params.validation_set_size else: set_size = params.test_set_size if (set_size % total_batch_size) != 0: raise ValueError("in modes 'VALIDATION' and 'TEST', total batch size should divide set size") dataset = dataset.apply(tf.contrib.data.batch_and_drop_remainder(total_batch_size)) if noise_batch_size is not None: def reshape(images, labels): if order == 'NHWC': images = tf.reshape(images, [noise_batch_size, batch_size, params.image_size, params.image_size, params.number_of_channels]) if order == 'NCHW': images = tf.reshape(images, [noise_batch_size, batch_size, params.number_of_channels, params.image_size, params.image_size]) labels = tf.reshape(labels, [noise_batch_size, batch_size, params.num_classes]) return images, labels dataset = dataset.map(lambda images,labels: reshape(images,labels)) self._next = dataset.make_one_shot_iterator().get_next() self.image, self.label = self._next
from cifar10_data_fetcher import Cifar10ConvertToTFRecords from params import Cifar10Params Cifar10ConvertToTFRecords(Cifar10Params())
from utils import CreateOutputDir, GetLogger from hypernetwork import Hypernetwork from cifar10_data_fetcher import Cifar10DataFetcher from params import GeneralParameters, HypernetworkHyperParameters, Cifar10Params, ResNetCifar10HyperParameters initialize_from_checkpoint = False output_dir = CreateOutputDir('output/', __file__) logger = GetLogger(log_file_mode='a' if initialize_from_checkpoint else 'w', log_file_path=os.path.join(output_dir, 'log.txt')) # create parameter objects general_params = GeneralParameters() hparams = HypernetworkHyperParameters() target_hparams = ResNetCifar10HyperParameters() image_params = Cifar10Params() # override hparams.initialization_std = 1e-3 hparams.learning_rate = 1e-6 hparams.learning_rate_rate = 0.9999 hparams.lamBda = 1e8 target_hparams.batch_type = 'BATCH_TYPE5' # set random seeds np.random.seed(general_params.seed) tf.set_random_seed(general_params.seed) # get training data pipeline. TODO: get also validation data training_data = Cifar10DataFetcher('TRAIN',
def __init__(self, x, y, mode: str = 'EVAL', general_params=GeneralParameters(), hnet_hparams=HypernetworkHyperParameters(), target_hparams: ResNetHyperParameters = ResNetCifar10HyperParameters(), image_params: DataParams = Cifar10Params(), devices=None, graph=None): """ Args: x: image batch y: labels (one hot) mode: either `'TRAIN'` or `'EVAL'`. determines whether to add optimization ops general_params: hnet_hparams: target_hparams: image_params: devices: (optional) a dictionary, with keys `'cpu'` and `'gpus'`, where the former is the cpu device and the latter is a list of gpu devices. graph: """ if mode.upper() in ['TRAIN', 'TRAINING']: mode = 'TRAIN' elif mode.upper() in [ 'VALIDATE', 'VALIDATION', 'VALIDATING', 'EVALUATE', 'EVAL', 'EVALUATION', 'EVALUATING', 'TEST', 'TESTING' ]: mode = 'EVAL' else: raise ValueError( "mode needs to be one of 'TRAIN','VALIDATE','INFER'") self.general_params = general_params self.hnet_hparams = hnet_hparams self.target_hparams = target_hparams self.image_params = image_params self.graph = graph if devices is None: devices = GetDevices() self.devices = devices self.cpu = devices['cpu'] # we will cycle over the GPU list, so that our code is agnostic to the number of GPUs self.gpus = cycle( devices['gpus']) if len(devices['gpus']) > 0 else cycle( [devices['cpu']]) self.x = x self.y = y if graph is None: graph = tf.get_default_graph() self.graph = graph with self.graph.as_default(): self.z = tf.placeholder(tf.float32, [None, hnet_hparams.input_noise_size]) self.is_training = tf.Variable(False, trainable=False, name='is_training') with tf.variable_scope('generator', reuse=tf.AUTO_REUSE): self.__AddGeneratorOps() self.__AddEvaluationOps() if mode == 'TRAIN': self.__AddOptimizationOps() self.initializer = tf.variables_initializer( self.graph.get_collection('variables'), name='initializer')