def _build_graph(self, tf_graph, scope, model_dir): """Construct a TensorGraph containing the policy and loss calculations.""" state_shape = self._env.state_shape state_dtype = self._env.state_dtype if not self._state_is_list: state_shape = [state_shape] state_dtype = [state_dtype] features = [] for s, d in zip(state_shape, state_dtype): features.append(Feature(shape=[None] + list(s), dtype=tf.as_dtype(d))) policy_layers = self._policy.create_layers(features) action_prob = policy_layers['action_prob'] value = policy_layers['value'] search_prob = Label(shape=(None, self._env.n_actions)) search_value = Label(shape=(None,)) loss = MCTSLoss( self.value_weight, in_layers=[action_prob, value, search_prob, search_value]) graph = TensorGraph( batch_size=self.max_search_depth, use_queue=False, graph=tf_graph, model_dir=model_dir) for f in features: graph._add_layer(f) graph.add_output(action_prob) graph.add_output(value) graph.set_loss(loss) graph.set_optimizer(self._optimizer) with graph._get_tf("Graph").as_default(): with tf.variable_scope(scope): graph.build() if len(graph.rnn_initial_states) > 0: raise ValueError('MCTS does not support policies with recurrent layers') return graph, features, action_prob, value, search_prob, search_value
def _setup_training(self): """Sets up graph, model and trainer.""" self._graph = tf.Graph() with self._graph.as_default(): tf.set_random_seed(self.tf_random_seed) self._global_step = tf.Variable(0, name="global_step", trainable=False) # Setting up input and output placeholders. input_shape = [None] + self._data_feeder.input_shape[1:] output_shape = [None] + self._data_feeder.output_shape[1:] self._inp = tf.placeholder( tf.as_dtype(self._data_feeder.input_dtype), input_shape, name="input") self._out = tf.placeholder( tf.as_dtype(self._data_feeder.output_dtype), output_shape, name="output") # Create model's graph. self._model_predictions, self._model_loss = self.model_fn(self._inp, self._out) # Create trainer and augment graph with gradients and optimizer. self._trainer = TensorFlowTrainer(self._model_loss, self._global_step, self.optimizer, self.learning_rate) self._session = tf.Session(self.tf_master, config=tf.ConfigProto(log_device_placement=self.log_device_placement))
def _setup_training(self): """Sets up graph, model and trainer.""" self._graph = tf.Graph() self._graph.add_to_collection("IS_TRAINING", True) with self._graph.as_default(): tf.set_random_seed(self.tf_random_seed) self._global_step = tf.Variable( 0, name="global_step", trainable=False) # Setting up input and output placeholders. input_shape = [None] + self._data_feeder.input_shape[1:] output_shape = [None] + self._data_feeder.output_shape[1:] self._inp = tf.placeholder( tf.as_dtype(self._data_feeder.input_dtype), input_shape, name="input") self._out = tf.placeholder( tf.as_dtype(self._data_feeder.output_dtype), output_shape, name="output") # If class weights are provided, add them to the graph. # Different loss functions can use this tensor by name. if self.class_weight: self._class_weight_node = tf.constant( self.class_weight, name='class_weight') # Add histograms for X and y if they are floats. if self._data_feeder.input_dtype in (np.float32, np.float64): tf.histogram_summary("X", self._inp) if self._data_feeder.output_dtype in (np.float32, np.float64): tf.histogram_summary("y", self._out) # Create model's graph. self._model_predictions, self._model_loss = self.model_fn( self._inp, self._out) # Create summary to monitor loss tf.scalar_summary("loss", self._model_loss) # Set up a single operator to merge all the summaries self._summaries = tf.merge_all_summaries() # Create trainer and augment graph with gradients and optimizer. # Additionally creates initialization ops. self._trainer = TensorFlowTrainer( loss=self._model_loss, global_step=self._global_step, optimizer=self.optimizer, learning_rate=self.learning_rate) # Create model's saver capturing all the nodes created up until now. self._saver = tf.train.Saver( max_to_keep=self.max_to_keep, keep_checkpoint_every_n_hours=self.keep_checkpoint_every_n_hours) # Enable monitor to create validation data dict with appropriate tf placeholders self._monitor.create_val_feed_dict(self._inp, self._out) # Create session to run model with. if self.config_addon is None: self.config_addon = ConfigAddon(verbose=self.verbose) self._session = tf.Session(self.tf_master, config=self.config_addon.config)
def _setup_training(self): """Sets up graph, model and trainer.""" self._graph = tf.Graph() self._graph.add_to_collection("IS_TRAINING", True) with self._graph.as_default(): tf.set_random_seed(self.tf_random_seed) self._global_step = tf.Variable( 0, name="global_step", trainable=False) # Setting up input and output placeholders. input_shape = [None] + self._data_feeder.input_shape[1:] output_shape = [None] + self._data_feeder.output_shape[1:] self._inp = tf.placeholder( tf.as_dtype(self._data_feeder.input_dtype), input_shape, name="input") self._out = tf.placeholder( tf.as_dtype(self._data_feeder.output_dtype), output_shape, name="output") # Add histograms for X and y if they are floats. if self._data_feeder.input_dtype in (np.float32, np.float64): tf.histogram_summary("X", self._inp) if self._data_feeder.output_dtype in (np.float32, np.float64): tf.histogram_summary("y", self._out) # Create model's graph. # pdb.set_trace() self._model_predictions, self._model_loss = self.model_fn( self._inp, self._out) # Create summary to monitor loss tf.scalar_summary("loss", self._model_loss) # Set up a single operator to merge all the summaries self._summaries = tf.merge_all_summaries() with tf.name_scope("test") as scope: if self.n_classes > 0: correct_prediction = tf.equal(tf.argmax(self._out,1), tf.argmax(self._model_predictions,1)) self._cv_accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) self._cv_accuracy_summary = tf.scalar_summary("cv accuracy", self._cv_accuracy) # Create trainer and augment graph with gradients and optimizer. # Additionally creates initialization ops. self._trainer = TensorFlowTrainer( loss=self._model_loss, global_step=self._global_step, optimizer=self.optimizer, learning_rate=self.learning_rate) # Create model's saver capturing all the nodes created up until now. self._saver = tf.train.Saver( max_to_keep=self.max_to_keep, keep_checkpoint_every_n_hours=self.keep_checkpoint_every_n_hours) # Create session to run model with. self._session = tf.Session(self.tf_master, config=tf.ConfigProto( log_device_placement=self.verbose > 1, inter_op_parallelism_threads=self.num_cores, intra_op_parallelism_threads=self.num_cores))
def testAllTypesConvertibleToNumpyDtype(self): for datatype_enum in types_pb2.DataType.values(): if not _is_numeric_dtype_enum(datatype_enum): continue dtype = tf.as_dtype(datatype_enum) numpy_dtype = dtype.as_numpy_dtype _ = np.empty((1, 1, 1, 1), dtype=numpy_dtype) if dtype.base_dtype != tf.bfloat16: # NOTE(touts): Intentionally no way to feed a DT_BFLOAT16. self.assertEqual(tf.as_dtype(datatype_enum).base_dtype, tf.as_dtype(numpy_dtype))
def testIsInteger(self): self.assertEqual(tf.as_dtype("int8").is_integer, True) self.assertEqual(tf.as_dtype("int16").is_integer, True) self.assertEqual(tf.as_dtype("int32").is_integer, True) self.assertEqual(tf.as_dtype("int64").is_integer, True) self.assertEqual(tf.as_dtype("uint8").is_integer, True) self.assertEqual(tf.as_dtype("complex64").is_integer, False) self.assertEqual(tf.as_dtype("float").is_integer, False) self.assertEqual(tf.as_dtype("double").is_integer, False) self.assertEqual(tf.as_dtype("string").is_integer, False) self.assertEqual(tf.as_dtype("bool").is_integer, False)
def testIsFloating(self): self.assertEqual(tf.as_dtype("int8").is_floating, False) self.assertEqual(tf.as_dtype("int16").is_floating, False) self.assertEqual(tf.as_dtype("int32").is_floating, False) self.assertEqual(tf.as_dtype("int64").is_floating, False) self.assertEqual(tf.as_dtype("uint8").is_floating, False) self.assertEqual(tf.as_dtype("complex64").is_floating, False) self.assertEqual(tf.as_dtype("float32").is_floating, True) self.assertEqual(tf.as_dtype("float64").is_floating, True) self.assertEqual(tf.as_dtype("string").is_floating, False) self.assertEqual(tf.as_dtype("bool").is_floating, False)
def testIsUnsigned(self): self.assertEqual(tf.as_dtype("int8").is_unsigned, False) self.assertEqual(tf.as_dtype("int16").is_unsigned, False) self.assertEqual(tf.as_dtype("int32").is_unsigned, False) self.assertEqual(tf.as_dtype("int64").is_unsigned, False) self.assertEqual(tf.as_dtype("uint8").is_unsigned, True) self.assertEqual(tf.as_dtype("float32").is_unsigned, False) self.assertEqual(tf.as_dtype("float64").is_unsigned, False) self.assertEqual(tf.as_dtype("bool").is_unsigned, False) self.assertEqual(tf.as_dtype("string").is_unsigned, False) self.assertEqual(tf.as_dtype("complex64").is_unsigned, False)
def fprop(self, x, **kwargs): del kwargs with tf.variable_scope(self.scope, reuse=tf.AUTO_REUSE): w1 = tf.constant([[1.5, .3], [-2, 0.3]], dtype=tf.as_dtype(x.dtype)) w2 = tf.constant([[-2.4, 1.2], [0.5, -2.3]], dtype=tf.as_dtype(x.dtype)) h1 = tf.nn.sigmoid(tf.matmul(x, w1)) res = tf.matmul(h1, w2) return {self.O_LOGITS: res, self.O_PROBS: tf.nn.softmax(res)}
def __init__(self, images, labels, fake_data=False, one_hot=False, dtype=tf.float32): """Construct a DataSet. one_hot arg is used only if fake_data is true. `dtype` can be either `uint8` to leave the input as `[0, 255]`, or `float32` to rescale into `[0, 1]`. """ dtype = tf.as_dtype(dtype).base_dtype if dtype not in (tf.uint8, tf.float32): raise TypeError('Invalid image dtype %r, expected uint8 or float32' % dtype) if fake_data: self._num_examples = 10000 self.one_hot = one_hot else: assert images.shape[0] == labels.shape[0], ( 'images.shape: %s labels.shape: %s' % (images.shape, labels.shape)) self._num_examples = images.shape[0] self._images = images self._labels = labels self._epochs_completed = 0 self._index_in_epoch = 0
def __init__(self, images, labels, preprocess="scale", da=False, dtype=tf.float32): dtype = tf.as_dtype(dtype).base_dtype if dtype not in (tf.uint8, tf.float32): raise TypeError('Invalid image dtype %r, expected uint8 or float32' % dtype) assert images.shape[0] == labels.shape[0], ( 'images.shape: %s labels.shape: %s' % (images.shape, labels.shape)) self._num_examples = images.shape[0] if dtype == tf.float32: # Convert from [0, 255] -> [0.0, 1.0]. images = images.astype(np.float32) if preprocess=="IMAGENET": VGG_MEAN=np.array([123.68, 116.779, 103.99]) images[:,:,:,0]-= VGG_MEAN[0] images[:,:,:,1]-= VGG_MEAN[1] images[:,:,:,2]-= VGG_MEAN[2] elif preprocess=="scale": images = np.multiply(images, 1.0 / 255.0) self._images = images self._labels = labels self._DA=da self._epochs_completed = 0 self._index_in_epoch = 0
def __init__(self, images, labels, fake_data=False, one_hot=False, dtype=tf.float32): """Construct a DataSet. one_hot arg is used only if fake_data is true. `dtype` can be either `uint8` to leave the input as `[0, 255]`, or `float32` to rescale into `[0, 1]`. """ dtype = tf.as_dtype(dtype).base_dtype if dtype not in (tf.uint8, tf.float32): raise TypeError('Invalid image dtype %r, expected uint8 or float32' % dtype) if fake_data: self._num_examples = 10000 self.one_hot = one_hot else: assert images.shape[0] == labels.shape[0], ( 'images.shape: %s labels.shape: %s' % (images.shape, labels.shape)) self._num_examples = images.shape[0] # Convert shape from [num examples, rows, columns, depth] # to [num examples, rows*columns] (assuming depth == 1) assert images.shape[3] == 1 images = images.reshape(images.shape[0], images.shape[1] * images.shape[2]) if dtype == tf.float32: # Convert from [0, 255] -> [0.0, 1.0]. images = images.astype(numpy.float32) images = numpy.multiply(images, 1.0 / 255.0) self._images = images self._labels = labels self._epochs_completed = 0 self._index_in_epoch = 0
def random_sign_uniform( shape, minval=None, maxval=None, dtype=tf.float32, seed=None): """Tensor with (possibly complex) random entries from a "sign Uniform". Letting `Z` be a random variable equal to `-1` and `1` with equal probability, Samples from this `Op` are distributed like ``` Z * X, where X ~ Uniform[minval, maxval], if dtype is real, Z * (X + iY), where X, Y ~ Uniform[minval, maxval], if dtype is complex. ``` Args: shape: `TensorShape` or Python list. Shape of the returned tensor. minval: `0-D` `Tensor` giving the minimum values. maxval: `0-D` `Tensor` giving the maximum values. dtype: `TensorFlow` `dtype` or Python dtype seed: Python integer seed for the RNG. Returns: `Tensor` with desired shape and dtype. """ dtype = tf.as_dtype(dtype) with tf.name_scope("random_sign_uniform"): unsigned_samples = random_uniform( shape, minval=minval, maxval=maxval, dtype=dtype, seed=seed) if seed is not None: seed += 12 signs = tf.sign(tf.random_uniform(shape, minval=-1., maxval=1., seed=seed)) return unsigned_samples * tf.cast(signs, unsigned_samples.dtype)
def __init__(self, instances, labels, xlength, ylength, imagedata, patch_image_width, dtype = tf.float32): dtype = tf.as_dtype(dtype).base_dtype if dtype not in (tf.uint8, tf.float32): raise TypeError('Invalid image dtype %r, expected uint8 or float32' % dtype) assert instances.shape[0] == labels.shape[0], ( 'instances.shape: %s labels.shape: %s' % (instances.shape, labels.shape)) self._num_examples = instances.shape[0] if dtype == tf.float32: for i in range(len(imagedata)): imagedata[i] = imagedata[i].astype(np.float32) imagedata[i] = np.multiply(imagedata[i], 1.0 / 255.0) self._instances = instances self._labels = labels self._epochs_completed = 0 self._index_in_epoch = 0 self._xlength = xlength self._ylength = ylength self._imagedata = imagedata self._patch_image_width= patch_image_width
def random_normal(shape, mean=0.0, stddev=1.0, dtype=tf.float32, seed=None): """Tensor with (possibly complex) Gaussian entries. Samples are distributed like ``` N(mean, stddev^2), if dtype is real, X + iY, where X, Y ~ N(mean, stddev^2) if dtype is complex. ``` Args: shape: `TensorShape` or Python list. Shape of the returned tensor. mean: `Tensor` giving mean of normal to sample from. stddev: `Tensor` giving stdev of normal to sample from. dtype: `TensorFlow` `dtype` or numpy dtype seed: Python integer seed for the RNG. Returns: `Tensor` with desired shape and dtype. """ dtype = tf.as_dtype(dtype) with tf.name_scope("random_normal"): samples = tf.random_normal( shape, mean=mean, stddev=stddev, dtype=dtype.real_dtype, seed=seed) if dtype.is_complex: if seed is not None: seed += 1234 more_samples = tf.random_normal( shape, mean=mean, stddev=stddev, dtype=dtype.real_dtype, seed=seed) samples = tf.complex(samples, more_samples) return samples
def __init__(self, expression_profile, labels, feature_name, label_name, dtype=tf.float32): """Construct a DataSet. `dtype` can be either `uint8` to leave the input as `[0, 255]`, or `float32` to rescale into `[0, 1]`. """ dtype = tf.as_dtype(dtype).base_dtype if dtype not in (tf.uint8, tf.float32): raise TypeError('Invalid Dataset input dtype %r, expected uint8 or float32' % dtype) assert expression_profile.shape[0] == labels.shape[0], ( 'expression_profile.shape: %s labels.shape: %s' % (expression_profile.shape, labels.shape)) self._num_examples = expression_profile.shape[0] # Convert shape from [num examples, rows, columns, depth] # to [num examples, rows*columns] (assuming depth == 1) assert expression_profile.shape[1] == 77 expression_profile = expression_profile.reshape(expression_profile.shape[0], expression_profile.shape[1]) self._expression_profile = expression_profile self._labels = labels self._epochs_completed = 0 self._index_in_epoch = 0 self._feature_name = feature_name self._label_name = label_name
def __init__(self, band_count, images, labels, dtype=tf.float32): """ Construct a DataSet. `dtype` can be either `uint8` to leave the input as `[0, 255]`, or `float32` to rescale into `[0, 1]`. """ dtype = tf.as_dtype(dtype).base_dtype if dtype not in (tf.uint8, tf.float32): raise TypeError('Invalid image dtype %r, expected uint8 or float32' % dtype) assert images.shape[0] == labels.shape[0], ( 'images.shape: %s labels.shape: %s' % (images.shape, labels.shape)) self._num_examples = images.shape[0] # Convert shape from [num examples, rows, columns, depth] # to [num examples, rows*columns] (assuming depth == 1) assert images.shape[3] == band_count # Store the width and height of the images before flattening it, if only for reference. image_height, image_width = images.shape[1], images.shape[2] self.original_image_width = image_width self.original_image_height = image_height images = images.reshape(images.shape[0], images.shape[1] * images.shape[2]*images.shape[3]) if dtype == tf.float32: # Convert from [0, 255] -> [0.0, 1.0] images = images.astype(numpy.float32) images = numpy.multiply(images, 1.0 / 255.0) self._images = images self._labels = labels self._epochs_completed = 0 self._index_in_epoch = 0
def _ProcessHealthPill(self, wall_time, step, device_name, node_name, output_slot, elements): """Processes a health pill value by adding it to accumulated state. Args: wall_time: The time at which the health pill was created. Provided by the debugger. step: The step at which the health pill was created. Provided by the debugger. device_name: The name of the node's device. node_name: The name of the node for this health pill. output_slot: The output slot for this health pill. elements: An ND array of 20 floats. The elements of the health pill. """ # Key by the node name for fast retrieval of health pills by node name. The # array is cast to a list so that it is JSON-able. The debugger data plugin # serves a JSON response. self._health_pills.AddItem(node_name, HealthPillEvent( wall_time=wall_time, step=step, device_name=device_name, node_name=node_name, output_slot=output_slot, dtype=repr(tf.as_dtype(elements[12])), shape=list(elements[14:]), value=list(elements)))
def _testTensorArrayGradientWriteReadType(self, use_gpu, dtype): with self.test_session(use_gpu=use_gpu) as sess: h = data_flow_ops.TensorArray( dtype=tf.as_dtype(dtype), tensor_array_name="foo", size=3) c = lambda x: np.array(x, dtype=dtype) value_0 = tf.constant(c([[4.0, 5.0]])) value_1 = tf.constant(c(3.0)) w0 = h.write(0, value_0) w1 = w0.write(1, value_1) r0 = w1.read(0) r1 = w1.read(1) # Test individual components' gradients grad_just_r0 = tf.gradients( ys=[r0], xs=[value_0], grad_ys=[c([[2.0, 3.0]])]) grad_just_r0_vals = sess.run(grad_just_r0) self.assertAllEqual(c([[2.0, 3.0]]), grad_just_r0_vals[0]) grad_just_r1 = tf.gradients( ys=[r1], xs=[value_1], grad_ys=[c(-2.0)]) grad_just_r1_vals = sess.run(grad_just_r1) self.assertAllEqual(c(-2.0), grad_just_r1_vals[0]) # Test combined gradients grad = tf.gradients( ys=[r0, r1], xs=[value_0, value_1], grad_ys=[c(-1.0), c([[2.0, 3.0]])]) grad_vals = sess.run(grad) self.assertEqual(len(grad_vals), 2) self.assertAllClose(c(-1.0), grad_vals[0]) self.assertAllEqual(c([[2.0, 3.0]]), grad_vals[1])
def _compute_health_pill(self, x): x_clean = x[np.where( np.logical_and( np.logical_not(np.isnan(x)), np.logical_not(np.isinf(x))))] if np.size(x_clean): x_min = np.min(x_clean) x_max = np.max(x_clean) x_mean = np.mean(x_clean) x_var = np.var(x_clean) else: x_min = np.inf x_max = -np.inf x_mean = np.nan x_var = np.nan return np.array([ 1.0, # Assume is initialized. np.size(x), np.sum(np.isnan(x)), np.sum(x == -np.inf), np.sum(np.logical_and(x < 0.0, x != -np.inf)), np.sum(x == 0.0), np.sum(np.logical_and(x > 0.0, x != np.inf)), np.sum(x == np.inf), x_min, x_max, x_mean, x_var, float(tf.as_dtype(x.dtype).as_datatype_enum), float(len(x.shape)), ] + list(x.shape))
def testDtypeErrors(self): def _TryMakingScalarSummary(dtype): base = dtype.base_dtype if base == tf.bool: v = False elif base == tf.string: v = '' elif base.is_complex: v = complex(0, 0) else: v = base.min c = tf.constant(v, dtype) return tf.summary.scalar('name', c) for datatype_enum in types_pb2.DataType.values(): if datatype_enum == types_pb2.DT_INVALID: continue dtype = tf.as_dtype(datatype_enum) if dtype.is_quantized: # Quantized ops are funky, and not expected to work. continue if dtype.is_integer or dtype.is_floating: _TryMakingScalarSummary(dtype) # No exception should be thrown else: with self.assertRaises(ValueError): _TryMakingScalarSummary(dtype)
def common_dtype(args_list, preferred_dtype=None): """Returns explict dtype from `args_list` if there is one.""" dtype = None while args_list: a = args_list.pop() if hasattr(a, 'dtype'): dt = tf.as_dtype(getattr(a, 'dtype')).base_dtype.as_numpy_dtype else: if isinstance(a, list): # Allows for nested types, e.g. Normal([np.float16(1.0)], [2.0]) args_list.extend(a) continue if dtype is None: dtype = dt elif dtype != dt: raise TypeError('Found incompatible dtypes, {} and {}.'.format(dtype, dt)) return preferred_dtype if dtype is None else tf.as_dtype(dtype)
def as_dtype(dtype): """ Convert the specified dtype to backend dtype. :param dtype: String or numpy dtype. :return: Backend dtype. """ return tf.as_dtype(dtype)
def _check_asset_node_def(node_def): """Raises TypeError if `node_def` does not match the expectations.""" if node_def.op != "Const": raise TypeError("Asset node must be of type constant.") if tf.as_dtype(node_def.attr["dtype"].type) != tf.string: raise TypeError("Asset node must be of dtype string.") if len(node_def.attr["value"].tensor.string_val) != 1: raise TypeError("Asset node must be a scalar.")
def run(self, *in_arrays, return_as_list = False, # True = return a list of NumPy arrays, False = return a single NumPy array, or a tuple if there are multiple outputs. print_progress = False, # Print progress to the console? Useful for very large input arrays. minibatch_size = None, # Maximum minibatch size to use, None = disable batching. num_gpus = 1, # Number of GPUs to use. out_mul = 1.0, # Multiplicative constant to apply to the output(s). out_add = 0.0, # Additive constant to apply to the output(s). out_shrink = 1, # Shrink the spatial dimensions of the output(s) by the given factor. out_dtype = None, # Convert the output to the specified data type. **dynamic_kwargs): # Additional keyword arguments to pass into the network construction function. assert len(in_arrays) == self.num_inputs num_items = in_arrays[0].shape[0] if minibatch_size is None: minibatch_size = num_items key = str([list(sorted(dynamic_kwargs.items())), num_gpus, out_mul, out_add, out_shrink, out_dtype]) # Build graph. if key not in self._run_cache: with absolute_name_scope(self.scope + '/Run'), tf.control_dependencies(None): in_split = list(zip(*[tf.split(x, num_gpus) for x in self.input_templates])) out_split = [] for gpu in range(num_gpus): with tf.device('/gpu:%d' % gpu): out_expr = self.get_output_for(*in_split[gpu], return_as_list=True, **dynamic_kwargs) if out_mul != 1.0: out_expr = [x * out_mul for x in out_expr] if out_add != 0.0: out_expr = [x + out_add for x in out_expr] if out_shrink > 1: ksize = [1, 1, out_shrink, out_shrink] out_expr = [tf.nn.avg_pool(x, ksize=ksize, strides=ksize, padding='VALID', data_format='NCHW') for x in out_expr] if out_dtype is not None: if tf.as_dtype(out_dtype).is_integer: out_expr = [tf.round(x) for x in out_expr] out_expr = [tf.saturate_cast(x, out_dtype) for x in out_expr] out_split.append(out_expr) self._run_cache[key] = [tf.concat(outputs, axis=0) for outputs in zip(*out_split)] # Run minibatches. out_expr = self._run_cache[key] out_arrays = [np.empty([num_items] + shape_to_list(expr.shape)[1:], expr.dtype.name) for expr in out_expr] for mb_begin in range(0, num_items, minibatch_size): if print_progress: print('\r%d / %d' % (mb_begin, num_items), end='') mb_end = min(mb_begin + minibatch_size, num_items) mb_in = [src[mb_begin : mb_end] for src in in_arrays] mb_out = tf.get_default_session().run(out_expr, dict(zip(self.input_templates, mb_in))) for dst, src in zip(out_arrays, mb_out): dst[mb_begin : mb_end] = src # Done. if print_progress: print('\r%d / %d' % (num_items, num_items)) if not return_as_list: out_arrays = out_arrays[0] if len(out_arrays) == 1 else tuple(out_arrays) return out_arrays
def testDTypesHaveUniqueNames(self): dtypes = [] names = set() for datatype_enum in types_pb2.DataType.values(): if datatype_enum == types_pb2.DT_INVALID: continue dtype = tf.as_dtype(datatype_enum) dtypes.append(dtype) names.add(dtype.name) self.assertEqual(len(dtypes), len(names))
def _setup_training(self): """Sets up graph, model and trainer.""" self._graph = tf.Graph() with self._graph.as_default(): tf.set_random_seed(self.tf_random_seed) self._global_step = tf.Variable( 0, name="global_step", trainable=False) # Setting up input and output placeholders. input_shape = [None] + self._data_feeder.input_shape[1:] output_shape = [None] + self._data_feeder.output_shape[1:] self._inp = tf.placeholder( tf.as_dtype(self._data_feeder.input_dtype), input_shape, name="input") self._out = tf.placeholder( tf.as_dtype(self._data_feeder.output_dtype), output_shape, name="output") # Add histograms for X and y. tf.histogram_summary("X", self._inp) tf.histogram_summary("y", self._out) # Create model's graph. self._model_predictions, self._model_loss = self.model_fn( self._inp, self._out) # Create trainer and augment graph with gradients and optimizer. # Additionally creates initialization ops. self._trainer = TensorFlowTrainer( loss=self._model_loss, global_step=self._global_step, optimizer=self.optimizer, learning_rate=self.learning_rate) # Create model's saver capturing all the nodes created up until now. self._saver = tf.train.Saver( max_to_keep=self.max_to_keep, keep_checkpoint_every_n_hours=self.keep_checkpoint_every_n_hours) # Create session to run model with. self._session = tf.Session(self.tf_master, config=tf.ConfigProto( log_device_placement=self.verbose > 1, inter_op_parallelism_threads=self.num_cores, intra_op_parallelism_threads=self.num_cores))
def test_constant(dtype, sess): val = np.random.randn(10, 10).astype(np.float64) signals = SignalDict(tf.float32, 1) const0 = signals.constant(val, dtype=dtype) const1 = signals.constant(val, dtype=dtype, cutoff=0) assert const0.op.type == "Const" assert const1.op.type == "Identity" assert const0.dtype == (dtype if dtype else tf.as_dtype(val.dtype)) assert const1.dtype == (dtype if dtype else tf.as_dtype(val.dtype)) sess.run(tf.variables_initializer(tf.get_collection("constants")), feed_dict=signals.constant_phs) c0, c1 = sess.run([const0, const1]) assert np.allclose(c0, val) assert np.allclose(c1, val)
def make_iterator(self, batch_size=100, epochs=1, deterministic=False, pad_batches=False): """Create a tf.data.Iterator that iterates over the data in this Dataset. The iterator's get_next() method returns a tuple of three tensors (X, y, w) which can be used to retrieve the features, labels, and weights respectively. Parameters ---------- batch_size: int the number of samples to include in each batch epochs: int the number of times to iterate over the Dataset deterministic: bool if True, the data is produced in order. If False, a different random permutation of the data is used for each epoch. pad_batches: bool if True, batches are padded as necessary to make the size of each batch exactly equal batch_size. """ # Retrieve the first sample so we can determine the dtypes. import tensorflow as tf X, y, w, ids = next(self.itersamples()) dtypes = (tf.as_dtype(X.dtype), tf.as_dtype(y.dtype), tf.as_dtype(w.dtype)) shapes = (tf.TensorShape([None] + list(X.shape)), tf.TensorShape([None] + list(y.shape)), tf.TensorShape([None] + list(w.shape))) # Create a Tensorflow Dataset and have it create an Iterator. def gen_data(): for epoch in range(epochs): for X, y, w, ids in self.iterbatches(batch_size, epoch, deterministic, pad_batches): yield (X, y, w) dataset = tf.data.Dataset.from_generator(gen_data, dtypes, shapes) return dataset.make_one_shot_iterator()
def __init__(self, data_dir, coord, train=True, threshold=None, queue_size=16, min_after_dequeue=4, q_shape=None, pattern='*.npy', n_threads=1, multi=True, label_file=LABEL_FILE, label_type=tf.int32, pref=False, dtype=np.float32): self.data_dir = data_dir self.coord = coord self.n_threads = n_threads self.threshold = threshold self.ftype = pattern self.corpus_size = get_corpus_size(self.data_dir, pattern=self.ftype) self.threads = [] self.q_shape = q_shape self.multi = multi self.train = train self.npdtype = dtype self.tfdtype = tf.as_dtype(self.npdtype) self.sample_placeholder = tf.placeholder(dtype=self.tfdtype, shape=None) self.label_shape = [] self.label_type = label_type self.pattern = pattern self.pref = pref self.labels_df = None self.label_shape = [] if self.train: if label_file is not None: self.labels_df = load_label_df(label_file) self.label_shape = [len(self.labels_df.columns)] self.label_placeholder = tf.placeholder(dtype=self.label_type, shape=self.label_shape, name='label') #!!! if self.q_shape: #self.queue = tf.FIFOQueue(queue_size,[tf.float32,tf.int32], shapes=[q_shape,[]]) self.queue = tf.RandomShuffleQueue(queue_size, min_after_dequeue, [self.tfdtype,label_type], shapes=[q_shape, self.label_shape]) else: self.q_shape = [(1, None, None, 1)] self.queue = tf.PaddingFIFOQueue(queue_size, [self.tfdtype, label_type], shapes=[self.q_shape,self.label_shape]) else: self.label_placeholder = tf.placeholder(dtype=tf.string, shape=[], name='label') #!!! self.queue = tf.FIFOQueue(queue_size,[self.tfdtype,tf.string], shapes=[q_shape,[]]) self.enqueue = self.queue.enqueue([self.sample_placeholder, self.label_placeholder])
def run_test(self, name, backend="caffe2", onnx_file=None, opset=None, extra_opset=None, perf=None, fold_const=None): """Run complete test against backend.""" self.perf = perf # get the model if self.url: _, dir_name = self.download_model() logger.info("Downloaded to %s", dir_name) model_path = os.path.join(dir_name, self.local) else: model_path = self.local logger.info("Load model from %s", model_path) input_names = list(self.input_names.keys()) outputs = self.output_names if self.model_type in ["checkpoint"]: graph_def, input_names, outputs = tf_loader.from_checkpoint(model_path, input_names, outputs) elif self.model_type in ["saved_model"]: graph_def, input_names, outputs = tf_loader.from_saved_model(model_path, input_names, outputs) else: graph_def, input_names, outputs = tf_loader.from_graphdef(model_path, input_names, outputs) if utils.is_debug_mode(): utils.save_protobuf(os.path.join(TEMP_DIR, name + "_after_tf_optimize.pb"), graph_def) inputs = {} shape_override = {} tf_reset_default_graph() g = tf.import_graph_def(graph_def, name='') # with tf_session(config=tf.ConfigProto(allow_soft_placement=True), graph=g) as sess: with tf_session(graph=g) as sess: # create the input data for k in input_names: v = self.input_names[k] t = sess.graph.get_tensor_by_name(k) expected_dtype = tf.as_dtype(t.dtype).name if isinstance(v, six.text_type) and v.startswith("np."): np_value = eval(v) # pylint: disable=eval-used if expected_dtype != np_value.dtype: logger.warning("dtype mismatch for input %s: expected=%s, actual=%s", k, expected_dtype, np_value.dtype) inputs[k] = np_value.astype(expected_dtype) else: inputs[k] = self.make_input(v).astype(expected_dtype) if self.force_input_shape: for k, v in inputs.items(): shape_override[k] = list(v.shape) # run the model with tensorflow if self.skip_tensorflow: logger.info("TensorFlow SKIPPED") else: tf_results = self.run_tensorflow(sess, inputs) logger.info("TensorFlow OK") model_proto = None try: # convert model to onnx onnx_graph = self.to_onnx(sess.graph, opset=opset, extra_opset=extra_opset, shape_override=shape_override, input_names=inputs.keys()) onnx_graph = optimizer.optimize_graph(onnx_graph) model_proto = onnx_graph.make_model("converted from tf2onnx") logger.info("To_ONNX, OK") if onnx_file: self.create_onnx_file(name, model_proto, inputs, onnx_file) except Exception: logger.error("To_ONNX FAIL", exc_info=1) return False try: onnx_results = None if backend == "caffe2": onnx_results = self.run_caffe2(name, model_proto, inputs) elif backend == "onnxruntime": onnx_results = self.run_onnxruntime(name, model_proto, inputs) else: raise ValueError("unknown backend") logger.info("Run_ONNX OK") try: if self.skip_tensorflow: logger.info("Results: skipped tensorflow") else: if self.check_only_shape: for tf_res, onnx_res in zip(tf_results, onnx_results): np.testing.assert_array_equal(tf_res.shape, onnx_res.shape) else: for tf_res, onnx_res in zip(tf_results, onnx_results): np.testing.assert_allclose(tf_res, onnx_res, rtol=self.rtol, atol=self.atol) logger.info("Results: OK") return True except Exception: logger.error("Results", exc_info=1) except Exception: logger.error("Run_ONNX FAIL", exc_info=1) return False
def D_stylegan2( images_in, # First input: Images [minibatch, channel, height, width]. labels_in, # Second input: Labels [minibatch, label_size]. num_channels=3, # Number of input color channels. Overridden based on dataset. resolution=1024, # Input resolution. Overridden based on dataset. label_size=0, # Dimensionality of the labels, 0 if no labels. Overridden based on dataset. fmap_base=16 << 10, # Overall multiplier for the number of feature maps. fmap_decay=1.0, # log2 feature map reduction when doubling the resolution. fmap_min=1, # Minimum number of feature maps in any layer. fmap_max=512, # Maximum number of feature maps in any layer. architecture='resnet', # Architecture: 'orig', 'skip', 'resnet'. nonlinearity='lrelu', # Activation function: 'relu', 'lrelu', etc. mbstd_group_size=4, # Group size for the minibatch standard deviation layer, 0 = disable. mbstd_num_features=1, # Number of features for the minibatch standard deviation layer. dtype='float32', # Data type to use for activations and outputs. resample_kernel=[ 1, 3, 3, 1 ], # Low-pass filter to apply when resampling activations. None = no filtering. **_kwargs): # Ignore unrecognized keyword args. resolution_log2 = int(np.log2(resolution)) assert resolution == 2**resolution_log2 and resolution >= 4 def nf(stage): return np.clip(int(fmap_base / (2.0**(stage * fmap_decay))), fmap_min, fmap_max) assert architecture in ['orig', 'skip', 'resnet'] act = nonlinearity images_in.set_shape([None, num_channels, resolution, resolution]) labels_in.set_shape([None, label_size]) images_in = tf.cast(images_in, dtype) labels_in = tf.cast(labels_in, dtype) # Building blocks for main layers. def fromrgb(x, y, res): # res = 2..resolution_log2 with tf.variable_scope('FromRGB'): t = apply_bias_act(conv2d_layer(y, fmaps=nf(res - 1), kernel=1), act=act) return t if x is None else x + t def block(x, res): # res = 2..resolution_log2 t = x with tf.variable_scope('Conv0'): x = apply_bias_act(conv2d_layer(x, fmaps=nf(res - 1), kernel=3), act=act) with tf.variable_scope('Conv1_down'): x = apply_bias_act(conv2d_layer(x, fmaps=nf(res - 2), kernel=3, down=True, resample_kernel=resample_kernel), act=act) if architecture == 'resnet': with tf.variable_scope('Skip'): t = conv2d_layer(t, fmaps=nf(res - 2), kernel=1, down=True, resample_kernel=resample_kernel) x = (x + t) * (1 / np.sqrt(2)) return x def downsample(y): with tf.variable_scope('Downsample'): return downsample_2d(y, k=resample_kernel) # Main layers. x = None y = images_in for res in range(resolution_log2, 2, -1): with tf.variable_scope('%dx%d' % (2**res, 2**res)): if architecture == 'skip' or res == resolution_log2: x = fromrgb(x, y, res) x = block(x, res) if architecture == 'skip': y = downsample(y) # Final layers. with tf.variable_scope('4x4'): if architecture == 'skip': x = fromrgb(x, y, 2) if mbstd_group_size > 1: with tf.variable_scope('MinibatchStddev'): x = minibatch_stddev_layer(x, mbstd_group_size, mbstd_num_features) with tf.variable_scope('Conv'): x = apply_bias_act(conv2d_layer(x, fmaps=nf(1), kernel=3), act=act) with tf.variable_scope('Dense0'): x = apply_bias_act(dense_layer(x, fmaps=nf(0)), act=act) # Output layer with label conditioning from "Which Training Methods for GANs do actually Converge?" with tf.variable_scope('Output'): x = apply_bias_act(dense_layer(x, fmaps=max(labels_in.shape[1], 1))) if labels_in.shape[1] > 0: x = tf.reduce_sum(x * labels_in, axis=1, keepdims=True) scores_out = x # Output. assert scores_out.dtype == tf.as_dtype(dtype) scores_out = tf.identity(scores_out, name='scores_out') return scores_out
def slice_dtype(self): return tf.as_dtype(self._hparams.slice_dtype)
netG_gbc = tf.reduce_mean(input_tensor=tf.stack([tf.reduce_mean(input_tensor=tf.abs(v)) for v in netG_gbc])) netG_gac = tf.reduce_mean(input_tensor=tf.stack([tf.reduce_mean(input_tensor=tf.abs(v)) for v in netG_gac])) with tf.compat.v1.name_scope("netD_SGD"): netD_optimizer = netD.OPTIMIZER netD_gvs = netD_optimizer.compute_gradients(netD_total_loss, tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.TRAINABLE_VARIABLES, scope=netD.variable_scope_name)) netD_gbc = [grad for grad, var in netD_gvs] netD_capped_gvs = [(tf.clip_by_value(grad, -netD.GLOBAL_GRADIENT_CLIPPING, netD.GLOBAL_GRADIENT_CLIPPING), var) for grad, var in netD_gvs] netD_gac = [grad for grad, var in netD_capped_gvs] netD_opt = netD_optimizer.apply_gradients(netD_capped_gvs) netD_gbc = tf.reduce_mean(input_tensor=tf.stack([tf.reduce_mean(input_tensor=tf.abs(v)) for v in netD_gbc])) netD_gac = tf.reduce_mean(input_tensor=tf.stack([tf.reduce_mean(input_tensor=tf.abs(v)) for v in netD_gac])) netG_train_output1_crop_round = [tf.cast(tf.round(tf.clip_by_value(netG_train_output1_crop[i], 0, 1) * train_df.input1_label_src.dtype.max), tf.as_dtype(FLAGS['data_label_dtype'])) for i in range(FLAGS['data_train_batch_size'])] netG_train_input1_label_crop_round = [tf.cast(tf.round(tf.clip_by_value(netG_train_input1_label_crop[i], 0, 1) * train_df.input1_label_src.dtype.max), tf.as_dtype(FLAGS['data_label_dtype'])) for i in range(FLAGS['data_train_batch_size'])] netG_tr_psnr1 = tf.reduce_mean(input_tensor=tf.stack([tf_psnr(netG_train_output1_crop_round[i], netG_train_input1_label_crop_round[i], 0) for i in range(FLAGS['data_train_batch_size'])])) netG_train_output2_crop_round = [tf.cast(tf.round(tf.clip_by_value(netG_train_output2_crop[i], 0, 1) * train_df.input2_label_src.dtype.max), tf.as_dtype(FLAGS['data_input_dtype'])) for i in range(FLAGS['data_train_batch_size'])] netG_train_input2_label_crop_round = [tf.cast(tf.round(tf.clip_by_value(netG_train_input2_label_crop[i], 0, 1) * train_df.input2_label_src.dtype.max), tf.as_dtype(FLAGS['data_input_dtype'])) for i in range(FLAGS['data_train_batch_size'])] netG_tr_psnr2 = tf.reduce_mean(input_tensor=tf.stack([tf_psnr(netG_train_output2_crop_round[i], netG_train_input2_label_crop_round[i], 0) for i in range(FLAGS['data_train_batch_size'])])) netG_tr_psnr2 = tf.constant(0, dtype=FLAGS['data_compute_dtype']) netG_test_output1_crop_round = tf.cast(tf.round(tf.clip_by_value(netG_test_output1_crop, 0, 1) * test_df.input2_src.dtype.max), tf.as_dtype(FLAGS['data_label_dtype'])) netG_test_input2_crop_round = tf.cast(tf.round(tf.clip_by_value(netG_test_input2_crop, 0, 1) * test_df.input2_src.dtype.max), tf.as_dtype(FLAGS['data_label_dtype'])) netG_psnr1 = tf_psnr(netG_test_output1_crop_round, netG_test_input2_crop_round, 0) netG_test_output2_crop_round = tf.cast(tf.round(tf.clip_by_value(netG_test_output2_crop, 0, 1) * test_df.input1_src.dtype.max), tf.as_dtype(FLAGS['data_input_dtype'])) netG_test_input1_crop_round = tf.cast(tf.round(tf.clip_by_value(netG_test_input1_crop, 0, 1) * test_df.input1_src.dtype.max), tf.as_dtype(FLAGS['data_input_dtype'])) netG_psnr2 = tf_psnr(netG_test_output2_crop_round, netG_test_input1_crop_round, 0) netG_train_summary = [netG_train_loss, netG_train_1_1, netG_train_2_1, netG_train_1_2, netG_train_2_2, train_data_term_1, train_data_term_2, train_constant_term_1, train_constant_term_2, netG_tr_psnr1, netG_tr_psnr2, netG_r_loss, netG_gbc, netG_gac]
def __init__(self, base_env_name=None, batch_size=1, grayscale=False, resize_height_factor=2, resize_width_factor=2, base_env_timesteps_limit=-1, max_num_noops=0, **kwargs): if base_env_name is None: base_env_name = self.base_env_name self._base_env_name = base_env_name super(T2TGymEnv, self).__init__(batch_size, **kwargs) self.grayscale = grayscale self.resize_height_factor = resize_height_factor self.resize_width_factor = resize_width_factor if not self.name: # Set problem name if not registered. self.name = "Gym%s" % base_env_name self._envs = [ make_gym_env(base_env_name, base_env_timesteps_limit) for _ in range(self.batch_size) ] # max_num_noops works only with atari envs. if max_num_noops > 0: assert self._envs[0].unwrapped.get_action_meanings()[ self.noop_action] == "NOOP" self.max_num_noops = max_num_noops orig_observ_space = self._envs[0].observation_space if not all(env.observation_space == orig_observ_space for env in self._envs): raise ValueError( "All environments must use the same observation space.") self.observation_space = self._derive_observation_space( orig_observ_space) self.action_space = self._envs[0].action_space if not all(env.action_space == self.action_space for env in self._envs): raise ValueError( "All environments must use the same action space.") with self._tf_graph.obj.as_default(): self._resize = dict() orig_height, orig_width = orig_observ_space.shape[:2] self._img_batch_t = _Noncopyable( tf.placeholder(dtype=tf.uint8, shape=(None, orig_height, orig_width, 3))) height, width = self.observation_space.shape[:2] resized = tf.image.resize_images(self._img_batch_t.obj, [height, width], tf.image.ResizeMethod.AREA) resized = tf.cast(resized, tf.as_dtype(self.observation_space.dtype)) if self.grayscale: resized = tf.image.rgb_to_grayscale(resized) self._resized_img_batch_t = _Noncopyable(resized)
def do_testing(now_epoch, data_loader, best_value_history, indices_1, indices_2, list_train_loss, \ summary_writer, train_summary_datas, is_training): timer = Timer() timer.start() test_avg = [0] * len(netG_test_summary_names) #temporaly removed #test_count = FLAGS['data_test_image_count'] if FLAGS['process_write_test_img_count'] == 0 or is_training == False else FLAGS['process_write_test_img_count'] test_count = FLAGS['data_test_image_count'] test_loss_list = np.zeros((len(netG_test_summary_names), test_count)) for i in range(test_count): timer2 = Timer() timer2.start() label_img = data_loader.get_next_test_label() input_img, data = data_loader.get_next_test_input_batch() update_cache_dict(data['csr_ind1'], data['csr_val1'], data['csr_ind_r1'], data['csr_val_r1'], data['csr_ind_g1'], data['csr_val_g1'], data['csr_ind_b1'], data['csr_val_b1'], data['csr_names1']) update_cache_dict(data['csr_ind2'], data['csr_val2'], data['csr_ind_r2'], data['csr_val_r2'], data['csr_ind_g2'], data['csr_val_g2'], data['csr_ind_b2'], data['csr_val_b2'], data['csr_names2']) dict_d = [\ input_img, label_img] + \ data['rect1'] + data['rot1'] + \ data['rect2'] + data['rot2'] + \ data['csr_ind1'] + data['csr_val1'] + \ data['csr_ind_r1'] + data['csr_val_r1'] + \ data['csr_ind_g1'] + data['csr_val_g1'] + \ data['csr_ind_b1'] + data['csr_val_b1'] + data['csr_sha1'] + \ data['csr_ind2'] + data['csr_val2'] + \ data['csr_ind_r2'] + data['csr_val_r2'] + \ data['csr_ind_g2'] + data['csr_val_g2'] + \ data['csr_ind_b2'] + data['csr_val_b2'] + data['csr_sha2'] dict_t = [\ test_df.input1_src, test_df.input2_src] + \ test_df.mat1.rect + test_df.mat1.rot + \ test_df.mat2.rect + test_df.mat2.rot + \ test_df.mat1.csr_ind + test_df.mat1.csr_val + \ test_df.mat1.csr_ind_r + test_df.mat1.csr_val_r + \ test_df.mat1.csr_ind_g + test_df.mat1.csr_val_g + \ test_df.mat1.csr_ind_b + test_df.mat1.csr_val_b + test_df.mat1.csr_sha + \ test_df.mat2.csr_ind + test_df.mat2.csr_val + \ test_df.mat2.csr_ind_r + test_df.mat2.csr_val_r + \ test_df.mat2.csr_ind_g + test_df.mat2.csr_val_g + \ test_df.mat2.csr_ind_b + test_df.mat2.csr_val_b + test_df.mat2.csr_sha if FLAGS['loss_constant_term_weight'] > 0: test_s, enhance_test_img1, enhance_test_img1_inv, enhance_test_img2, enhance_test_img2_inv = sess.run([netG_test_summary, netG_test_output1_crop, netG_test_output1_inv_crop, netG_test_output2_crop, netG_test_output2_inv_crop], feed_dict={t:d for t, d in zip(dict_t, dict_d)}) enhance_test_img1 = safe_casting(enhance_test_img1 * tf.as_dtype(FLAGS['data_input_dtype']).max, FLAGS['data_input_dtype']) enhance_test_img1_inv = safe_casting(enhance_test_img1_inv * tf.as_dtype(FLAGS['data_input_dtype']).max, FLAGS['data_input_dtype']) enhance_test_img2 = safe_casting(enhance_test_img2 * tf.as_dtype(FLAGS['data_input_dtype']).max, FLAGS['data_input_dtype']) enhance_test_img2_inv = safe_casting(enhance_test_img2_inv * tf.as_dtype(FLAGS['data_input_dtype']).max, FLAGS['data_input_dtype']) if is_training: cv2.imwrite(FLAGS['folder_test_img'] + 'A-' + test_image_name_list[i] + '-' + now_epoch + '.tif', enhance_test_img1) cv2.imwrite(FLAGS['folder_test_img'] + 'A-' + test_image_name_list[i] + '-i' + now_epoch +'.tif', enhance_test_img1_inv) cv2.imwrite(FLAGS['folder_test_img'] + 'B-' + test_image_name_list[i] + '-' + now_epoch + '.tif', enhance_test_img2) cv2.imwrite(FLAGS['folder_test_img'] + 'B-' + test_image_name_list[i] + '-i' + now_epoch +'.tif', enhance_test_img2_inv) else: assert False, 'not yet' cv2.imwrite(FLAGS['folder_test_img'] + test_image_name_list[i] + '.tif', enhance_test_img) cv2.imwrite(FLAGS['folder_test_img'] + test_image_name_list[i] + '-i' + '.tif', enhance_test_img_inv) else: assert False, 'not yet' test_s, enhance_test_img = sess.run([netG_test_summary, netG_test_output1_crop], feed_dict={t:d for t, d in zip(dict_t, dict_d)}) enhance_test_img = safe_casting(enhance_test_img * tf.as_dtype(FLAGS['data_input_dtype']).max, FLAGS['data_input_dtype']) if is_training: cv2.imwrite(FLAGS['folder_test_img'] + test_image_name_list[i] + '-' + now_epoch + '.tif', enhance_test_img) else: cv2.imwrite(FLAGS['folder_test_img'] + test_image_name_list[i] + '.tif', enhance_test_img) if FLAGS['mode_use_debug']: print("i = %d, %s" % (i, str(timer2.end()))) for j in range(len(netG_test_summary_names)): test_loss_list[j][i] = test_s[j] test_avg = [test_avg[j] + v for j, v in enumerate(test_s)] test_avg = [v / test_count for v in test_avg] best_value_history = [max(v, test_avg[i]) for i, v in enumerate(best_value_history)] if is_training: step = round(float(now_epoch)*FLAGS['process_test_log_interval_epoch']) if step >= FLAGS['process_test_drop_summary_step']: for i, v in enumerate(test_avg): summary = sess.run(test_summary_list[i], feed_dict={test_summary_placeholders[i]: v}) summary_writer.add_summary(summary, step) # weights_data = sess.run(main_net.weights) # save_weights(weights_data, main_net.parameter_names, FLAGS['weight_folder'], now_epoch) if is_training: write_list_to_file(FLAGS['folder_test_netG_loss'] + now_epoch + ".txt", test_loss_list[ 0], test_image_name_list[:test_count]) write_list_to_file(FLAGS['folder_test_netG_psnr1'] + now_epoch + ".txt", test_loss_list[-2], test_image_name_list[:test_count]) write_list_to_file(FLAGS['folder_test_netG_psnr2'] + now_epoch + ".txt", test_loss_list[-1], test_image_name_list[:test_count]) write_list_to_file(FLAGS['folder_train_netG_loss'] + now_epoch + ".txt", list_train_loss, train_image_name_list_input) write_list_to_file(FLAGS['folder_train_ind_input'] + now_epoch + ".txt", indices_1, train_image_name_list_input, True) write_list_to_file(FLAGS['folder_train_ind_label'] + now_epoch + ".txt", indices_2, train_image_name_list_label, True) save_model(saver, sess, FLAGS['folder_model'], now_epoch) for train_c_summary, step in train_summary_datas: summary_writer.add_summary(train_c_summary, step) info = current_time() + ", epoch = " + now_epoch for i, v in enumerate(test_avg): info = info + ", %s = %s" % (netG_test_summary_names[i], FLAGS['format_log_value'].format(v).replace(' ', '*')) info = info + ", (max:" for i, v in enumerate(best_value_history): info = info + "%s" % FLAGS['format_log_value'].format(v).replace(' ', '*') info = info + (")" if i == len(best_value_history) - 1 else ", ") info = info + ", tt = " + str(timer.end()).split(".")[0][2:] print(info) return best_value_history, []
def __setstate__(self, state): self._dtype = tf.as_dtype(state)
def __setstate__(self, state): self._dtype = tf.as_dtype(state['dtype']) self._is_categorical = state['is_categorical'] self._min_value = state['min_value'] self._max_value = state['max_value'] self._vocabulary_file = state['vocabulary_file']
def generator(self, labels_in): assert self.resolution == 2**self.resolution_log2 and self.resolution >= 4 def nf(stage): return min(int(self.fmap_base / (2.0 ** (stage * self.fmap_decay))), self.fmap_max) # First Layer from Image: A x A x 3 ==> A x A x Channels(A) # -------------------------------------------------------------------------- def fromrgb(x, res): with tf.variable_scope('Gen_Enc_FromRGB_lod%d' % (self.resolution_log2 - res)): return leaky_relu(apply_bias(conv2d(x, fmaps=nf(res-1), kernel=1, cf = self.channel_first), cf = self.channel_first)) # Building Blocks Encoder: Input --> Convolution (+Bias) --> Batchnormalisation # --> Activation function --> Downsample by factor 2 # -------------------------------------------------------------------------- def block_e(x,res): with tf.variable_scope('Gen_Enc%dx%d' % (2**res, 2**res)): with tf.variable_scope('Conv'): x = leaky_relu(batchnorm(apply_bias(conv2d(x, fmaps = nf(res-2), kernel = 3, cf = self.channel_first), cf = self.channel_first), cf = self.channel_first)) x = downscale2d(x, cf = self.channel_first) return x # ========================================================================= # Encoder # ========================================================================= if self.structure == 'linear': skip = [] img = labels_in x = fromrgb(img, self.resolution_log2) #print(x.shape) for res in range(self.resolution_log2, 4, -1): lod = self.resolution_log2 - res x = block_e(x, res) #print(x.shape) img = downscale2d(img, cf = self.channel_first) y = fromrgb(img, res - 1) with tf.variable_scope('Gen_Enc_Grow_lod%d' % lod): x = lerp_clip(x, y, self.lod_in - lod) skip.append(x) # #print('Encoder after', 2**res, "conv", x.shape) for res in range(4,0,-1): lod = self.resolution_log2 - res x = block_e(x, res) #print(x.shape) if res >= 2: skip.append(x) #print("Encoder Output", x.shape) combo_out = x # ======================================================================== # Decoder # ======================================================================== # Last Layer to Image: A x A x Channels(A) ==> A x A x 3 # -------------------------------------------------------------------------- def torgb(x, res): # res = 2..resolution_log2 lod = self.resolution_log2 - res with tf.variable_scope('Gen_Dec_ToRGB_lod%d' % lod): return apply_bias(conv2d(x, fmaps=self.num_channels, kernel=1, cf = self.channel_first), cf = self.channel_first) # Building Blocks Encoder: Input --> Upsampling by factor 2 --> Convolution (+Bias) # --> Batchnormalisation --> Activation # -------------------------------------------------------------------------- def block_d(x,res): layers = [] with tf.variable_scope('Gen_Dec_%dx%d' % (2**res, 2**res)): x = upscale2d(x, cf = self.channel_first) with tf.variable_scope('Conv'): x = leaky_relu(batchnorm(apply_bias(conv2d(x, fmaps = nf(res-1), kernel = 3, cf = self.channel_first), cf = self.channel_first), cf = self.channel_first)) return x # Growing the Decoder # --------------------------------------------------------------------------- if self.structure == 'linear': #print('Decode Input:', x.shape) x = combo_out #print('start decoder',x.shape) x = block_d(x, 1) # 1x1x512 ==> 2x2x512 #print(x.shape) x = tf.concat([x,skip[-1]], axis=self.conc_axis) # concat ==> 2x2x1024 x = block_d(x, 2) # 2x2x1024 ==> 4x4x512 #print(x.shape) x = tf.concat([x,skip[-2]], axis=self.conc_axis) x = block_d(x, 3) # 4x4x1024 ==> 8x8x512 #print(x.shape) x = tf.concat([x,skip[-3]], axis=self.conc_axis) x = block_d(x, 4) # 8x8x1024 ==> 16x16x512 #print(x.shape) #print("-----") images_out = torgb(x,4) # Extracted Output layer 16x16 #print("Image Out:", images_out.shape) x = tf.concat([x,skip[-4]], axis=self.conc_axis) # concat ==> 16x16x1024 #print('Decode after const:', x.shape) for res in range(5,self.resolution_log2+1): lod = self.resolution_log2 - res x = block_d(x,res) #print(x.shape) img = torgb(x,res) if res < self.resolution_log2: x = tf.concat([x,skip[-res]], axis = self.conc_axis) images_out = upscale2d(images_out, cf = self.channel_first) with tf.variable_scope('Gen_Dec_Grow_lod%d' % lod): images_out = lerp_clip(img, images_out, self.lod_in - lod) #print("Images Out:", images_out.shape) #print('Decode res:', 2**res,'shape x:', x.shape, "shape img out:", images_out.shape) #print('output',images_out.shape) assert images_out.dtype == tf.as_dtype(self.dtype) images_out = tf.identity(images_out, name='images_out') return images_out
def optimistic_restore(session, save_file, ignore_vars=None, verbose=False, ignore_incompatible_shapes=False): """This function tries to restore all variables in the save file. This function ignores variables that do not exist or have incompatible shape. Raises TypeError if the there is a type mismatch for compatible shapes. session: tf.Session The tf session save_file: str Path to the checkpoint without the .index, .meta or .data extensions. ignore_vars: list, tuple or set of str These variables will be ignored. verbose: bool If True prints which variables will be restored ignore_incompatible_shapes: bool If True ignores variables with incompatible shapes. If False raises a runtime error f shapes are incompatible. """ print(save_file) def vprint(*args, **kwargs): if verbose: print(*args, flush=True, **kwargs) # def dbg(*args, **kwargs): # print(*args, flush=True, **kwargs) def dbg(*args, **kwargs): pass if ignore_vars is None: ignore_vars = [] reader = tf.train.NewCheckpointReader(save_file) saved_shapes = reader.get_variable_to_shape_map() var_names = sorted([ (var.name, var.dtype, var.name.split(':')[0]) for var in tf.global_variables() if not var.name.split(':')[0] in ignore_vars and 'Adam' not in var.name ]) restore_vars = [] dbg(saved_shapes) for v in tf.global_variables(): dbg(v) nonfinite_values = False with tf.variable_scope('', reuse=True): for var_name, var_dtype, saved_var_name in var_names: dbg(var_name, var_dtype, saved_var_name, end='') curr_var = tf.get_variable(saved_var_name, dtype=var_dtype) var_shape = curr_var.get_shape().as_list() try: if var_shape == saved_shapes[saved_var_name]: dbg(' shape OK') tmp = reader.get_tensor(saved_var_name) dbg(tmp.dtype) # check if there are nonfinite values in the tensor if not np.all(np.isfinite(tmp)): nonfinite_values = True print('{0} contains nonfinite values!'.format( saved_var_name), flush=True) if isinstance(tmp, np.ndarray): saved_dtype = tf.as_dtype(tmp.dtype) else: print(var_name) saved_dtype = tf.as_dtype(type(tmp)) dbg(saved_dtype, var_dtype, saved_dtype.is_compatible_with(var_dtype)) if not saved_dtype.is_compatible_with(var_dtype): raise TypeError( 'types are not compatible for {0}: saved type {1}, variable type {2}.' .format(saved_var_name, saved_dtype.name, var_dtype.name)) vprint('restoring ', saved_var_name) restore_vars.append(curr_var) else: vprint('not restoring', saved_var_name, 'incompatible shape:', var_shape, 'vs', saved_shapes[saved_var_name]) if not ignore_incompatible_shapes: raise RuntimeError( 'failed to restore "{0}" because of incompatible shapes: var: {1} vs saved: {2} ' .format(saved_var_name, var_shape, saved_shapes[saved_var_name])) except KeyError: print('not restoring {}', saved_var_name) if nonfinite_values: raise RuntimeError( '"{0}" contains nonfinite values!'.format(save_file)) dbg('-1-') saver = tf.train.Saver( var_list=restore_vars, restore_sequentially=True, ) dbg('-2-') saver.restore(session, save_file) dbg('-3-')
def create_train_op(named_scalars, grads_and_vars, optimizer, global_step, params): tf.get_variable_scope().set_dtype(tf.as_dtype(dtype.floatx())) gradients = [item[0] for item in grads_and_vars] variables = [item[1] for item in grads_and_vars] if params.update_cycle == 1: zero_variables_op = tf.no_op("zero_variables") collect_op = tf.no_op("collect_op") else: named_vars = {} for name in named_scalars: named_var = tf.Variable(tf.zeros([], dtype=tf.float32), name="{}/CTrainOpReplica".format(name), trainable=False) named_vars[name] = named_var count_var = tf.Variable(tf.zeros([], dtype=tf.as_dtype(dtype.floatx())), name="count/CTrainOpReplica", trainable=False) slot_variables = _replicate_variables(variables, suffix='CTrainOpReplica') zero_variables_op = _zero_variables( slot_variables + [count_var] + list(named_vars.values())) collect_ops = [] # collect gradients collect_grads_op = _collect_gradients(gradients, slot_variables) collect_ops.append(collect_grads_op) # collect other scalars for name in named_scalars: scalar = named_scalars[name] named_var = named_vars[name] collect_op = tf.assign_add(named_var, scalar) collect_ops.append(collect_op) # collect counting variable collect_count_op = tf.assign_add(count_var, 1.0) collect_ops.append(collect_count_op) collect_op = tf.group(*collect_ops, name="collect_op") scale = 1.0 / (tf.cast(count_var, tf.float32) + 1.0) gradients = [scale * (g + s) for (g, s) in zip(gradients, slot_variables)] for name in named_scalars: named_scalars[name] = scale * ( named_scalars[name] + named_vars[name]) grand_norm = tf.global_norm(gradients) param_norm = tf.global_norm(variables) # Gradient clipping if isinstance(params.clip_grad_norm or None, float): gradients, _ = tf.clip_by_global_norm(gradients, params.clip_grad_norm, use_norm=grand_norm) # Update variables grads_and_vars = list(zip(gradients, variables)) train_op = optimizer.apply_gradients(grads_and_vars, global_step) ops = { "zero_op": zero_variables_op, "collect_op": collect_op, "train_op": train_op } # apply ema if params.ema_decay > 0.: tf.logging.info('Using Exp Moving Average to train the model with decay {}.'.format(params.ema_decay)) ema = tf.train.ExponentialMovingAverage(decay=params.ema_decay, num_updates=global_step) ema_op = ema.apply(variables) with tf.control_dependencies([ops['train_op']]): ops['train_op'] = tf.group(ema_op) bck_vars = _replicate_variables(variables, suffix="CTrainOpBackUpReplica") ops['ema_backup_op'] = tf.group(*(tf.assign(bck, var.read_value()) for bck, var in zip(bck_vars, variables))) ops['ema_restore_op'] = tf.group(*(tf.assign(var, bck.read_value()) for bck, var in zip(bck_vars, variables))) ops['ema_assign_op'] = tf.group(*(tf.assign(var, ema.average(var).read_value()) for var in variables)) ret = named_scalars ret.update({ "gradient_norm": grand_norm, "parameter_norm": param_norm, }) return ret, ops
def create_tf_placeholder(self): """Convenience method that produces a placeholder of the type and shape defined by the port. Returns: a placeholder of same type, shape and name. """ return tf.placeholder(tf.as_dtype(self.dtype), self.shape, self.name)
def _get_dtype(dtype): if dtype is None: dtype = backend.floatx() return tf.as_dtype(dtype)
def G_synthesis( dlatents_in, # Input: Disentangled latents (W) [minibatch, num_layers, dlatent_size]. dlatent_size=512, # Disentangled latent (W) dimensionality. num_channels=3, # Number of output color channels. resolution=1024, # Output resolution. fmap_base=8192, # Overall multiplier for the number of feature maps. fmap_decay=1.0, # log2 feature map reduction when doubling the resolution. fmap_max=512, # Maximum number of feature maps in any layer. use_styles=True, # Enable style inputs? const_input_layer=True, # First layer is a learned constant? use_noise=True, # Enable noise inputs? randomize_noise=True, # True = randomize noise inputs every time (non-deterministic), False = read noise inputs from variables. nonlinearity='lrelu', # Activation function: 'relu', 'lrelu' use_wscale=True, # Enable equalized learning rate? use_pixel_norm=False, # Enable pixelwise feature vector normalization? use_instance_norm=True, # Enable instance normalization? dtype='float32', # Data type to use for activations and outputs. fused_scale='auto', # True = fused convolution + scaling, False = separate ops, 'auto' = decide automatically. blur_filter=[ 1, 2, 1 ], # Low-pass filter to apply when resampling activations. None = no filtering. structure='auto', # 'fixed' = no progressive growing, 'linear' = human-readable, 'recursive' = efficient, 'auto' = select automatically. is_template_graph=False, # True = template graph constructed by the Network class, False = actual evaluation. force_clean_graph=False, # True = construct a clean graph that looks nice in TensorBoard, False = default behavior. **_kwargs): # Ignore unrecognized keyword args. resolution_log2 = int(np.log2(resolution)) assert resolution == 2**resolution_log2 and resolution >= 4 def nf(stage): return min(int(fmap_base / (2.0**(stage * fmap_decay))), fmap_max) def blur(x): return blur2d(x, blur_filter) if blur_filter else x if is_template_graph: force_clean_graph = True if force_clean_graph: randomize_noise = False if structure == 'auto': structure = 'linear' if force_clean_graph else 'recursive' act, gain = { 'relu': (tf.nn.relu, np.sqrt(2)), 'lrelu': (leaky_relu, np.sqrt(2)) }[nonlinearity] num_layers = resolution_log2 * 2 - 2 num_styles = num_layers if use_styles else 1 images_out = None # Primary inputs. dlatents_in.set_shape([None, num_styles, dlatent_size]) dlatents_in = tf.cast(dlatents_in, dtype) lod_in = tf.cast( tf.get_variable('lod', initializer=np.float32(0), trainable=False), dtype) # Noise inputs. noise_inputs = [] if use_noise: for layer_idx in range(num_layers): res = layer_idx // 2 + 2 shape = [1, use_noise, 2**res, 2**res] noise_inputs.append( tf.get_variable('noise%d' % layer_idx, shape=shape, initializer=tf.initializers.random_normal(), trainable=False)) # Things to do at the end of each layer. def layer_epilogue(x, layer_idx): if use_noise: x = apply_noise(x, noise_inputs[layer_idx], randomize_noise=randomize_noise) x = apply_bias(x) x = act(x) if use_pixel_norm: x = pixel_norm(x) if use_instance_norm: x = instance_norm(x) if use_styles: x = style_mod(x, dlatents_in[:, layer_idx], use_wscale=use_wscale) return x # Early layers. with tf.variable_scope('4x4'): if const_input_layer: with tf.variable_scope('Const'): x = tf.get_variable('const', shape=[1, nf(1), 4, 4], initializer=tf.initializers.ones()) x = layer_epilogue( tf.tile(tf.cast(x, dtype), [tf.shape(dlatents_in)[0], 1, 1, 1]), 0) else: with tf.variable_scope('Dense'): x = dense( dlatents_in[:, 0], fmaps=nf(1) * 16, gain=gain / 4, use_wscale=use_wscale ) # tweak gain to match the official implementation of Progressing GAN x = layer_epilogue(tf.reshape(x, [-1, nf(1), 4, 4]), 0) with tf.variable_scope('Conv'): x = layer_epilogue( conv2d(x, fmaps=nf(1), kernel=3, gain=gain, use_wscale=use_wscale), 1) # Building blocks for remaining layers. def block(res, x): # res = 3..resolution_log2 with tf.variable_scope('%dx%d' % (2**res, 2**res)): with tf.variable_scope('Conv0_up'): x = layer_epilogue( blur( upscale2d_conv2d(x, fmaps=nf(res - 1), kernel=3, gain=gain, use_wscale=use_wscale, fused_scale=fused_scale)), res * 2 - 4) with tf.variable_scope('Conv1'): x = layer_epilogue( conv2d(x, fmaps=nf(res - 1), kernel=3, gain=gain, use_wscale=use_wscale), res * 2 - 3) return x def torgb(res, x): # res = 2..resolution_log2 lod = resolution_log2 - res with tf.variable_scope('ToRGB_lod%d' % lod): return apply_bias( conv2d(x, fmaps=num_channels, kernel=1, gain=1, use_wscale=use_wscale)) # Fixed structure: simple and efficient, but does not support progressive growing. if structure == 'fixed': for res in range(3, resolution_log2 + 1): x = block(res, x) images_out = torgb(resolution_log2, x) # Linear structure: simple but inefficient. if structure == 'linear': images_out = torgb(2, x) for res in range(3, resolution_log2 + 1): lod = resolution_log2 - res x = block(res, x) img = torgb(res, x) images_out = upscale2d(images_out) with tf.variable_scope('Grow_lod%d' % lod): images_out = tflib.lerp_clip(img, images_out, lod_in - lod) # Recursive structure: complex but efficient. if structure == 'recursive': def cset(cur_lambda, new_cond, new_lambda): return lambda: tf.cond(new_cond, new_lambda, cur_lambda) def grow(x, res, lod): y = block(res, x) img = lambda: upscale2d(torgb(res, y), 2**lod) img = cset( img, (lod_in > lod), lambda: upscale2d( tflib.lerp(torgb(res, y), upscale2d(torgb(res - 1, x)), lod_in - lod), 2**lod)) if lod > 0: img = cset(img, (lod_in < lod), lambda: grow(y, res + 1, lod - 1)) return img() images_out = grow(x, 3, resolution_log2 - 3) assert images_out.dtype == tf.as_dtype(dtype) return tf.identity(images_out, name='images_out')
def _get_tf_dtype(space): if isinstance(space, gym.spaces.Discrete): return tf.int32 if isinstance(space, gym.spaces.Box): return tf.as_dtype(space.dtype) raise NotImplementedError()
def G_paper( latents_in, # First input: Latent vectors [minibatch, latent_size]. labels_in, # Second input: Labels [minibatch, label_size]. probimages_in, # Third input: probimages [minibatch, 1, resolution, resolution]. num_channels=1, # Number of output color channels. Overridden based on dataset. resolution=64, # Output resolution. Overridden based on dataset. label_size=0, # Dimensionality of the labels, 0 if no labels. Overridden based on dataset. fmap_base=2048, # Overall multiplier for the number of feature maps. fmap_decay=1.0, # log2 feature map reduction when doubling the resolution. fmap_max=128, # Maximum number of feature maps in any layer. latent_size=None, # Dimensionality of the latent vectors. None = min(fmap_base, fmap_max). normalize_latents=False, # Normalize latent vectors before feeding them to the network? use_wscale=True, # Enable equalized learning rate? use_pixelnorm=True, # Enable pixelwise feature vector normalization? pixelnorm_epsilon=1e-8, # Constant epsilon for pixelwise feature vector normalization. use_leakyrelu=True, # True = leaky ReLU, False = ReLU. dtype='float32', # Data type to use for activations and outputs. fused_scale=False, # True = use fused upscale2d + conv2d, False = separate upscale2d layers. structure=None, # 'linear' = human-readable, 'recursive' = efficient, None = select automatically. is_template_graph=False, # True = template graph constructed by the Network class, False = actual evaluation. labels_augment=False, #True, # Augment labels or not. labels_augment_times=8, # Augment labels by how many times. e.g., [0.3, 0.9]---> [0.3, 0.3, 0.3, 0.9, 0.9, 0.9], so that more attention would be put on labels. prob_conv_channels=16, **kwargs): # Ignore unrecognized keyword args. resolution_log2 = int(np.log2(resolution)) assert resolution == 2**resolution_log2 and resolution >= 4 def nf(stage): return min(int(fmap_base / (2.0**(stage * fmap_decay))), fmap_max) def PN(x): return pixel_norm(x, epsilon=pixelnorm_epsilon) if use_pixelnorm else x if latent_size is None: latent_size = nf(0) if structure is None: structure = 'linear' if is_template_graph else 'recursive' act = leaky_relu if use_leakyrelu else tf.nn.relu latents_in.set_shape([None, latent_size]) labels_in.set_shape([None, label_size]) probimages_in.set_shape([None, 1, resolution, resolution]) probimages_in = tf.cast(probimages_in, tf.float32) if labels_augment: labels_in = labels_augment_func(labels_in, label_size, labels_augment_times) combo_in = tf.cast(tf.concat([latents_in, labels_in], axis=1), dtype) lod_in = tf.cast( tf.get_variable('lod', initializer=np.float32(0.0), trainable=False), dtype) # Building blocks. def block(x, prob, res): # res = 2..resolution_log2 with tf.variable_scope('%dx%d' % (2**res, 2**res)): if res == 2: # 4x4 if normalize_latents: x = pixel_norm(x, epsilon=pixelnorm_epsilon) with tf.variable_scope('Dense'): x = dense( x, fmaps=nf(res - 1) * 16, gain=np.sqrt(2) / 4, use_wscale=use_wscale ) # override gain to match the original Theano implementation x = tf.reshape(x, [-1, nf(res - 1), 4, 4]) x = PN(act(apply_bias(x))) with tf.variable_scope('Add_Prob'): prob_downscaled = downscale2d(prob, factor=int(resolution / (2**res))) prob_downscaled_conv = apply_bias( conv2d(prob_downscaled, fmaps=prob_conv_channels, kernel=1, gain=1, use_wscale=use_wscale)) x = tf.concat([x, prob_downscaled_conv], axis=1) with tf.variable_scope('Conv'): x = PN( act( apply_bias( conv2d(x, fmaps=nf(res - 1), kernel=3, use_wscale=use_wscale)))) else: # 8x8 and up x = upscale2d(x) with tf.variable_scope('Add_Prob'): prob_downscaled = downscale2d(prob, factor=int(resolution / (2**res))) prob_downscaled_conv = apply_bias( conv2d(prob_downscaled, fmaps=prob_conv_channels, kernel=1, gain=1, use_wscale=use_wscale)) x = tf.concat([x, prob_downscaled_conv], axis=1) with tf.variable_scope('Conv0'): x = PN( act( apply_bias( conv2d(x, fmaps=nf(res - 1), kernel=3, use_wscale=use_wscale)))) with tf.variable_scope('Conv1'): x = PN( act( apply_bias( conv2d(x, fmaps=nf(res - 1), kernel=3, use_wscale=use_wscale)))) return x def torgb(x, res): # res = 2..resolution_log2 lod = resolution_log2 - res with tf.variable_scope('ToRGB_lod%d' % lod): return apply_bias( conv2d(x, fmaps=num_channels, kernel=1, gain=1, use_wscale=use_wscale)) # Linear structure: simple but inefficient. if structure == 'linear': x = block(combo_in, probimages_in, 2) images_out = torgb(x, 2) for res in range(3, resolution_log2 + 1): lod = resolution_log2 - res x = block(x, probimages_in, res) img = torgb(x, res) images_out = upscale2d(images_out) with tf.variable_scope('Grow_lod%d' % lod): images_out = lerp_clip(img, images_out, lod_in - lod) # Recursive structure: complex but efficient. if structure == 'recursive': def grow(x, prob, res, lod): y = block(x, prob, res) img = lambda: upscale2d(torgb(y, res), 2**lod) if res > 2: img = cset( img, (lod_in > lod), lambda: upscale2d( lerp(torgb(y, res), upscale2d(torgb(x, res - 1)), lod_in - lod), 2**lod)) if lod > 0: img = cset(img, (lod_in < lod), lambda: grow(y, prob, res + 1, lod - 1)) return img() images_out = grow(combo_in, probimages_in, 2, resolution_log2 - 2) assert images_out.dtype == tf.as_dtype(dtype) images_out = tf.identity(images_out, name='images_out') return images_out
def __init__(self, dtype): self._dtype = tf.as_dtype(dtype)
def D_stylegan2( images_in, # First input: Images [minibatch, channel, height, width]. labels_in, # Second input: Labels [minibatch, label_size]. num_channels=3, # Number of input color channels. Overridden based on dataset. resolution=256, # Input resolution. Overridden based on dataset. label_size=127, # Dimensionality of the labels, 0 if no labels. Overridden based on dataset. fmap_base=16 << 10, # Overall multiplier for the number of feature maps. fmap_decay=1.0, # log2 feature map reduction when doubling the resolution. fmap_min=1, # Minimum number of feature maps in any layer. fmap_max=512, # Maximum number of feature maps in any layer. architecture='resnet', # Architecture: 'orig', 'skip', 'resnet'. nonlinearity='lrelu', # Activation function: 'relu', 'lrelu', etc. mbstd_group_size=4, # Group size for the minibatch standard deviation layer, 0 = disable. mbstd_num_features=1, # Number of features for the minibatch standard deviation layer. dtype='float32', # Data type to use for activations and outputs. resample_kernel=[ 1, 3, 3, 1 ], # Low-pass filter to apply when resampling activations. None = no filtering. dlabel_size=32, **_kwargs): # Ignore unrecognized keyword args. resolution_log2 = int(np.log2(resolution)) assert resolution == 2**resolution_log2 and resolution >= 4 def nf(stage): return np.clip(int(fmap_base / (2.0**(stage * fmap_decay))), fmap_min, fmap_max) assert architecture in ['orig', 'skip', 'resnet'] act = nonlinearity dlabel = D_mapping_label(labels_in=labels_in, label_size=label_size, dlabel_size=dlabel_size) images_in.set_shape([None, num_channels, resolution, resolution]) labels_in.set_shape([None, label_size]) dlabel.set_shape([None, dlabel_size]) images_in = tf.cast(images_in, dtype) labels_in = tf.cast(labels_in, dtype) dlabel = tf.cast(dlabel, dtype) # Building blocks for main layers. def fromrgb(x, y, res): # res = 2..resolution_log2 with tf.variable_scope('FromRGB'): t = apply_bias_act(conv2d_layer(y, fmaps=nf(res - 1), kernel=1), act=act) return t if x is None else x + t def downsample(y): with tf.variable_scope('Downsample'): return downsample_2d(y, k=resample_kernel) def block(x, res): # res = 2..resolution_log2 t = x v_x_upsample = tf.constant(0) with tf.variable_scope('Conv0'): x = apply_bias_act(conv2d_layer(x, fmaps=nf(res - 1), kernel=3), act=act) with tf.variable_scope('Conv1_down'): x = apply_bias_act(conv2d_layer(x, fmaps=nf(res - 2), kernel=3, down=True, resample_kernel=resample_kernel), act=act) if 3 < res < 8: with tf.variable_scope('Downsample'): x_downsample = downsample_2d(x) height = x_downsample.shape[2] width = x_downsample.shape[3] c_reduced = 1 label_mapping_fmaps = 16 with tf.variable_scope('F_Attention'): f_x = conv2d_layer(x_downsample, fmaps=1, kernel=1) f_x = tf.reshape(f_x, [-1, height * width]) with tf.variable_scope('Label_F_Attention'): label_f = dense_layer(dlabel, fmaps=label_mapping_fmaps) + 1 with tf.variable_scope('F_concat_Attention'): f_x_s = dense_layer(tf.concat([f_x, label_f], axis=-1), fmaps=c_reduced * height * width) f_x_s = tf.reshape(f_x_s, [-1, c_reduced, height * width]) f_x_s = tf.transpose(f_x_s, perm=[0, 2, 1]) with tf.variable_scope('G_Attention'): g_x = conv2d_layer(x_downsample, fmaps=1, kernel=1) g_x = tf.reshape(g_x, [-1, height * width]) with tf.variable_scope('Label_G_Attention'): label_g = dense_layer(dlabel, fmaps=label_mapping_fmaps) + 1 with tf.variable_scope('G_concat_Attention'): g_x_s = dense_layer(tf.concat([g_x, label_g], axis=-1), fmaps=c_reduced * height * width) g_x_s = tf.reshape(g_x_s, [-1, c_reduced, height * width]) with tf.variable_scope('H_Attention'): h_x = conv2d_layer(x_downsample, fmaps=c_reduced, kernel=1) h_x = tf.reshape(h_x, [-1, c_reduced, height * width]) f_g_multiply = tf.matmul(f_x_s, g_x_s) attention_map = tf.nn.softmax(f_g_multiply, axis=-1) attention_map_h_multiply = tf.matmul( h_x, tf.transpose(attention_map, [0, 2, 1])) attention_map_h_multiply_reshape = tf.reshape( attention_map_h_multiply, [-1, c_reduced, height, width]) with tf.variable_scope('V_Attention'): v_x = conv2d_layer(attention_map_h_multiply_reshape, fmaps=x_downsample.shape[1], kernel=1) with tf.variable_scope('Upsample'): v_x_upsample = upsample_2d(v_x) with tf.variable_scope('Gamma_Attention'): gamma = tf.get_variable(shape=[], initializer=tf.initializers.zeros(), name='attention_gamma') x = x + v_x_upsample * gamma if architecture == 'resnet': with tf.variable_scope('Skip'): t = conv2d_layer(t, fmaps=nf(res - 2), kernel=1, down=True, resample_kernel=resample_kernel) x = (x + t) * (1 / np.sqrt(2)) return x, v_x_upsample # Main layers. x = None y = images_in for res in range(resolution_log2, 2, -1): with tf.variable_scope('%dx%d' % (2**res, 2**res)): if architecture == 'skip' or res == resolution_log2: x = fromrgb(x, y, res) x, attention_map = block(x, res) if res == 3: fmap_out = x current_attention_map = attention_map if architecture == 'skip': y = downsample(y) # Final layers. with tf.variable_scope('4x4'): if architecture == 'skip': x = fromrgb(x, y, 2) if mbstd_group_size > 1: with tf.variable_scope('MinibatchStddev'): x = minibatch_stddev_layer(x, mbstd_group_size, mbstd_num_features) with tf.variable_scope('Conv'): x = apply_bias_act(conv2d_layer(x, fmaps=nf(1), kernel=3), act=act) with tf.variable_scope('Dense0'): x = apply_bias_act(dense_layer(x, fmaps=nf(0)), act=act) # Output layer with label conditioning from "Which Training Methods for GANs do actually Converge?" with tf.variable_scope('Output'): x = apply_bias_act(dense_layer(x, fmaps=max(labels_in.shape[1], 1))) return x, fmap_out, current_attention_map if labels_in.shape[1] > 0: x = tf.reduce_sum(x * labels_in, axis=1, keepdims=True) scores_out = x # Output. assert scores_out.dtype == tf.as_dtype(dtype) scores_out = tf.identity(scores_out, name='scores_out') return scores_out #----------------------------------------------------------------------------
def D_basic( images_in, # First input: Images [minibatch, channel, height, width]. labels_in, # Second input: Labels [minibatch, label_size]. num_channels=1, # Number of input color channels. Overridden based on dataset. resolution=32, # Input resolution. Overridden based on dataset. label_size=0, # Dimensionality of the labels, 0 if no labels. Overridden based on dataset. aux_output_size=0, # Auxilliary output fmap_base=8192, # Overall multiplier for the number of feature maps. fmap_decay=1.0, # log2 feature map reduction when doubling the resolution. fmap_max=512, # Maximum number of feature maps in any layer. nonlinearity='lrelu', # Activation function: 'relu', 'lrelu', use_wscale=True, # Enable equalized learning rate? mbstd_group_size=4, # Group size for the minibatch standard deviation layer, 0 = disable. mbstd_num_features=1, # Number of features for the minibatch standard deviation layer. dtype='float32', # Data type to use for activations and outputs. fused_scale='auto', # True = fused convolution + scaling, False = separate ops, 'auto' = decide automatically. blur_filter=[ 1, 2, 1 ], # Low-pass filter to apply when resampling activations. None = no filtering. structure='auto', # 'fixed' = no progressive growing, 'linear' = human-readable, 'recursive' = efficient, 'auto' = select automatically. is_template_graph=False, # True = template graph constructed by the Network class, False = actual evaluation. **_kwargs): # Ignore unrecognized keyword args. resolution_log2 = int(np.log2(resolution)) assert resolution == 2**resolution_log2 and resolution >= 4 assert label_size == 0 or aux_output_size == 0 def nf(stage): return min(int(fmap_base / (2.0**(stage * fmap_decay))), fmap_max) def blur(x): return blur2d(x, blur_filter) if blur_filter else x if structure == 'auto': structure = 'linear' if is_template_graph else 'recursive' act, gain = { 'relu': (tf.nn.relu, np.sqrt(2)), 'lrelu': (leaky_relu, np.sqrt(2)) }[nonlinearity] images_in.set_shape([None, num_channels, resolution, resolution]) labels_in.set_shape([None, label_size]) images_in = tf.cast(images_in, dtype) labels_in = tf.cast(labels_in, dtype) lod_in = tf.cast( tf.get_variable('lod', initializer=np.float32(0.0), trainable=False), dtype) scores_out = None # Building blocks. def fromrgb(x, res): # res = 2..resolution_log2 with tf.variable_scope('FromRGB_lod%d' % (resolution_log2 - res)): return act( apply_bias( conv2d(x, fmaps=nf(res - 1), kernel=1, gain=gain, use_wscale=use_wscale))) def block(x, res): # res = 2..resolution_log2 with tf.variable_scope('%dx%d' % (2**res, 2**res)): if res >= 3: # 8x8 and up with tf.variable_scope('Conv0'): x = act( apply_bias( conv2d(x, fmaps=nf(res - 1), kernel=3, gain=gain, use_wscale=use_wscale))) with tf.variable_scope('Conv1_down'): x = act( apply_bias( conv2d_downscale2d(blur(x), fmaps=nf(res - 2), kernel=3, gain=gain, use_wscale=use_wscale, fused_scale=fused_scale))) else: # 4x4 if mbstd_group_size > 1: x = minibatch_stddev_layer(x, mbstd_group_size, mbstd_num_features) with tf.variable_scope('Conv'): x = act( apply_bias( conv2d(x, fmaps=nf(res - 1), kernel=3, gain=gain, use_wscale=use_wscale))) with tf.variable_scope('Dense0'): x = act( apply_bias( dense(x, fmaps=nf(res - 2), gain=gain, use_wscale=use_wscale))) with tf.variable_scope('Dense1'): x = apply_bias( dense(x, fmaps=max(label_size + aux_output_size, 1), gain=1, use_wscale=use_wscale)) return x # Fixed structure: simple and efficient, but does not support progressive growing. if structure == 'fixed': x = fromrgb(images_in, resolution_log2) for res in range(resolution_log2, 2, -1): x = block(x, res) scores_out = block(x, 2) # Linear structure: simple but inefficient. if structure == 'linear': img = images_in x = fromrgb(img, resolution_log2) for res in range(resolution_log2, 2, -1): lod = resolution_log2 - res x = block(x, res) img = downscale2d(img) y = fromrgb(img, res - 1) with tf.variable_scope('Grow_lod%d' % lod): x = tflib.lerp_clip(x, y, lod_in - lod) scores_out = block(x, 2) # Recursive structure: complex but efficient. if structure == 'recursive': def cset(cur_lambda, new_cond, new_lambda): return lambda: tf.cond(new_cond, new_lambda, cur_lambda) def grow(res, lod): x = lambda: fromrgb(downscale2d(images_in, 2**lod), res) if lod > 0: x = cset(x, (lod_in < lod), lambda: grow(res + 1, lod - 1)) x = block(x(), res) y = lambda: x if res > 2: y = cset( y, (lod_in > lod), lambda: tflib.lerp( x, fromrgb(downscale2d(images_in, 2**(lod + 1)), res - 1), lod_in - lod)) return y() scores_out = grow(2, resolution_log2 - 2) # Label conditioning from "Which Training Methods for GANs do actually Converge?" if label_size: with tf.variable_scope('LabelSwitch'): scores_out = tf.reduce_sum(scores_out * labels_in, axis=1, keepdims=True) assert scores_out.dtype == tf.as_dtype(dtype) scores_out = tf.identity(scores_out, name='scores_out') return scores_out
def _setup_training(self): """Sets up graph, model and trainer.""" self._graph = tf.Graph() self._graph.add_to_collection("IS_TRAINING", True) with self._graph.as_default(): tf.set_random_seed(self.tf_random_seed) self._global_step = tf.Variable(0, name="global_step", trainable=False) # Setting up input and output placeholders. input_shape = [None] + self._data_feeder.input_shape[1:] output_shape = [None] + self._data_feeder.output_shape[1:] self._inp = tf.placeholder(tf.as_dtype( self._data_feeder.input_dtype), input_shape, name="input") self._out = tf.placeholder(tf.as_dtype( self._data_feeder.output_dtype), output_shape, name="output") # If class weights are provided, add them to the graph. # Different loss functions can use this tensor by name. if self.class_weight: self._class_weight_node = tf.constant(self.class_weight, name='class_weight') # Add histograms for X and y if they are floats. if self._data_feeder.input_dtype in (np.float32, np.float64): tf.histogram_summary("X", self._inp) if self._data_feeder.output_dtype in (np.float32, np.float64): tf.histogram_summary("y", self._out) # Create model's graph. self._model_predictions, self._model_loss = self.model_fn( self._inp, self._out) # Create summary to monitor loss tf.scalar_summary("loss", self._model_loss) # Set up a single operator to merge all the summaries self._summaries = tf.merge_all_summaries() # Create trainer and augment graph with gradients and optimizer. # Additionally creates initialization ops. self._trainer = TensorFlowTrainer(loss=self._model_loss, global_step=self._global_step, optimizer=self.optimizer, learning_rate=self.learning_rate) # Create model's saver capturing all the nodes created up until now. self._saver = tf.train.Saver(max_to_keep=self.max_to_keep, keep_checkpoint_every_n_hours=self. keep_checkpoint_every_n_hours) # Create session to run model with. self._session = tf.Session( self.tf_master, config=tf.ConfigProto( log_device_placement=self.verbose > 1, inter_op_parallelism_threads=self.num_cores, intra_op_parallelism_threads=self.num_cores))
def activation_dtype(self): return tf.as_dtype(self._hparams.activation_dtype)
import argparse import h5py import numpy as np import tensorflow as tf import nobrainer from nobrainer.io import read_mapping from nobrainer.preprocessing import binarize, preprocess_aparcaseg from nobrainer.util import iter_hdf5, input_fn_builder # Data types of labels (x) and features (y). DT_X = 'float32' DT_Y = 'int32' _DT_X_NP = np.dtype(DT_X) _DT_X_TF = tf.as_dtype(DT_X) _DT_Y_NP = np.dtype(DT_Y) _DT_Y_TF = tf.as_dtype(DT_Y) def train(params): """Train estimator.""" x_dataset = params['xdset'] y_dataset = params['ydset'] tf.logging.info('Using features dataset {x} and labels dataset {y}'.format( x=x_dataset, y=y_dataset)) with h5py.File(params['hdf5path'], mode='r') as fp: examples_per_epoch = fp[x_dataset].shape[0]
def master_dtype(self): return tf.as_dtype(self._hparams.master_dtype)
def sample_bounded_spec(spec, seed=None, outer_dims=None): """Samples uniformily the given bounded spec. Args: spec: A BoundedSpec to sample. seed: A seed used for sampling ops outer_dims: An optional `Tensor` specifying outer dimensions to add to the spec shape before sampling. Returns: A Tensor sample of the requested spec. """ minval = spec.minimum maxval = spec.maximum dtype = tf.as_dtype(spec.dtype) # To sample uint8 we will use int32 and cast later. This is needed for two # reasons: # - tf.random_uniform does not currently support uint8 # - if you want to sample [0, 255] range, there's no way to do this since # tf.random_uniform has exclusive upper bound and 255 + 1 would overflow. is_uint8 = dtype == tf.uint8 sampling_dtype = tf.int32 if is_uint8 else dtype if dtype in [tf.float64, tf.float32]: # Avoid under/over-flow as random_uniform can't sample over the full range # for these types. minval = np.maximum(dtype.min / 8, minval) maxval = np.minimum(dtype.max / 8, maxval) if outer_dims is None: outer_dims = tf.constant([], dtype=tf.int32) else: outer_dims = tf.convert_to_tensor(outer_dims, dtype=tf.int32) def _unique_vals(vals): if vals.size > 0: if vals.ndim > 0: return np.all(vals == vals[0]) return True if (minval.ndim != 0 or maxval.ndim != 0) and not (_unique_vals(minval) and _unique_vals(maxval)): # tf.random_uniform can only handle minval/maxval 0-d tensors. res = _random_uniform_int(shape=spec.shape, outer_dims=outer_dims, minval=minval, maxval=maxval, dtype=sampling_dtype, seed=seed) else: minval = minval.item(0) if minval.ndim != 0 else minval maxval = maxval.item(0) if maxval.ndim != 0 else maxval # BoundedTensorSpec are bounds inclusive. # tf.random_uniform is upper bound exclusive, +1 to fix the sampling # behavior. # However +1 will cause overflow, in such cases we use the original maxval. if sampling_dtype.is_integer and maxval < sampling_dtype.max: maxval = maxval + 1 shape = tf.convert_to_tensor(spec.shape, dtype=tf.int32) full_shape = tf.concat((outer_dims, shape), axis=0) res = tf.random.uniform(full_shape, minval=minval, maxval=maxval, dtype=sampling_dtype, seed=seed) if is_uint8: res = tf.cast(res, dtype=dtype) return res
def backward_tensor(self, y): ys = tf.maximum(y - self._lower, tf.as_dtype(settings.float_type).min) return ys + tf.log(-tf.expm1(-ys))
from cleverhansl2l import utils from cleverhansl2l.attacks.attack import Attack from cleverhansl2l.attacks.basic_iterative_method import BasicIterativeMethod from cleverhansl2l.attacks.carlini_wagner_l2 import CarliniWagnerL2 from cleverhansl2l.attacks.deep_fool import DeepFool from cleverhansl2l.attacks.elastic_net_method import ElasticNetMethod from cleverhansl2l.attacks.fast_feature_adversaries import FastFeatureAdversaries from cleverhansl2l.attacks.fast_gradient_method import FastGradientMethod, fgm, optimize_linear from cleverhansl2l.attacks.lbfgs import LBFGS from cleverhansl2l.attacks.madry_et_al import MadryEtAl from cleverhansl2l.attacks.max_confidence import MaxConfidence from cleverhansl2l.attacks.momentum_iterative_method import MomentumIterativeMethod from cleverhansl2l.attacks.noise import Noise from cleverhansl2l.attacks.projected_gradient_descent import ProjectedGradientDescent from cleverhansl2l.attacks.saliency_map_method import SaliencyMapMethod from cleverhansl2l.attacks.semantic import Semantic from cleverhansl2l.attacks.spsa import SPSA, projected_optimization from cleverhansl2l.attacks.spatial_transformation_method import SpatialTransformationMethod from cleverhansl2l.attacks.virtual_adversarial_method import VirtualAdversarialMethod, vatm from cleverhansl2l.model import Model, CallableModelWrapper from cleverhansl2l.model import wrapper_warning, wrapper_warning_logits from cleverhansl2l.compat import reduce_sum, reduce_mean from cleverhansl2l.compat import reduce_max from cleverhansl2l.compat import softmax_cross_entropy_with_logits from cleverhansl2l.utils_tf import clip_eta from cleverhansl2l import utils_tf _logger = utils.create_logger("cleverhans.attacks") tf_dtype = tf.as_dtype('float32')
def __init__(self, fan_in=None, a=5, seed=None, dtype=tf.float32): self.fan_in = fan_in self.a = a self.seed = seed self.dtype = tf.as_dtype(dtype)
def D_paper( images_in, # Input: Images [minibatch, channel, height, width]. num_channels=1, # Number of input color channels. Overridden based on dataset. resolution=32, # Input resolution. Overridden based on dataset. label_size=0, # Dimensionality of the labels, 0 if no labels. Overridden based on dataset. fmap_base=8192, # Overall multiplier for the number of feature maps. fmap_decay=1.0, # log2 feature map reduction when doubling the resolution. fmap_max=512, # Maximum number of feature maps in any layer. use_wscale=True, # Enable equalized learning rate? mbstd_group_size=4, # Group size for the minibatch standard deviation layer, 0 = disable. dtype='float32', # Data type to use for activations and outputs. fused_scale=True, # True = use fused conv2d + downscale2d, False = separate downscale2d layers. structure=None, # 'linear' = human-readable, 'recursive' = efficient, None = select automatically is_template_graph=False, # True = template graph constructed by the Network class, False = actual evaluation. **kwargs): # Ignore unrecognized keyword args. resolution_log2 = int(np.log2(resolution)) assert resolution == 2**resolution_log2 and resolution >= 4 def nf(stage): return min(int(fmap_base / (2.0**(stage * fmap_decay))), fmap_max) if structure is None: structure = 'linear' if is_template_graph else 'recursive' act = leaky_relu images_in.set_shape([None, num_channels, resolution, resolution]) images_in = tf.cast(images_in, dtype) lod_in = tf.cast( tf.get_variable('lod', initializer=np.float32(0.0), trainable=False), dtype) # Building blocks. def fromrgb(x, res): # res = 2..resolution_log2 with tf.variable_scope('FromRGB_lod%d' % (resolution_log2 - res)): return act( apply_bias( conv2d(x, fmaps=nf(res - 1), kernel=1, use_wscale=use_wscale))) def block(x, res): # res = 2..resolution_log2 with tf.variable_scope('%dx%d' % (2**res, 2**res)): if res >= 3: # 8x8 and up with tf.variable_scope('Conv0'): x = act( apply_bias( conv2d(x, fmaps=nf(res - 1), kernel=3, use_wscale=use_wscale))) if fused_scale: with tf.variable_scope('Conv1_down'): x = act( apply_bias( conv2d_downscale2d(x, fmaps=nf(res - 2), kernel=3, use_wscale=use_wscale))) else: with tf.variable_scope('Conv1'): x = act( apply_bias( conv2d(x, fmaps=nf(res - 2), kernel=3, use_wscale=use_wscale))) x = downscale2d(x) else: # 4x4 if mbstd_group_size > 1: x = minibatch_stddev_layer(x, mbstd_group_size) with tf.variable_scope('Conv'): x = act( apply_bias( conv2d(x, fmaps=nf(res - 1), kernel=3, use_wscale=use_wscale))) with tf.variable_scope('Dense0'): x = act( apply_bias( dense(x, fmaps=nf(res - 2), use_wscale=use_wscale))) with tf.variable_scope('Dense1'): x = apply_bias( dense(x, fmaps=1 + label_size, gain=1, use_wscale=use_wscale)) return x # Linear structure: simple but inefficient. if structure == 'linear': img = images_in x = fromrgb(img, resolution_log2) for res in range(resolution_log2, 2, -1): lod = resolution_log2 - res x = block(x, res) img = downscale2d(img) y = fromrgb(img, res - 1) with tf.variable_scope('Grow_lod%d' % lod): x = lerp_clip(x, y, lod_in - lod) combo_out = block(x, 2) # Recursive structure: complex but efficient. if structure == 'recursive': def grow(res, lod): x = lambda: fromrgb(downscale2d(images_in, 2**lod), res) if lod > 0: x = cset(x, (lod_in < lod), lambda: grow(res + 1, lod - 1)) x = block(x(), res) y = lambda: x if res > 2: y = cset( y, (lod_in > lod), lambda: lerp( x, fromrgb(downscale2d(images_in, 2**(lod + 1)), res - 1), lod_in - lod)) return y() combo_out = grow(2, resolution_log2 - 2) assert combo_out.dtype == tf.as_dtype(dtype) scores_out = tf.identity(combo_out[:, :1], name='scores_out') labels_out = tf.identity(combo_out[:, 1:], name='labels_out') return scores_out, labels_out
def __init__(self, shape, dtype, minimum=None, maximum=None, name=None): """Initializes a new `BoundedArraySpec`. Args: shape: An iterable specifying the array shape. dtype: numpy dtype or string specifying the array dtype. minimum: Number or sequence specifying the maximum element bounds (inclusive). Must be broadcastable to `shape`. maximum: Number or sequence specifying the maximum element bounds (inclusive). Must be broadcastable to `shape`. name: Optional string containing a semantic name for the corresponding array. Defaults to `None`. Raises: ValueError: If `minimum` or `maximum` are not broadcastable to `shape` or if the limits are outside of the range of the specified dtype. TypeError: If the shape is not an iterable or if the `dtype` is an invalid numpy dtype. """ super(BoundedArraySpec, self).__init__(shape, dtype, name) try: np.broadcast_to(minimum, shape=shape) except ValueError as numpy_exception: raise ValueError('minimum is not compatible with shape. ' 'Message: {!r}.'.format(numpy_exception)) try: np.broadcast_to(maximum, shape=shape) except ValueError as numpy_exception: raise ValueError('maximum is not compatible with shape. ' 'Message: {!r}.'.format(numpy_exception)) tf_dtype = tf.as_dtype(self._dtype) low = tf_dtype.min high = tf_dtype.max if minimum is None: minimum = low if maximum is None: maximum = high self._minimum = np.array(minimum) self._maximum = np.array(maximum) if tf_dtype.is_floating: # Replacing infinities with extreme finite float values. self._minimum[self._minimum == -np.inf] = low self._minimum[self._minimum == np.inf] = high self._maximum[self._maximum == -np.inf] = low self._maximum[self._maximum == np.inf] = high if np.any(self._minimum > self._maximum): raise ValueError( 'Spec bounds min has values greater than max: [{},{}]'.format( self._minimum, self._maximum)) if (np.any(self._minimum < low) or np.any(self._minimum > high) or np.any(self._maximum < low) or np.any(self._maximum > high)): raise ValueError( 'Spec bounds [{},{}] not within the range [{}, {}] of the given ' 'dtype ({})'.format(self._minimum, self._maximum, low, high, self._dtype)) self._minimum = self._minimum.astype(self._dtype) self._minimum.setflags(write=False) self._maximum = self._maximum.astype(self._dtype) self._maximum.setflags(write=False)
def G_mapping( latents_in, # First input: Latent vectors (Z) [minibatch, latent_size]. labels_in, # Second input: Conditioning labels [minibatch, label_size]. latent_size=512, # Latent vector (Z) dimensionality. label_size=0, # Label dimensionality, 0 if no labels. dlatent_size=512, # Disentangled latent (W) dimensionality. dlatent_broadcast=None, # Output disentangled latent (W) as [minibatch, dlatent_size] or [minibatch, dlatent_broadcast, dlatent_size]. mapping_layers=8, # Number of mapping layers. mapping_fmaps=512, # Number of activations in the mapping layers. mapping_lrmul=0.01, # Learning rate multiplier for the mapping layers. mapping_nonlinearity='lrelu', # Activation function: 'relu', 'lrelu'. use_wscale=True, # Enable equalized learning rate? normalize_latents=True, # Normalize latent vectors (Z) before feeding them to the mapping layers? dtype='float32', # Data type to use for activations and outputs. **_kwargs): # Ignore unrecognized keyword args. act, gain = { 'relu': (tf.nn.relu, np.sqrt(2)), 'lrelu': (leaky_relu, np.sqrt(2)) }[mapping_nonlinearity] # Inputs. latents_in.set_shape([None, latent_size]) labels_in.set_shape([None, label_size]) latents_in = tf.cast(latents_in, dtype) labels_in = tf.cast(labels_in, dtype) x = latents_in # Embed labels and concatenate them with latents. if label_size: with tf.variable_scope('LabelConcat'): w = tf.get_variable('weight', shape=[label_size, latent_size], initializer=tf.initializers.random_normal()) y = tf.matmul(labels_in, tf.cast(w, dtype)) x = tf.concat([x, y], axis=1) # Normalize latents. if normalize_latents: x = pixel_norm(x) # Mapping layers. for layer_idx in range(mapping_layers): with tf.variable_scope('Dense%d' % layer_idx): fmaps = dlatent_size if layer_idx == mapping_layers - 1 else mapping_fmaps x = dense(x, fmaps=fmaps, gain=gain, use_wscale=use_wscale, lrmul=mapping_lrmul) x = apply_bias(x, lrmul=mapping_lrmul) x = act(x) # Broadcast. if dlatent_broadcast is not None: with tf.variable_scope('Broadcast'): x = tf.tile(x[:, np.newaxis], [1, dlatent_broadcast, 1]) # Output. assert x.dtype == tf.as_dtype(dtype) return tf.identity(x, name='dlatents_out')