def _setup_training(self):
"""Sets up graph, model and trainer."""
self._graph = tf.Graph()
with self._graph.as_default():
tf.set_random_seed(self.tf_random_seed)
self._global_step = tf.Variable(0, name="global_step", trainable=False)
# Setting up input and output placeholders.
input_shape = [None] + self._data_feeder.input_shape[1:]
output_shape = [None] + self._data_feeder.output_shape[1:]
self._inp = tf.placeholder(
tf.as_dtype(self._data_feeder.input_dtype), input_shape,
name="input")
self._out = tf.placeholder(
tf.as_dtype(self._data_feeder.output_dtype), output_shape,
name="output")
# Create model's graph.
self._model_predictions, self._model_loss = self.model_fn(self._inp, self._out)
# Create trainer and augment graph with gradients and optimizer.
self._trainer = TensorFlowTrainer(self._model_loss,
self._global_step, self.optimizer, self.learning_rate)
self._session = tf.Session(self.tf_master,
config=tf.ConfigProto(log_device_placement=self.log_device_placement))
def _setup_training(self):
"""Sets up graph, model and trainer."""
self._graph = tf.Graph()
self._graph.add_to_collection("IS_TRAINING", True)
with self._graph.as_default():
tf.set_random_seed(self.tf_random_seed)
self._global_step = tf.Variable(
0, name="global_step", trainable=False)
# Setting up input and output placeholders.
input_shape = [None] + self._data_feeder.input_shape[1:]
output_shape = [None] + self._data_feeder.output_shape[1:]
self._inp = tf.placeholder(
tf.as_dtype(self._data_feeder.input_dtype), input_shape,
name="input")
self._out = tf.placeholder(
tf.as_dtype(self._data_feeder.output_dtype), output_shape,
name="output")
# If class weights are provided, add them to the graph.
# Different loss functions can use this tensor by name.
if self.class_weight:
self._class_weight_node = tf.constant(
self.class_weight, name='class_weight')
# Add histograms for X and y if they are floats.
if self._data_feeder.input_dtype in (np.float32, np.float64):
tf.histogram_summary("X", self._inp)
if self._data_feeder.output_dtype in (np.float32, np.float64):
tf.histogram_summary("y", self._out)
# Create model's graph.
self._model_predictions, self._model_loss = self.model_fn(
self._inp, self._out)
# Create summary to monitor loss
tf.scalar_summary("loss", self._model_loss)
# Set up a single operator to merge all the summaries
self._summaries = tf.merge_all_summaries()
# Create trainer and augment graph with gradients and optimizer.
# Additionally creates initialization ops.
self._trainer = TensorFlowTrainer(
loss=self._model_loss, global_step=self._global_step,
optimizer=self.optimizer, learning_rate=self.learning_rate)
# Create model's saver capturing all the nodes created up until now.
self._saver = tf.train.Saver(
max_to_keep=self.max_to_keep,
keep_checkpoint_every_n_hours=self.keep_checkpoint_every_n_hours)
# Enable monitor to create validation data dict with appropriate tf placeholders
self._monitor.create_val_feed_dict(self._inp, self._out)
# Create session to run model with.
if self.config_addon is None:
self.config_addon = ConfigAddon(verbose=self.verbose)
self._session = tf.Session(self.tf_master, config=self.config_addon.config)
def _setup_training(self):
"""Sets up graph, model and trainer."""
self._graph = tf.Graph()
self._graph.add_to_collection("IS_TRAINING", True)
with self._graph.as_default():
tf.set_random_seed(self.tf_random_seed)
self._global_step = tf.Variable(
0, name="global_step", trainable=False)
# Setting up input and output placeholders.
input_shape = [None] + self._data_feeder.input_shape[1:]
output_shape = [None] + self._data_feeder.output_shape[1:]
self._inp = tf.placeholder(
tf.as_dtype(self._data_feeder.input_dtype), input_shape,
name="input")
self._out = tf.placeholder(
tf.as_dtype(self._data_feeder.output_dtype), output_shape,
name="output")
# Add histograms for X and y if they are floats.
if self._data_feeder.input_dtype in (np.float32, np.float64):
tf.histogram_summary("X", self._inp)
if self._data_feeder.output_dtype in (np.float32, np.float64):
tf.histogram_summary("y", self._out)
# Create model's graph.
# pdb.set_trace()
self._model_predictions, self._model_loss = self.model_fn(
self._inp, self._out)
# Create summary to monitor loss
tf.scalar_summary("loss", self._model_loss)
# Set up a single operator to merge all the summaries
self._summaries = tf.merge_all_summaries()
with tf.name_scope("test") as scope:
if self.n_classes > 0:
correct_prediction = tf.equal(tf.argmax(self._out,1), tf.argmax(self._model_predictions,1))
self._cv_accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
self._cv_accuracy_summary = tf.scalar_summary("cv accuracy", self._cv_accuracy)
# Create trainer and augment graph with gradients and optimizer.
# Additionally creates initialization ops.
self._trainer = TensorFlowTrainer(
loss=self._model_loss, global_step=self._global_step,
optimizer=self.optimizer, learning_rate=self.learning_rate)
# Create model's saver capturing all the nodes created up until now.
self._saver = tf.train.Saver(
max_to_keep=self.max_to_keep,
keep_checkpoint_every_n_hours=self.keep_checkpoint_every_n_hours)
# Create session to run model with.
self._session = tf.Session(self.tf_master,
config=tf.ConfigProto(
log_device_placement=self.verbose > 1,
inter_op_parallelism_threads=self.num_cores,
intra_op_parallelism_threads=self.num_cores))
def testAllTypesConvertibleToNumpyDtype(self):
for datatype_enum in types_pb2.DataType.values():
if not _is_numeric_dtype_enum(datatype_enum):
continue
dtype = tf.as_dtype(datatype_enum)
numpy_dtype = dtype.as_numpy_dtype
_ = np.empty((1, 1, 1, 1), dtype=numpy_dtype)
if dtype.base_dtype != tf.bfloat16:
# NOTE(touts): Intentionally no way to feed a DT_BFLOAT16.
self.assertEqual(tf.as_dtype(datatype_enum).base_dtype, tf.as_dtype(numpy_dtype))
def testIsInteger(self):
self.assertEqual(tf.as_dtype("int8").is_integer, True)
self.assertEqual(tf.as_dtype("int16").is_integer, True)
self.assertEqual(tf.as_dtype("int32").is_integer, True)
self.assertEqual(tf.as_dtype("int64").is_integer, True)
self.assertEqual(tf.as_dtype("uint8").is_integer, True)
self.assertEqual(tf.as_dtype("complex64").is_integer, False)
self.assertEqual(tf.as_dtype("float").is_integer, False)
self.assertEqual(tf.as_dtype("double").is_integer, False)
self.assertEqual(tf.as_dtype("string").is_integer, False)
self.assertEqual(tf.as_dtype("bool").is_integer, False)
def testIsFloating(self):
self.assertEqual(tf.as_dtype("int8").is_floating, False)
self.assertEqual(tf.as_dtype("int16").is_floating, False)
self.assertEqual(tf.as_dtype("int32").is_floating, False)
self.assertEqual(tf.as_dtype("int64").is_floating, False)
self.assertEqual(tf.as_dtype("uint8").is_floating, False)
self.assertEqual(tf.as_dtype("complex64").is_floating, False)
self.assertEqual(tf.as_dtype("float32").is_floating, True)
self.assertEqual(tf.as_dtype("float64").is_floating, True)
self.assertEqual(tf.as_dtype("string").is_floating, False)
self.assertEqual(tf.as_dtype("bool").is_floating, False)
def testIsUnsigned(self):
self.assertEqual(tf.as_dtype("int8").is_unsigned, False)
self.assertEqual(tf.as_dtype("int16").is_unsigned, False)
self.assertEqual(tf.as_dtype("int32").is_unsigned, False)
self.assertEqual(tf.as_dtype("int64").is_unsigned, False)
self.assertEqual(tf.as_dtype("uint8").is_unsigned, True)
self.assertEqual(tf.as_dtype("float32").is_unsigned, False)
self.assertEqual(tf.as_dtype("float64").is_unsigned, False)
self.assertEqual(tf.as_dtype("bool").is_unsigned, False)
self.assertEqual(tf.as_dtype("string").is_unsigned, False)
self.assertEqual(tf.as_dtype("complex64").is_unsigned, False)
def fprop(self, x, **kwargs):
del kwargs
with tf.variable_scope(self.scope, reuse=tf.AUTO_REUSE):
w1 = tf.constant([[1.5, .3], [-2, 0.3]],
dtype=tf.as_dtype(x.dtype))
w2 = tf.constant([[-2.4, 1.2], [0.5, -2.3]],
dtype=tf.as_dtype(x.dtype))
h1 = tf.nn.sigmoid(tf.matmul(x, w1))
res = tf.matmul(h1, w2)
return {self.O_LOGITS: res,
self.O_PROBS: tf.nn.softmax(res)}
def __init__(self, images, labels, fake_data=False, one_hot=False,
dtype=tf.float32):
"""Construct a DataSet.
one_hot arg is used only if fake_data is true. `dtype` can be either
`uint8` to leave the input as `[0, 255]`, or `float32` to rescale into
`[0, 1]`.
"""
dtype = tf.as_dtype(dtype).base_dtype
if dtype not in (tf.uint8, tf.float32):
raise TypeError('Invalid image dtype %r, expected uint8 or float32' %
dtype)
if fake_data:
self._num_examples = 10000
self.one_hot = one_hot
else:
assert images.shape[0] == labels.shape[0], (
'images.shape: %s labels.shape: %s' % (images.shape,
labels.shape))
self._num_examples = images.shape[0]
self._images = images
self._labels = labels
self._epochs_completed = 0
self._index_in_epoch = 0
def __init__(self, images, labels, preprocess="scale", da=False,
dtype=tf.float32):
dtype = tf.as_dtype(dtype).base_dtype
if dtype not in (tf.uint8, tf.float32):
raise TypeError('Invalid image dtype %r, expected uint8 or float32' %
dtype)
assert images.shape[0] == labels.shape[0], (
'images.shape: %s labels.shape: %s' % (images.shape,
labels.shape))
self._num_examples = images.shape[0]
if dtype == tf.float32:
# Convert from [0, 255] -> [0.0, 1.0].
images = images.astype(np.float32)
if preprocess=="IMAGENET":
VGG_MEAN=np.array([123.68, 116.779, 103.99])
images[:,:,:,0]-= VGG_MEAN[0]
images[:,:,:,1]-= VGG_MEAN[1]
images[:,:,:,2]-= VGG_MEAN[2]
elif preprocess=="scale":
images = np.multiply(images, 1.0 / 255.0)
self._images = images
self._labels = labels
self._DA=da
self._epochs_completed = 0
self._index_in_epoch = 0
def __init__(self, instances, labels, xlength, ylength, imagedata,
patch_image_width, dtype = tf.float32):
dtype = tf.as_dtype(dtype).base_dtype
if dtype not in (tf.uint8, tf.float32):
raise TypeError('Invalid image dtype %r, expected uint8 or float32'
% dtype)
assert instances.shape[0] == labels.shape[0], (
'instances.shape: %s labels.shape: %s' % (instances.shape,
labels.shape))
self._num_examples = instances.shape[0]
if dtype == tf.float32:
for i in range(len(imagedata)):
imagedata[i] = imagedata[i].astype(np.float32)
imagedata[i] = np.multiply(imagedata[i], 1.0 / 255.0)
self._instances = instances
self._labels = labels
self._epochs_completed = 0
self._index_in_epoch = 0
self._xlength = xlength
self._ylength = ylength
self._imagedata = imagedata
self._patch_image_width= patch_image_width
def __init__(self, images, labels, fake_data=False, one_hot=False,
dtype=tf.float32):
"""Construct a DataSet.
one_hot arg is used only if fake_data is true. `dtype` can be either
`uint8` to leave the input as `[0, 255]`, or `float32` to rescale into
`[0, 1]`.
"""
dtype = tf.as_dtype(dtype).base_dtype
if dtype not in (tf.uint8, tf.float32):
raise TypeError('Invalid image dtype %r, expected uint8 or float32' %
dtype)
if fake_data:
self._num_examples = 10000
self.one_hot = one_hot
else:
assert images.shape[0] == labels.shape[0], (
'images.shape: %s labels.shape: %s' % (images.shape,
labels.shape))
self._num_examples = images.shape[0]
# Convert shape from [num examples, rows, columns, depth]
# to [num examples, rows*columns] (assuming depth == 1)
assert images.shape[3] == 1
images = images.reshape(images.shape[0],
images.shape[1] * images.shape[2])
if dtype == tf.float32:
# Convert from [0, 255] -> [0.0, 1.0].
images = images.astype(numpy.float32)
images = numpy.multiply(images, 1.0 / 255.0)
self._images = images
self._labels = labels
self._epochs_completed = 0
self._index_in_epoch = 0
def random_normal(shape, mean=0.0, stddev=1.0, dtype=tf.float32, seed=None):
"""Tensor with (possibly complex) Gaussian entries.
Samples are distributed like
```
N(mean, stddev^2), if dtype is real,
X + iY, where X, Y ~ N(mean, stddev^2) if dtype is complex.
```
Args:
shape: `TensorShape` or Python list. Shape of the returned tensor.
mean: `Tensor` giving mean of normal to sample from.
stddev: `Tensor` giving stdev of normal to sample from.
dtype: `TensorFlow` `dtype` or numpy dtype
seed: Python integer seed for the RNG.
Returns:
`Tensor` with desired shape and dtype.
"""
dtype = tf.as_dtype(dtype)
with tf.name_scope("random_normal"):
samples = tf.random_normal(
shape, mean=mean, stddev=stddev, dtype=dtype.real_dtype, seed=seed)
if dtype.is_complex:
if seed is not None:
seed += 1234
more_samples = tf.random_normal(
shape, mean=mean, stddev=stddev, dtype=dtype.real_dtype, seed=seed)
samples = tf.complex(samples, more_samples)
return samples
def __init__(self, band_count, images, labels, dtype=tf.float32):
"""
Construct a DataSet.
`dtype` can be either `uint8` to leave the input as `[0, 255]`,
or `float32` to rescale into `[0, 1]`.
"""
dtype = tf.as_dtype(dtype).base_dtype
if dtype not in (tf.uint8, tf.float32):
raise TypeError('Invalid image dtype %r, expected uint8 or float32' % dtype)
assert images.shape[0] == labels.shape[0], (
'images.shape: %s labels.shape: %s' % (images.shape, labels.shape))
self._num_examples = images.shape[0]
# Convert shape from [num examples, rows, columns, depth]
# to [num examples, rows*columns] (assuming depth == 1)
assert images.shape[3] == band_count
# Store the width and height of the images before flattening it, if only for reference.
image_height, image_width = images.shape[1], images.shape[2]
self.original_image_width = image_width
self.original_image_height = image_height
images = images.reshape(images.shape[0], images.shape[1] * images.shape[2]*images.shape[3])
if dtype == tf.float32:
# Convert from [0, 255] -> [0.0, 1.0]
images = images.astype(numpy.float32)
images = numpy.multiply(images, 1.0 / 255.0)
self._images = images
self._labels = labels
self._epochs_completed = 0
self._index_in_epoch = 0
def random_sign_uniform(
shape, minval=None, maxval=None, dtype=tf.float32, seed=None):
"""Tensor with (possibly complex) random entries from a "sign Uniform".
Letting `Z` be a random variable equal to `-1` and `1` with equal probability,
Samples from this `Op` are distributed like
```
Z * X, where X ~ Uniform[minval, maxval], if dtype is real,
Z * (X + iY), where X, Y ~ Uniform[minval, maxval], if dtype is complex.
```
Args:
shape: `TensorShape` or Python list. Shape of the returned tensor.
minval: `0-D` `Tensor` giving the minimum values.
maxval: `0-D` `Tensor` giving the maximum values.
dtype: `TensorFlow` `dtype` or Python dtype
seed: Python integer seed for the RNG.
Returns:
`Tensor` with desired shape and dtype.
"""
dtype = tf.as_dtype(dtype)
with tf.name_scope("random_sign_uniform"):
unsigned_samples = random_uniform(
shape, minval=minval, maxval=maxval, dtype=dtype, seed=seed)
if seed is not None:
seed += 12
signs = tf.sign(tf.random_uniform(shape, minval=-1., maxval=1., seed=seed))
return unsigned_samples * tf.cast(signs, unsigned_samples.dtype)
def _ProcessHealthPill(self, wall_time, step, device_name, node_name,
output_slot, elements):
"""Processes a health pill value by adding it to accumulated state.
Args:
wall_time: The time at which the health pill was created. Provided by the
debugger.
step: The step at which the health pill was created. Provided by the
debugger.
device_name: The name of the node's device.
node_name: The name of the node for this health pill.
output_slot: The output slot for this health pill.
elements: An ND array of 20 floats. The elements of the health pill.
"""
# Key by the node name for fast retrieval of health pills by node name. The
# array is cast to a list so that it is JSON-able. The debugger data plugin
# serves a JSON response.
self._health_pills.AddItem(node_name,
HealthPillEvent(
wall_time=wall_time,
step=step,
device_name=device_name,
node_name=node_name,
output_slot=output_slot,
dtype=repr(tf.as_dtype(elements[12])),
shape=list(elements[14:]),
value=list(elements)))
def __init__(self, expression_profile, labels, feature_name, label_name, dtype=tf.float32):
"""Construct a DataSet.
`dtype` can be either
`uint8` to leave the input as `[0, 255]`, or `float32` to rescale into
`[0, 1]`.
"""
dtype = tf.as_dtype(dtype).base_dtype
if dtype not in (tf.uint8, tf.float32):
raise TypeError('Invalid Dataset input dtype %r, expected uint8 or float32' %
dtype)
assert expression_profile.shape[0] == labels.shape[0], (
'expression_profile.shape: %s labels.shape: %s' % (expression_profile.shape,
labels.shape))
self._num_examples = expression_profile.shape[0]
# Convert shape from [num examples, rows, columns, depth]
# to [num examples, rows*columns] (assuming depth == 1)
assert expression_profile.shape[1] == 77
expression_profile = expression_profile.reshape(expression_profile.shape[0], expression_profile.shape[1])
self._expression_profile = expression_profile
self._labels = labels
self._epochs_completed = 0
self._index_in_epoch = 0
self._feature_name = feature_name
self._label_name = label_name
def _testTensorArrayGradientWriteReadType(self, use_gpu, dtype):
with self.test_session(use_gpu=use_gpu) as sess:
h = data_flow_ops.TensorArray(
dtype=tf.as_dtype(dtype), tensor_array_name="foo", size=3)
c = lambda x: np.array(x, dtype=dtype)
value_0 = tf.constant(c([[4.0, 5.0]]))
value_1 = tf.constant(c(3.0))
w0 = h.write(0, value_0)
w1 = w0.write(1, value_1)
r0 = w1.read(0)
r1 = w1.read(1)
# Test individual components' gradients
grad_just_r0 = tf.gradients(
ys=[r0], xs=[value_0], grad_ys=[c([[2.0, 3.0]])])
grad_just_r0_vals = sess.run(grad_just_r0)
self.assertAllEqual(c([[2.0, 3.0]]), grad_just_r0_vals[0])
grad_just_r1 = tf.gradients(
ys=[r1], xs=[value_1], grad_ys=[c(-2.0)])
grad_just_r1_vals = sess.run(grad_just_r1)
self.assertAllEqual(c(-2.0), grad_just_r1_vals[0])
# Test combined gradients
grad = tf.gradients(
ys=[r0, r1], xs=[value_0, value_1],
grad_ys=[c(-1.0), c([[2.0, 3.0]])])
grad_vals = sess.run(grad)
self.assertEqual(len(grad_vals), 2)
self.assertAllClose(c(-1.0), grad_vals[0])
self.assertAllEqual(c([[2.0, 3.0]]), grad_vals[1])
def _build_graph(self, tf_graph, scope, model_dir):
"""Construct a TensorGraph containing the policy and loss calculations."""
state_shape = self._env.state_shape
state_dtype = self._env.state_dtype
if not self._state_is_list:
state_shape = [state_shape]
state_dtype = [state_dtype]
features = []
for s, d in zip(state_shape, state_dtype):
features.append(Feature(shape=[None] + list(s), dtype=tf.as_dtype(d)))
policy_layers = self._policy.create_layers(features)
action_prob = policy_layers['action_prob']
value = policy_layers['value']
search_prob = Label(shape=(None, self._env.n_actions))
search_value = Label(shape=(None,))
loss = MCTSLoss(
self.value_weight,
in_layers=[action_prob, value, search_prob, search_value])
graph = TensorGraph(
batch_size=self.max_search_depth,
use_queue=False,
graph=tf_graph,
model_dir=model_dir)
for f in features:
graph._add_layer(f)
graph.add_output(action_prob)
graph.add_output(value)
graph.set_loss(loss)
graph.set_optimizer(self._optimizer)
with graph._get_tf("Graph").as_default():
with tf.variable_scope(scope):
graph.build()
if len(graph.rnn_initial_states) > 0:
raise ValueError('MCTS does not support policies with recurrent layers')
return graph, features, action_prob, value, search_prob, search_value
def _compute_health_pill(self, x):
x_clean = x[np.where(
np.logical_and(
np.logical_not(np.isnan(x)), np.logical_not(np.isinf(x))))]
if np.size(x_clean):
x_min = np.min(x_clean)
x_max = np.max(x_clean)
x_mean = np.mean(x_clean)
x_var = np.var(x_clean)
else:
x_min = np.inf
x_max = -np.inf
x_mean = np.nan
x_var = np.nan
return np.array([
1.0, # Assume is initialized.
np.size(x),
np.sum(np.isnan(x)),
np.sum(x == -np.inf),
np.sum(np.logical_and(x < 0.0, x != -np.inf)),
np.sum(x == 0.0),
np.sum(np.logical_and(x > 0.0, x != np.inf)),
np.sum(x == np.inf),
x_min,
x_max,
x_mean,
x_var,
float(tf.as_dtype(x.dtype).as_datatype_enum),
float(len(x.shape)),
] + list(x.shape))
def testDtypeErrors(self):
def _TryMakingScalarSummary(dtype):
base = dtype.base_dtype
if base == tf.bool:
v = False
elif base == tf.string:
v = ''
elif base.is_complex:
v = complex(0, 0)
else:
v = base.min
c = tf.constant(v, dtype)
return tf.summary.scalar('name', c)
for datatype_enum in types_pb2.DataType.values():
if datatype_enum == types_pb2.DT_INVALID:
continue
dtype = tf.as_dtype(datatype_enum)
if dtype.is_quantized:
# Quantized ops are funky, and not expected to work.
continue
if dtype.is_integer or dtype.is_floating:
_TryMakingScalarSummary(dtype)
# No exception should be thrown
else:
with self.assertRaises(ValueError):
_TryMakingScalarSummary(dtype)
def common_dtype(args_list, preferred_dtype=None):
"""Returns explict dtype from `args_list` if there is one."""
dtype = None
while args_list:
a = args_list.pop()
if hasattr(a, 'dtype'):
dt = tf.as_dtype(getattr(a, 'dtype')).base_dtype.as_numpy_dtype
else:
if isinstance(a, list):
# Allows for nested types, e.g. Normal([np.float16(1.0)], [2.0])
args_list.extend(a)
continue
if dtype is None:
dtype = dt
elif dtype != dt:
raise TypeError('Found incompatible dtypes, {} and {}.'.format(dtype, dt))
return preferred_dtype if dtype is None else tf.as_dtype(dtype)
def as_dtype(dtype):
"""
Convert the specified dtype to backend dtype.
:param dtype: String or numpy dtype.
:return: Backend dtype.
"""
return tf.as_dtype(dtype)
def _check_asset_node_def(node_def):
"""Raises TypeError if `node_def` does not match the expectations."""
if node_def.op != "Const":
raise TypeError("Asset node must be of type constant.")
if tf.as_dtype(node_def.attr["dtype"].type) != tf.string:
raise TypeError("Asset node must be of dtype string.")
if len(node_def.attr["value"].tensor.string_val) != 1:
raise TypeError("Asset node must be a scalar.")
def run(self, *in_arrays,
return_as_list = False, # True = return a list of NumPy arrays, False = return a single NumPy array, or a tuple if there are multiple outputs.
print_progress = False, # Print progress to the console? Useful for very large input arrays.
minibatch_size = None, # Maximum minibatch size to use, None = disable batching.
num_gpus = 1, # Number of GPUs to use.
out_mul = 1.0, # Multiplicative constant to apply to the output(s).
out_add = 0.0, # Additive constant to apply to the output(s).
out_shrink = 1, # Shrink the spatial dimensions of the output(s) by the given factor.
out_dtype = None, # Convert the output to the specified data type.
**dynamic_kwargs): # Additional keyword arguments to pass into the network construction function.
assert len(in_arrays) == self.num_inputs
num_items = in_arrays[0].shape[0]
if minibatch_size is None:
minibatch_size = num_items
key = str([list(sorted(dynamic_kwargs.items())), num_gpus, out_mul, out_add, out_shrink, out_dtype])
# Build graph.
if key not in self._run_cache:
with absolute_name_scope(self.scope + '/Run'), tf.control_dependencies(None):
in_split = list(zip(*[tf.split(x, num_gpus) for x in self.input_templates]))
out_split = []
for gpu in range(num_gpus):
with tf.device('/gpu:%d' % gpu):
out_expr = self.get_output_for(*in_split[gpu], return_as_list=True, **dynamic_kwargs)
if out_mul != 1.0:
out_expr = [x * out_mul for x in out_expr]
if out_add != 0.0:
out_expr = [x + out_add for x in out_expr]
if out_shrink > 1:
ksize = [1, 1, out_shrink, out_shrink]
out_expr = [tf.nn.avg_pool(x, ksize=ksize, strides=ksize, padding='VALID', data_format='NCHW') for x in out_expr]
if out_dtype is not None:
if tf.as_dtype(out_dtype).is_integer:
out_expr = [tf.round(x) for x in out_expr]
out_expr = [tf.saturate_cast(x, out_dtype) for x in out_expr]
out_split.append(out_expr)
self._run_cache[key] = [tf.concat(outputs, axis=0) for outputs in zip(*out_split)]
# Run minibatches.
out_expr = self._run_cache[key]
out_arrays = [np.empty([num_items] + shape_to_list(expr.shape)[1:], expr.dtype.name) for expr in out_expr]
for mb_begin in range(0, num_items, minibatch_size):
if print_progress:
print('\r%d / %d' % (mb_begin, num_items), end='')
mb_end = min(mb_begin + minibatch_size, num_items)
mb_in = [src[mb_begin : mb_end] for src in in_arrays]
mb_out = tf.get_default_session().run(out_expr, dict(zip(self.input_templates, mb_in)))
for dst, src in zip(out_arrays, mb_out):
dst[mb_begin : mb_end] = src
# Done.
if print_progress:
print('\r%d / %d' % (num_items, num_items))
if not return_as_list:
out_arrays = out_arrays[0] if len(out_arrays) == 1 else tuple(out_arrays)
return out_arrays
def testDTypesHaveUniqueNames(self):
dtypes = []
names = set()
for datatype_enum in types_pb2.DataType.values():
if datatype_enum == types_pb2.DT_INVALID:
continue
dtype = tf.as_dtype(datatype_enum)
dtypes.append(dtype)
names.add(dtype.name)
self.assertEqual(len(dtypes), len(names))
def _setup_training(self):
"""Sets up graph, model and trainer."""
self._graph = tf.Graph()
with self._graph.as_default():
tf.set_random_seed(self.tf_random_seed)
self._global_step = tf.Variable(
0, name="global_step", trainable=False)
# Setting up input and output placeholders.
input_shape = [None] + self._data_feeder.input_shape[1:]
output_shape = [None] + self._data_feeder.output_shape[1:]
self._inp = tf.placeholder(
tf.as_dtype(self._data_feeder.input_dtype), input_shape,
name="input")
self._out = tf.placeholder(
tf.as_dtype(self._data_feeder.output_dtype), output_shape,
name="output")
# Add histograms for X and y.
tf.histogram_summary("X", self._inp)
tf.histogram_summary("y", self._out)
# Create model's graph.
self._model_predictions, self._model_loss = self.model_fn(
self._inp, self._out)
# Create trainer and augment graph with gradients and optimizer.
# Additionally creates initialization ops.
self._trainer = TensorFlowTrainer(
loss=self._model_loss, global_step=self._global_step,
optimizer=self.optimizer, learning_rate=self.learning_rate)
# Create model's saver capturing all the nodes created up until now.
self._saver = tf.train.Saver(
max_to_keep=self.max_to_keep,
keep_checkpoint_every_n_hours=self.keep_checkpoint_every_n_hours)
# Create session to run model with.
self._session = tf.Session(self.tf_master,
config=tf.ConfigProto(
log_device_placement=self.verbose > 1,
inter_op_parallelism_threads=self.num_cores,
intra_op_parallelism_threads=self.num_cores))
def test_constant(dtype, sess):
val = np.random.randn(10, 10).astype(np.float64)
signals = SignalDict(tf.float32, 1)
const0 = signals.constant(val, dtype=dtype)
const1 = signals.constant(val, dtype=dtype, cutoff=0)
assert const0.op.type == "Const"
assert const1.op.type == "Identity"
assert const0.dtype == (dtype if dtype else tf.as_dtype(val.dtype))
assert const1.dtype == (dtype if dtype else tf.as_dtype(val.dtype))
sess.run(tf.variables_initializer(tf.get_collection("constants")),
feed_dict=signals.constant_phs)
c0, c1 = sess.run([const0, const1])
assert np.allclose(c0, val)
assert np.allclose(c1, val)
def make_iterator(self,
batch_size=100,
epochs=1,
deterministic=False,
pad_batches=False):
"""Create a tf.data.Iterator that iterates over the data in this Dataset.
The iterator's get_next() method returns a tuple of three tensors (X, y, w)
which can be used to retrieve the features, labels, and weights respectively.
Parameters
----------
batch_size: int
the number of samples to include in each batch
epochs: int
the number of times to iterate over the Dataset
deterministic: bool
if True, the data is produced in order. If False, a different random
permutation of the data is used for each epoch.
pad_batches: bool
if True, batches are padded as necessary to make the size of each batch
exactly equal batch_size.
"""
# Retrieve the first sample so we can determine the dtypes.
import tensorflow as tf
X, y, w, ids = next(self.itersamples())
dtypes = (tf.as_dtype(X.dtype), tf.as_dtype(y.dtype), tf.as_dtype(w.dtype))
shapes = (tf.TensorShape([None] + list(X.shape)),
tf.TensorShape([None] + list(y.shape)),
tf.TensorShape([None] + list(w.shape)))
# Create a Tensorflow Dataset and have it create an Iterator.
def gen_data():
for epoch in range(epochs):
for X, y, w, ids in self.iterbatches(batch_size, epoch, deterministic,
pad_batches):
yield (X, y, w)
dataset = tf.data.Dataset.from_generator(gen_data, dtypes, shapes)
return dataset.make_one_shot_iterator()
def __init__(self,
data_dir,
coord,
train=True,
threshold=None,
queue_size=16,
min_after_dequeue=4,
q_shape=None,
pattern='*.npy',
n_threads=1,
multi=True,
label_file=LABEL_FILE,
label_type=tf.int32,
pref=False,
dtype=np.float32):
self.data_dir = data_dir
self.coord = coord
self.n_threads = n_threads
self.threshold = threshold
self.ftype = pattern
self.corpus_size = get_corpus_size(self.data_dir, pattern=self.ftype)
self.threads = []
self.q_shape = q_shape
self.multi = multi
self.train = train
self.npdtype = dtype
self.tfdtype = tf.as_dtype(self.npdtype)
self.sample_placeholder = tf.placeholder(dtype=self.tfdtype, shape=None)
self.label_shape = []
self.label_type = label_type
self.pattern = pattern
self.pref = pref
self.labels_df = None
self.label_shape = []
if self.train:
if label_file is not None:
self.labels_df = load_label_df(label_file)
self.label_shape = [len(self.labels_df.columns)]
self.label_placeholder = tf.placeholder(dtype=self.label_type, shape=self.label_shape, name='label') #!!!
if self.q_shape:
#self.queue = tf.FIFOQueue(queue_size,[tf.float32,tf.int32], shapes=[q_shape,[]])
self.queue = tf.RandomShuffleQueue(queue_size, min_after_dequeue,
[self.tfdtype,label_type], shapes=[q_shape, self.label_shape])
else:
self.q_shape = [(1, None, None, 1)]
self.queue = tf.PaddingFIFOQueue(queue_size,
[self.tfdtype, label_type],
shapes=[self.q_shape,self.label_shape])
else:
self.label_placeholder = tf.placeholder(dtype=tf.string, shape=[], name='label') #!!!
self.queue = tf.FIFOQueue(queue_size,[self.tfdtype,tf.string], shapes=[q_shape,[]])
self.enqueue = self.queue.enqueue([self.sample_placeholder, self.label_placeholder])
def run_test(self, name, backend="caffe2", onnx_file=None, opset=None, extra_opset=None,
perf=None, fold_const=None):
"""Run complete test against backend."""
self.perf = perf
# get the model
if self.url:
_, dir_name = self.download_model()
logger.info("Downloaded to %s", dir_name)
model_path = os.path.join(dir_name, self.local)
else:
model_path = self.local
logger.info("Load model from %s", model_path)
input_names = list(self.input_names.keys())
outputs = self.output_names
if self.model_type in ["checkpoint"]:
graph_def, input_names, outputs = tf_loader.from_checkpoint(model_path, input_names, outputs)
elif self.model_type in ["saved_model"]:
graph_def, input_names, outputs = tf_loader.from_saved_model(model_path, input_names, outputs)
else:
graph_def, input_names, outputs = tf_loader.from_graphdef(model_path, input_names, outputs)
if utils.is_debug_mode():
utils.save_protobuf(os.path.join(TEMP_DIR, name + "_after_tf_optimize.pb"), graph_def)
inputs = {}
shape_override = {}
tf_reset_default_graph()
g = tf.import_graph_def(graph_def, name='')
# with tf_session(config=tf.ConfigProto(allow_soft_placement=True), graph=g) as sess:
with tf_session(graph=g) as sess:
# create the input data
for k in input_names:
v = self.input_names[k]
t = sess.graph.get_tensor_by_name(k)
expected_dtype = tf.as_dtype(t.dtype).name
if isinstance(v, six.text_type) and v.startswith("np."):
np_value = eval(v) # pylint: disable=eval-used
if expected_dtype != np_value.dtype:
logger.warning("dtype mismatch for input %s: expected=%s, actual=%s", k, expected_dtype,
np_value.dtype)
inputs[k] = np_value.astype(expected_dtype)
else:
inputs[k] = self.make_input(v).astype(expected_dtype)
if self.force_input_shape:
for k, v in inputs.items():
shape_override[k] = list(v.shape)
# run the model with tensorflow
if self.skip_tensorflow:
logger.info("TensorFlow SKIPPED")
else:
tf_results = self.run_tensorflow(sess, inputs)
logger.info("TensorFlow OK")
model_proto = None
try:
# convert model to onnx
onnx_graph = self.to_onnx(sess.graph, opset=opset, extra_opset=extra_opset,
shape_override=shape_override, input_names=inputs.keys())
onnx_graph = optimizer.optimize_graph(onnx_graph)
model_proto = onnx_graph.make_model("converted from tf2onnx")
logger.info("To_ONNX, OK")
if onnx_file:
self.create_onnx_file(name, model_proto, inputs, onnx_file)
except Exception:
logger.error("To_ONNX FAIL", exc_info=1)
return False
try:
onnx_results = None
if backend == "caffe2":
onnx_results = self.run_caffe2(name, model_proto, inputs)
elif backend == "onnxruntime":
onnx_results = self.run_onnxruntime(name, model_proto, inputs)
else:
raise ValueError("unknown backend")
logger.info("Run_ONNX OK")
try:
if self.skip_tensorflow:
logger.info("Results: skipped tensorflow")
else:
if self.check_only_shape:
for tf_res, onnx_res in zip(tf_results, onnx_results):
np.testing.assert_array_equal(tf_res.shape, onnx_res.shape)
else:
for tf_res, onnx_res in zip(tf_results, onnx_results):
np.testing.assert_allclose(tf_res, onnx_res, rtol=self.rtol, atol=self.atol)
logger.info("Results: OK")
return True
except Exception:
logger.error("Results", exc_info=1)
except Exception:
logger.error("Run_ONNX FAIL", exc_info=1)
return False
def G_synthesis(
dlatents_in, # Input: Disentangled latents (W) [minibatch, num_layers, dlatent_size].
dlatent_size=512, # Disentangled latent (W) dimensionality.
num_channels=3, # Number of output color channels.
resolution=1024, # Output resolution.
fmap_base=8192, # Overall multiplier for the number of feature maps.
fmap_decay=1.0, # log2 feature map reduction when doubling the resolution.
fmap_max=512, # Maximum number of feature maps in any layer.
use_styles=True, # Enable style inputs?
const_input_layer=True, # First layer is a learned constant?
use_noise=True, # Enable noise inputs?
randomize_noise=True, # True = randomize noise inputs every time (non-deterministic), False = read noise inputs from variables.
nonlinearity='lrelu', # Activation function: 'relu', 'lrelu'
use_wscale=True, # Enable equalized learning rate?
use_pixel_norm=False, # Enable pixelwise feature vector normalization?
use_instance_norm=True, # Enable instance normalization?
dtype='float32', # Data type to use for activations and outputs.
fused_scale='auto', # True = fused convolution + scaling, False = separate ops, 'auto' = decide automatically.
blur_filter=[
1, 2, 1
], # Low-pass filter to apply when resampling activations. None = no filtering.
structure='auto', # 'fixed' = no progressive growing, 'linear' = human-readable, 'recursive' = efficient, 'auto' = select automatically.
is_template_graph=False, # True = template graph constructed by the Network class, False = actual evaluation.
force_clean_graph=False, # True = construct a clean graph that looks nice in TensorBoard, False = default behavior.
**_kwargs): # Ignore unrecognized keyword args.
resolution_log2 = int(np.log2(resolution))
assert resolution == 2**resolution_log2 and resolution >= 4
def nf(stage):
return min(int(fmap_base / (2.0**(stage * fmap_decay))), fmap_max)
def blur(x):
return blur2d(x, blur_filter) if blur_filter else x
if is_template_graph: force_clean_graph = True
if force_clean_graph: randomize_noise = False
if structure == 'auto':
structure = 'linear' if force_clean_graph else 'recursive'
act, gain = {
'relu': (tf.nn.relu, np.sqrt(2)),
'lrelu': (leaky_relu, np.sqrt(2))
}[nonlinearity]
num_layers = resolution_log2 * 2 - 2
num_styles = num_layers if use_styles else 1
images_out = None
# Primary inputs.
dlatents_in.set_shape([None, num_styles, dlatent_size])
dlatents_in = tf.cast(dlatents_in, dtype)
lod_in = tf.cast(
tf.get_variable('lod', initializer=np.float32(0), trainable=False),
dtype)
# Noise inputs.
noise_inputs = []
if use_noise:
for layer_idx in range(num_layers):
res = layer_idx // 2 + 2
shape = [1, use_noise, 2**res, 2**res]
noise_inputs.append(
tf.get_variable('noise%d' % layer_idx,
shape=shape,
initializer=tf.initializers.random_normal(),
trainable=False))
# Things to do at the end of each layer.
def layer_epilogue(x, layer_idx):
if use_noise:
x = apply_noise(x,
noise_inputs[layer_idx],
randomize_noise=randomize_noise)
x = apply_bias(x)
x = act(x)
if use_pixel_norm:
x = pixel_norm(x)
if use_instance_norm:
x = instance_norm(x)
if use_styles:
x = style_mod(x, dlatents_in[:, layer_idx], use_wscale=use_wscale)
return x
# Early layers.
with tf.variable_scope('4x4'):
if const_input_layer:
with tf.variable_scope('Const'):
x = tf.get_variable('const',
shape=[1, nf(1), 4, 4],
initializer=tf.initializers.ones())
x = layer_epilogue(
tf.tile(tf.cast(x, dtype),
[tf.shape(dlatents_in)[0], 1, 1, 1]), 0)
else:
with tf.variable_scope('Dense'):
x = dense(
dlatents_in[:, 0],
fmaps=nf(1) * 16,
gain=gain / 4,
use_wscale=use_wscale
) # tweak gain to match the official implementation of Progressing GAN
x = layer_epilogue(tf.reshape(x, [-1, nf(1), 4, 4]), 0)
with tf.variable_scope('Conv'):
x = layer_epilogue(
conv2d(x,
fmaps=nf(1),
kernel=3,
gain=gain,
use_wscale=use_wscale), 1)
# Building blocks for remaining layers.
def block(res, x): # res = 3..resolution_log2
with tf.variable_scope('%dx%d' % (2**res, 2**res)):
with tf.variable_scope('Conv0_up'):
x = layer_epilogue(
blur(
upscale2d_conv2d(x,
fmaps=nf(res - 1),
kernel=3,
gain=gain,
use_wscale=use_wscale,
fused_scale=fused_scale)),
res * 2 - 4)
with tf.variable_scope('Conv1'):
x = layer_epilogue(
conv2d(x,
fmaps=nf(res - 1),
kernel=3,
gain=gain,
use_wscale=use_wscale), res * 2 - 3)
return x
def torgb(res, x): # res = 2..resolution_log2
lod = resolution_log2 - res
with tf.variable_scope('ToRGB_lod%d' % lod):
return apply_bias(
conv2d(x,
fmaps=num_channels,
kernel=1,
gain=1,
use_wscale=use_wscale))
# Fixed structure: simple and efficient, but does not support progressive growing.
if structure == 'fixed':
for res in range(3, resolution_log2 + 1):
x = block(res, x)
images_out = torgb(resolution_log2, x)
# Linear structure: simple but inefficient.
if structure == 'linear':
images_out = torgb(2, x)
for res in range(3, resolution_log2 + 1):
lod = resolution_log2 - res
x = block(res, x)
img = torgb(res, x)
images_out = upscale2d(images_out)
with tf.variable_scope('Grow_lod%d' % lod):
images_out = tflib.lerp_clip(img, images_out, lod_in - lod)
# Recursive structure: complex but efficient.
if structure == 'recursive':
def cset(cur_lambda, new_cond, new_lambda):
return lambda: tf.cond(new_cond, new_lambda, cur_lambda)
def grow(x, res, lod):
y = block(res, x)
img = lambda: upscale2d(torgb(res, y), 2**lod)
img = cset(
img, (lod_in > lod), lambda: upscale2d(
tflib.lerp(torgb(res, y), upscale2d(torgb(res - 1, x)),
lod_in - lod), 2**lod))
if lod > 0:
img = cset(img, (lod_in < lod),
lambda: grow(y, res + 1, lod - 1))
return img()
images_out = grow(x, 3, resolution_log2 - 3)
assert images_out.dtype == tf.as_dtype(dtype)
return tf.identity(images_out, name='images_out')
def master_dtype(self):
return tf.as_dtype(self._hparams.master_dtype)
def activation_dtype(self):
return tf.as_dtype(self._hparams.activation_dtype)
def __init__(self,
base_env_name=None,
batch_size=1,
grayscale=False,
resize_height_factor=2,
resize_width_factor=2,
base_env_timesteps_limit=-1,
max_num_noops=0,
**kwargs):
if base_env_name is None:
base_env_name = self.base_env_name
self._base_env_name = base_env_name
super(T2TGymEnv, self).__init__(batch_size, **kwargs)
self.grayscale = grayscale
self.resize_height_factor = resize_height_factor
self.resize_width_factor = resize_width_factor
if not self.name:
# Set problem name if not registered.
self.name = "Gym%s" % base_env_name
self._envs = [
make_gym_env(base_env_name, base_env_timesteps_limit)
for _ in range(self.batch_size)
]
# max_num_noops works only with atari envs.
if max_num_noops > 0:
assert self._envs[0].unwrapped.get_action_meanings()[
self.noop_action] == "NOOP"
self.max_num_noops = max_num_noops
orig_observ_space = self._envs[0].observation_space
if not all(env.observation_space == orig_observ_space
for env in self._envs):
raise ValueError(
"All environments must use the same observation space.")
self.observation_space = self._derive_observation_space(
orig_observ_space)
self.action_space = self._envs[0].action_space
if not all(env.action_space == self.action_space
for env in self._envs):
raise ValueError(
"All environments must use the same action space.")
with self._tf_graph.obj.as_default():
self._resize = dict()
orig_height, orig_width = orig_observ_space.shape[:2]
self._img_batch_t = _Noncopyable(
tf.placeholder(dtype=tf.uint8,
shape=(None, orig_height, orig_width, 3)))
height, width = self.observation_space.shape[:2]
resized = tf.image.resize_images(self._img_batch_t.obj,
[height, width],
tf.image.ResizeMethod.AREA)
resized = tf.cast(resized,
tf.as_dtype(self.observation_space.dtype))
if self.grayscale:
resized = tf.image.rgb_to_grayscale(resized)
self._resized_img_batch_t = _Noncopyable(resized)
netG_gbc = tf.reduce_mean(input_tensor=tf.stack([tf.reduce_mean(input_tensor=tf.abs(v)) for v in netG_gbc]))
netG_gac = tf.reduce_mean(input_tensor=tf.stack([tf.reduce_mean(input_tensor=tf.abs(v)) for v in netG_gac]))
with tf.compat.v1.name_scope("netD_SGD"):
netD_optimizer = netD.OPTIMIZER
netD_gvs = netD_optimizer.compute_gradients(netD_total_loss, tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.TRAINABLE_VARIABLES, scope=netD.variable_scope_name))
netD_gbc = [grad for grad, var in netD_gvs]
netD_capped_gvs = [(tf.clip_by_value(grad, -netD.GLOBAL_GRADIENT_CLIPPING, netD.GLOBAL_GRADIENT_CLIPPING), var) for grad, var in netD_gvs]
netD_gac = [grad for grad, var in netD_capped_gvs]
netD_opt = netD_optimizer.apply_gradients(netD_capped_gvs)
netD_gbc = tf.reduce_mean(input_tensor=tf.stack([tf.reduce_mean(input_tensor=tf.abs(v)) for v in netD_gbc]))
netD_gac = tf.reduce_mean(input_tensor=tf.stack([tf.reduce_mean(input_tensor=tf.abs(v)) for v in netD_gac]))
netG_train_output1_crop_round = [tf.cast(tf.round(tf.clip_by_value(netG_train_output1_crop[i], 0, 1) * train_df.input1_label_src.dtype.max), tf.as_dtype(FLAGS['data_label_dtype'])) for i in range(FLAGS['data_train_batch_size'])]
netG_train_input1_label_crop_round = [tf.cast(tf.round(tf.clip_by_value(netG_train_input1_label_crop[i], 0, 1) * train_df.input1_label_src.dtype.max), tf.as_dtype(FLAGS['data_label_dtype'])) for i in range(FLAGS['data_train_batch_size'])]
netG_tr_psnr1 = tf.reduce_mean(input_tensor=tf.stack([tf_psnr(netG_train_output1_crop_round[i], netG_train_input1_label_crop_round[i], 0) for i in range(FLAGS['data_train_batch_size'])]))
netG_train_output2_crop_round = [tf.cast(tf.round(tf.clip_by_value(netG_train_output2_crop[i], 0, 1) * train_df.input2_label_src.dtype.max), tf.as_dtype(FLAGS['data_input_dtype'])) for i in range(FLAGS['data_train_batch_size'])]
netG_train_input2_label_crop_round = [tf.cast(tf.round(tf.clip_by_value(netG_train_input2_label_crop[i], 0, 1) * train_df.input2_label_src.dtype.max), tf.as_dtype(FLAGS['data_input_dtype'])) for i in range(FLAGS['data_train_batch_size'])]
netG_tr_psnr2 = tf.reduce_mean(input_tensor=tf.stack([tf_psnr(netG_train_output2_crop_round[i], netG_train_input2_label_crop_round[i], 0) for i in range(FLAGS['data_train_batch_size'])]))
netG_tr_psnr2 = tf.constant(0, dtype=FLAGS['data_compute_dtype'])
netG_test_output1_crop_round = tf.cast(tf.round(tf.clip_by_value(netG_test_output1_crop, 0, 1) * test_df.input2_src.dtype.max), tf.as_dtype(FLAGS['data_label_dtype']))
netG_test_input2_crop_round = tf.cast(tf.round(tf.clip_by_value(netG_test_input2_crop, 0, 1) * test_df.input2_src.dtype.max), tf.as_dtype(FLAGS['data_label_dtype']))
netG_psnr1 = tf_psnr(netG_test_output1_crop_round, netG_test_input2_crop_round, 0)
netG_test_output2_crop_round = tf.cast(tf.round(tf.clip_by_value(netG_test_output2_crop, 0, 1) * test_df.input1_src.dtype.max), tf.as_dtype(FLAGS['data_input_dtype']))
netG_test_input1_crop_round = tf.cast(tf.round(tf.clip_by_value(netG_test_input1_crop, 0, 1) * test_df.input1_src.dtype.max), tf.as_dtype(FLAGS['data_input_dtype']))
netG_psnr2 = tf_psnr(netG_test_output2_crop_round, netG_test_input1_crop_round, 0)
netG_train_summary = [netG_train_loss, netG_train_1_1, netG_train_2_1, netG_train_1_2, netG_train_2_2, train_data_term_1, train_data_term_2, train_constant_term_1, train_constant_term_2, netG_tr_psnr1, netG_tr_psnr2, netG_r_loss, netG_gbc, netG_gac]
def __setstate__(self, state):
self._dtype = tf.as_dtype(state)
def __setstate__(self, state):
self._dtype = tf.as_dtype(state['dtype'])
self._is_categorical = state['is_categorical']
self._min_value = state['min_value']
self._max_value = state['max_value']
self._vocabulary_file = state['vocabulary_file']
def generator(self, labels_in):
assert self.resolution == 2**self.resolution_log2 and self.resolution >= 4
def nf(stage): return min(int(self.fmap_base / (2.0 ** (stage * self.fmap_decay))), self.fmap_max)
# First Layer from Image: A x A x 3 ==> A x A x Channels(A)
# --------------------------------------------------------------------------
def fromrgb(x, res):
with tf.variable_scope('Gen_Enc_FromRGB_lod%d' % (self.resolution_log2 - res)):
return leaky_relu(apply_bias(conv2d(x,
fmaps=nf(res-1), kernel=1, cf = self.channel_first), cf = self.channel_first))
# Building Blocks Encoder: Input --> Convolution (+Bias) --> Batchnormalisation
# --> Activation function --> Downsample by factor 2
# --------------------------------------------------------------------------
def block_e(x,res):
with tf.variable_scope('Gen_Enc%dx%d' % (2**res, 2**res)):
with tf.variable_scope('Conv'):
x = leaky_relu(batchnorm(apply_bias(conv2d(x, fmaps = nf(res-2), kernel = 3,
cf = self.channel_first), cf = self.channel_first), cf = self.channel_first))
x = downscale2d(x, cf = self.channel_first)
return x
# =========================================================================
# Encoder
# =========================================================================
if self.structure == 'linear':
skip = []
img = labels_in
x = fromrgb(img, self.resolution_log2)
#print(x.shape)
for res in range(self.resolution_log2, 4, -1):
lod = self.resolution_log2 - res
x = block_e(x, res)
#print(x.shape)
img = downscale2d(img, cf = self.channel_first)
y = fromrgb(img, res - 1)
with tf.variable_scope('Gen_Enc_Grow_lod%d' % lod):
x = lerp_clip(x, y, self.lod_in - lod)
skip.append(x) #
#print('Encoder after', 2**res, "conv", x.shape)
for res in range(4,0,-1):
lod = self.resolution_log2 - res
x = block_e(x, res)
#print(x.shape)
if res >= 2: skip.append(x)
#print("Encoder Output", x.shape)
combo_out = x
# ========================================================================
# Decoder
# ========================================================================
# Last Layer to Image: A x A x Channels(A) ==> A x A x 3
# --------------------------------------------------------------------------
def torgb(x, res): # res = 2..resolution_log2
lod = self.resolution_log2 - res
with tf.variable_scope('Gen_Dec_ToRGB_lod%d' % lod):
return apply_bias(conv2d(x, fmaps=self.num_channels, kernel=1, cf = self.channel_first), cf = self.channel_first)
# Building Blocks Encoder: Input --> Upsampling by factor 2 --> Convolution (+Bias)
# --> Batchnormalisation --> Activation
# --------------------------------------------------------------------------
def block_d(x,res):
layers = []
with tf.variable_scope('Gen_Dec_%dx%d' % (2**res, 2**res)):
x = upscale2d(x, cf = self.channel_first)
with tf.variable_scope('Conv'):
x = leaky_relu(batchnorm(apply_bias(conv2d(x, fmaps = nf(res-1), kernel = 3, cf = self.channel_first),
cf = self.channel_first), cf = self.channel_first))
return x
# Growing the Decoder
# ---------------------------------------------------------------------------
if self.structure == 'linear':
#print('Decode Input:', x.shape)
x = combo_out
#print('start decoder',x.shape)
x = block_d(x, 1) # 1x1x512 ==> 2x2x512
#print(x.shape)
x = tf.concat([x,skip[-1]], axis=self.conc_axis) # concat ==> 2x2x1024
x = block_d(x, 2) # 2x2x1024 ==> 4x4x512
#print(x.shape)
x = tf.concat([x,skip[-2]], axis=self.conc_axis)
x = block_d(x, 3) # 4x4x1024 ==> 8x8x512
#print(x.shape)
x = tf.concat([x,skip[-3]], axis=self.conc_axis)
x = block_d(x, 4) # 8x8x1024 ==> 16x16x512
#print(x.shape)
#print("-----")
images_out = torgb(x,4) # Extracted Output layer 16x16
#print("Image Out:", images_out.shape)
x = tf.concat([x,skip[-4]], axis=self.conc_axis) # concat ==> 16x16x1024
#print('Decode after const:', x.shape)
for res in range(5,self.resolution_log2+1):
lod = self.resolution_log2 - res
x = block_d(x,res)
#print(x.shape)
img = torgb(x,res)
if res < self.resolution_log2:
x = tf.concat([x,skip[-res]], axis = self.conc_axis)
images_out = upscale2d(images_out, cf = self.channel_first)
with tf.variable_scope('Gen_Dec_Grow_lod%d' % lod):
images_out = lerp_clip(img, images_out, self.lod_in - lod)
#print("Images Out:", images_out.shape)
#print('Decode res:', 2**res,'shape x:', x.shape, "shape img out:", images_out.shape)
#print('output',images_out.shape)
assert images_out.dtype == tf.as_dtype(self.dtype)
images_out = tf.identity(images_out, name='images_out')
return images_out
def optimistic_restore(session,
save_file,
ignore_vars=None,
verbose=False,
ignore_incompatible_shapes=False):
"""This function tries to restore all variables in the save file.
This function ignores variables that do not exist or have incompatible shape.
Raises TypeError if the there is a type mismatch for compatible shapes.
session: tf.Session
The tf session
save_file: str
Path to the checkpoint without the .index, .meta or .data extensions.
ignore_vars: list, tuple or set of str
These variables will be ignored.
verbose: bool
If True prints which variables will be restored
ignore_incompatible_shapes: bool
If True ignores variables with incompatible shapes.
If False raises a runtime error f shapes are incompatible.
"""
print(save_file)
def vprint(*args, **kwargs):
if verbose: print(*args, flush=True, **kwargs)
# def dbg(*args, **kwargs):
# print(*args, flush=True, **kwargs)
def dbg(*args, **kwargs):
pass
if ignore_vars is None:
ignore_vars = []
reader = tf.train.NewCheckpointReader(save_file)
saved_shapes = reader.get_variable_to_shape_map()
var_names = sorted([
(var.name, var.dtype, var.name.split(':')[0])
for var in tf.global_variables()
if not var.name.split(':')[0] in ignore_vars and 'Adam' not in var.name
])
restore_vars = []
dbg(saved_shapes)
for v in tf.global_variables():
dbg(v)
nonfinite_values = False
with tf.variable_scope('', reuse=True):
for var_name, var_dtype, saved_var_name in var_names:
dbg(var_name, var_dtype, saved_var_name, end='')
curr_var = tf.get_variable(saved_var_name, dtype=var_dtype)
var_shape = curr_var.get_shape().as_list()
try:
if var_shape == saved_shapes[saved_var_name]:
dbg(' shape OK')
tmp = reader.get_tensor(saved_var_name)
dbg(tmp.dtype)
# check if there are nonfinite values in the tensor
if not np.all(np.isfinite(tmp)):
nonfinite_values = True
print('{0} contains nonfinite values!'.format(
saved_var_name),
flush=True)
if isinstance(tmp, np.ndarray):
saved_dtype = tf.as_dtype(tmp.dtype)
else:
print(var_name)
saved_dtype = tf.as_dtype(type(tmp))
dbg(saved_dtype, var_dtype,
saved_dtype.is_compatible_with(var_dtype))
if not saved_dtype.is_compatible_with(var_dtype):
raise TypeError(
'types are not compatible for {0}: saved type {1}, variable type {2}.'
.format(saved_var_name, saved_dtype.name,
var_dtype.name))
vprint('restoring ', saved_var_name)
restore_vars.append(curr_var)
else:
vprint('not restoring', saved_var_name,
'incompatible shape:', var_shape, 'vs',
saved_shapes[saved_var_name])
if not ignore_incompatible_shapes:
raise RuntimeError(
'failed to restore "{0}" because of incompatible shapes: var: {1} vs saved: {2} '
.format(saved_var_name, var_shape,
saved_shapes[saved_var_name]))
except KeyError:
print('not restoring {}', saved_var_name)
if nonfinite_values:
raise RuntimeError(
'"{0}" contains nonfinite values!'.format(save_file))
dbg('-1-')
saver = tf.train.Saver(
var_list=restore_vars,
restore_sequentially=True,
)
dbg('-2-')
saver.restore(session, save_file)
dbg('-3-')
def create_train_op(named_scalars, grads_and_vars, optimizer, global_step, params):
tf.get_variable_scope().set_dtype(tf.as_dtype(dtype.floatx()))
gradients = [item[0] for item in grads_and_vars]
variables = [item[1] for item in grads_and_vars]
if params.update_cycle == 1:
zero_variables_op = tf.no_op("zero_variables")
collect_op = tf.no_op("collect_op")
else:
named_vars = {}
for name in named_scalars:
named_var = tf.Variable(tf.zeros([], dtype=tf.float32),
name="{}/CTrainOpReplica".format(name),
trainable=False)
named_vars[name] = named_var
count_var = tf.Variable(tf.zeros([], dtype=tf.as_dtype(dtype.floatx())),
name="count/CTrainOpReplica",
trainable=False)
slot_variables = _replicate_variables(variables, suffix='CTrainOpReplica')
zero_variables_op = _zero_variables(
slot_variables + [count_var] + list(named_vars.values()))
collect_ops = []
# collect gradients
collect_grads_op = _collect_gradients(gradients, slot_variables)
collect_ops.append(collect_grads_op)
# collect other scalars
for name in named_scalars:
scalar = named_scalars[name]
named_var = named_vars[name]
collect_op = tf.assign_add(named_var, scalar)
collect_ops.append(collect_op)
# collect counting variable
collect_count_op = tf.assign_add(count_var, 1.0)
collect_ops.append(collect_count_op)
collect_op = tf.group(*collect_ops, name="collect_op")
scale = 1.0 / (tf.cast(count_var, tf.float32) + 1.0)
gradients = [scale * (g + s)
for (g, s) in zip(gradients, slot_variables)]
for name in named_scalars:
named_scalars[name] = scale * (
named_scalars[name] + named_vars[name])
grand_norm = tf.global_norm(gradients)
param_norm = tf.global_norm(variables)
# Gradient clipping
if isinstance(params.clip_grad_norm or None, float):
gradients, _ = tf.clip_by_global_norm(gradients,
params.clip_grad_norm,
use_norm=grand_norm)
# Update variables
grads_and_vars = list(zip(gradients, variables))
train_op = optimizer.apply_gradients(grads_and_vars, global_step)
ops = {
"zero_op": zero_variables_op,
"collect_op": collect_op,
"train_op": train_op
}
# apply ema
if params.ema_decay > 0.:
tf.logging.info('Using Exp Moving Average to train the model with decay {}.'.format(params.ema_decay))
ema = tf.train.ExponentialMovingAverage(decay=params.ema_decay, num_updates=global_step)
ema_op = ema.apply(variables)
with tf.control_dependencies([ops['train_op']]):
ops['train_op'] = tf.group(ema_op)
bck_vars = _replicate_variables(variables, suffix="CTrainOpBackUpReplica")
ops['ema_backup_op'] = tf.group(*(tf.assign(bck, var.read_value())
for bck, var in zip(bck_vars, variables)))
ops['ema_restore_op'] = tf.group(*(tf.assign(var, bck.read_value())
for bck, var in zip(bck_vars, variables)))
ops['ema_assign_op'] = tf.group(*(tf.assign(var, ema.average(var).read_value())
for var in variables))
ret = named_scalars
ret.update({
"gradient_norm": grand_norm,
"parameter_norm": param_norm,
})
return ret, ops
def create_tf_placeholder(self):
"""Convenience method that produces a placeholder of the type and shape defined by the port.
Returns: a placeholder of same type, shape and name.
"""
return tf.placeholder(tf.as_dtype(self.dtype), self.shape, self.name)
def _get_dtype(dtype):
if dtype is None:
dtype = backend.floatx()
return tf.as_dtype(dtype)
def G_mapping(
latents_in, # First input: Latent vectors (Z) [minibatch, latent_size].
labels_in, # Second input: Conditioning labels [minibatch, label_size].
latent_size=512, # Latent vector (Z) dimensionality.
label_size=0, # Label dimensionality, 0 if no labels.
dlatent_size=512, # Disentangled latent (W) dimensionality.
dlatent_broadcast=None, # Output disentangled latent (W) as [minibatch, dlatent_size] or [minibatch, dlatent_broadcast, dlatent_size].
mapping_layers=8, # Number of mapping layers.
mapping_fmaps=512, # Number of activations in the mapping layers.
mapping_lrmul=0.01, # Learning rate multiplier for the mapping layers.
mapping_nonlinearity='lrelu', # Activation function: 'relu', 'lrelu'.
use_wscale=True, # Enable equalized learning rate?
normalize_latents=True, # Normalize latent vectors (Z) before feeding them to the mapping layers?
dtype='float32', # Data type to use for activations and outputs.
**_kwargs): # Ignore unrecognized keyword args.
act, gain = {
'relu': (tf.nn.relu, np.sqrt(2)),
'lrelu': (leaky_relu, np.sqrt(2))
}[mapping_nonlinearity]
# Inputs.
latents_in.set_shape([None, latent_size])
labels_in.set_shape([None, label_size])
latents_in = tf.cast(latents_in, dtype)
labels_in = tf.cast(labels_in, dtype)
x = latents_in
# Embed labels and concatenate them with latents.
if label_size:
with tf.variable_scope('LabelConcat'):
w = tf.get_variable('weight',
shape=[label_size, latent_size],
initializer=tf.initializers.random_normal())
y = tf.matmul(labels_in, tf.cast(w, dtype))
x = tf.concat([x, y], axis=1)
# Normalize latents.
if normalize_latents:
x = pixel_norm(x)
# Mapping layers.
for layer_idx in range(mapping_layers):
with tf.variable_scope('Dense%d' % layer_idx):
fmaps = dlatent_size if layer_idx == mapping_layers - 1 else mapping_fmaps
x = dense(x,
fmaps=fmaps,
gain=gain,
use_wscale=use_wscale,
lrmul=mapping_lrmul)
x = apply_bias(x, lrmul=mapping_lrmul)
x = act(x)
# Broadcast.
if dlatent_broadcast is not None:
with tf.variable_scope('Broadcast'):
x = tf.tile(x[:, np.newaxis], [1, dlatent_broadcast, 1])
# Output.
assert x.dtype == tf.as_dtype(dtype)
return tf.identity(x, name='dlatents_out')
def _get_tf_dtype(space):
if isinstance(space, gym.spaces.Discrete):
return tf.int32
if isinstance(space, gym.spaces.Box):
return tf.as_dtype(space.dtype)
raise NotImplementedError()
def G_paper(
latents_in, # First input: Latent vectors [minibatch, latent_size].
labels_in, # Second input: Labels [minibatch, label_size].
probimages_in, # Third input: probimages [minibatch, 1, resolution, resolution].
num_channels=1, # Number of output color channels. Overridden based on dataset.
resolution=64, # Output resolution. Overridden based on dataset.
label_size=0, # Dimensionality of the labels, 0 if no labels. Overridden based on dataset.
fmap_base=2048, # Overall multiplier for the number of feature maps.
fmap_decay=1.0, # log2 feature map reduction when doubling the resolution.
fmap_max=128, # Maximum number of feature maps in any layer.
latent_size=None, # Dimensionality of the latent vectors. None = min(fmap_base, fmap_max).
normalize_latents=False, # Normalize latent vectors before feeding them to the network?
use_wscale=True, # Enable equalized learning rate?
use_pixelnorm=True, # Enable pixelwise feature vector normalization?
pixelnorm_epsilon=1e-8, # Constant epsilon for pixelwise feature vector normalization.
use_leakyrelu=True, # True = leaky ReLU, False = ReLU.
dtype='float32', # Data type to use for activations and outputs.
fused_scale=False, # True = use fused upscale2d + conv2d, False = separate upscale2d layers.
structure=None, # 'linear' = human-readable, 'recursive' = efficient, None = select automatically.
is_template_graph=False, # True = template graph constructed by the Network class, False = actual evaluation.
labels_augment=False, #True, # Augment labels or not.
labels_augment_times=8, # Augment labels by how many times. e.g., [0.3, 0.9]---> [0.3, 0.3, 0.3, 0.9, 0.9, 0.9], so that more attention would be put on labels.
prob_conv_channels=16,
**kwargs): # Ignore unrecognized keyword args.
resolution_log2 = int(np.log2(resolution))
assert resolution == 2**resolution_log2 and resolution >= 4
def nf(stage):
return min(int(fmap_base / (2.0**(stage * fmap_decay))), fmap_max)
def PN(x):
return pixel_norm(x, epsilon=pixelnorm_epsilon) if use_pixelnorm else x
if latent_size is None: latent_size = nf(0)
if structure is None:
structure = 'linear' if is_template_graph else 'recursive'
act = leaky_relu if use_leakyrelu else tf.nn.relu
latents_in.set_shape([None, latent_size])
labels_in.set_shape([None, label_size])
probimages_in.set_shape([None, 1, resolution, resolution])
probimages_in = tf.cast(probimages_in, tf.float32)
if labels_augment:
labels_in = labels_augment_func(labels_in, label_size,
labels_augment_times)
combo_in = tf.cast(tf.concat([latents_in, labels_in], axis=1), dtype)
lod_in = tf.cast(
tf.get_variable('lod', initializer=np.float32(0.0), trainable=False),
dtype)
# Building blocks.
def block(x, prob, res): # res = 2..resolution_log2
with tf.variable_scope('%dx%d' % (2**res, 2**res)):
if res == 2: # 4x4
if normalize_latents:
x = pixel_norm(x, epsilon=pixelnorm_epsilon)
with tf.variable_scope('Dense'):
x = dense(
x,
fmaps=nf(res - 1) * 16,
gain=np.sqrt(2) / 4,
use_wscale=use_wscale
) # override gain to match the original Theano implementation
x = tf.reshape(x, [-1, nf(res - 1), 4, 4])
x = PN(act(apply_bias(x)))
with tf.variable_scope('Add_Prob'):
prob_downscaled = downscale2d(prob,
factor=int(resolution /
(2**res)))
prob_downscaled_conv = apply_bias(
conv2d(prob_downscaled,
fmaps=prob_conv_channels,
kernel=1,
gain=1,
use_wscale=use_wscale))
x = tf.concat([x, prob_downscaled_conv], axis=1)
with tf.variable_scope('Conv'):
x = PN(
act(
apply_bias(
conv2d(x,
fmaps=nf(res - 1),
kernel=3,
use_wscale=use_wscale))))
else: # 8x8 and up
x = upscale2d(x)
with tf.variable_scope('Add_Prob'):
prob_downscaled = downscale2d(prob,
factor=int(resolution /
(2**res)))
prob_downscaled_conv = apply_bias(
conv2d(prob_downscaled,
fmaps=prob_conv_channels,
kernel=1,
gain=1,
use_wscale=use_wscale))
x = tf.concat([x, prob_downscaled_conv], axis=1)
with tf.variable_scope('Conv0'):
x = PN(
act(
apply_bias(
conv2d(x,
fmaps=nf(res - 1),
kernel=3,
use_wscale=use_wscale))))
with tf.variable_scope('Conv1'):
x = PN(
act(
apply_bias(
conv2d(x,
fmaps=nf(res - 1),
kernel=3,
use_wscale=use_wscale))))
return x
def torgb(x, res): # res = 2..resolution_log2
lod = resolution_log2 - res
with tf.variable_scope('ToRGB_lod%d' % lod):
return apply_bias(
conv2d(x,
fmaps=num_channels,
kernel=1,
gain=1,
use_wscale=use_wscale))
# Linear structure: simple but inefficient.
if structure == 'linear':
x = block(combo_in, probimages_in, 2)
images_out = torgb(x, 2)
for res in range(3, resolution_log2 + 1):
lod = resolution_log2 - res
x = block(x, probimages_in, res)
img = torgb(x, res)
images_out = upscale2d(images_out)
with tf.variable_scope('Grow_lod%d' % lod):
images_out = lerp_clip(img, images_out, lod_in - lod)
# Recursive structure: complex but efficient.
if structure == 'recursive':
def grow(x, prob, res, lod):
y = block(x, prob, res)
img = lambda: upscale2d(torgb(y, res), 2**lod)
if res > 2:
img = cset(
img, (lod_in > lod), lambda: upscale2d(
lerp(torgb(y, res), upscale2d(torgb(x, res - 1)),
lod_in - lod), 2**lod))
if lod > 0:
img = cset(img, (lod_in < lod),
lambda: grow(y, prob, res + 1, lod - 1))
return img()
images_out = grow(combo_in, probimages_in, 2, resolution_log2 - 2)
assert images_out.dtype == tf.as_dtype(dtype)
images_out = tf.identity(images_out, name='images_out')
return images_out
def __init__(self, dtype):
self._dtype = tf.as_dtype(dtype)
def D_stylegan2(
images_in, # First input: Images [minibatch, channel, height, width].
labels_in, # Second input: Labels [minibatch, label_size].
num_channels=3, # Number of input color channels. Overridden based on dataset.
resolution=256, # Input resolution. Overridden based on dataset.
label_size=127, # Dimensionality of the labels, 0 if no labels. Overridden based on dataset.
fmap_base=16 <<
10, # Overall multiplier for the number of feature maps.
fmap_decay=1.0, # log2 feature map reduction when doubling the resolution.
fmap_min=1, # Minimum number of feature maps in any layer.
fmap_max=512, # Maximum number of feature maps in any layer.
architecture='resnet', # Architecture: 'orig', 'skip', 'resnet'.
nonlinearity='lrelu', # Activation function: 'relu', 'lrelu', etc.
mbstd_group_size=4, # Group size for the minibatch standard deviation layer, 0 = disable.
mbstd_num_features=1, # Number of features for the minibatch standard deviation layer.
dtype='float32', # Data type to use for activations and outputs.
resample_kernel=[
1, 3, 3, 1
], # Low-pass filter to apply when resampling activations. None = no filtering.
dlabel_size=32,
**_kwargs): # Ignore unrecognized keyword args.
resolution_log2 = int(np.log2(resolution))
assert resolution == 2**resolution_log2 and resolution >= 4
def nf(stage):
return np.clip(int(fmap_base / (2.0**(stage * fmap_decay))), fmap_min,
fmap_max)
assert architecture in ['orig', 'skip', 'resnet']
act = nonlinearity
dlabel = D_mapping_label(labels_in=labels_in,
label_size=label_size,
dlabel_size=dlabel_size)
images_in.set_shape([None, num_channels, resolution, resolution])
labels_in.set_shape([None, label_size])
dlabel.set_shape([None, dlabel_size])
images_in = tf.cast(images_in, dtype)
labels_in = tf.cast(labels_in, dtype)
dlabel = tf.cast(dlabel, dtype)
# Building blocks for main layers.
def fromrgb(x, y, res): # res = 2..resolution_log2
with tf.variable_scope('FromRGB'):
t = apply_bias_act(conv2d_layer(y, fmaps=nf(res - 1), kernel=1),
act=act)
return t if x is None else x + t
def downsample(y):
with tf.variable_scope('Downsample'):
return downsample_2d(y, k=resample_kernel)
def block(x, res): # res = 2..resolution_log2
t = x
v_x_upsample = tf.constant(0)
with tf.variable_scope('Conv0'):
x = apply_bias_act(conv2d_layer(x, fmaps=nf(res - 1), kernel=3),
act=act)
with tf.variable_scope('Conv1_down'):
x = apply_bias_act(conv2d_layer(x,
fmaps=nf(res - 2),
kernel=3,
down=True,
resample_kernel=resample_kernel),
act=act)
if 3 < res < 8:
with tf.variable_scope('Downsample'):
x_downsample = downsample_2d(x)
height = x_downsample.shape[2]
width = x_downsample.shape[3]
c_reduced = 1
label_mapping_fmaps = 16
with tf.variable_scope('F_Attention'):
f_x = conv2d_layer(x_downsample, fmaps=1, kernel=1)
f_x = tf.reshape(f_x, [-1, height * width])
with tf.variable_scope('Label_F_Attention'):
label_f = dense_layer(dlabel, fmaps=label_mapping_fmaps) + 1
with tf.variable_scope('F_concat_Attention'):
f_x_s = dense_layer(tf.concat([f_x, label_f], axis=-1),
fmaps=c_reduced * height * width)
f_x_s = tf.reshape(f_x_s, [-1, c_reduced, height * width])
f_x_s = tf.transpose(f_x_s, perm=[0, 2, 1])
with tf.variable_scope('G_Attention'):
g_x = conv2d_layer(x_downsample, fmaps=1, kernel=1)
g_x = tf.reshape(g_x, [-1, height * width])
with tf.variable_scope('Label_G_Attention'):
label_g = dense_layer(dlabel, fmaps=label_mapping_fmaps) + 1
with tf.variable_scope('G_concat_Attention'):
g_x_s = dense_layer(tf.concat([g_x, label_g], axis=-1),
fmaps=c_reduced * height * width)
g_x_s = tf.reshape(g_x_s, [-1, c_reduced, height * width])
with tf.variable_scope('H_Attention'):
h_x = conv2d_layer(x_downsample, fmaps=c_reduced, kernel=1)
h_x = tf.reshape(h_x, [-1, c_reduced, height * width])
f_g_multiply = tf.matmul(f_x_s, g_x_s)
attention_map = tf.nn.softmax(f_g_multiply, axis=-1)
attention_map_h_multiply = tf.matmul(
h_x, tf.transpose(attention_map, [0, 2, 1]))
attention_map_h_multiply_reshape = tf.reshape(
attention_map_h_multiply, [-1, c_reduced, height, width])
with tf.variable_scope('V_Attention'):
v_x = conv2d_layer(attention_map_h_multiply_reshape,
fmaps=x_downsample.shape[1],
kernel=1)
with tf.variable_scope('Upsample'):
v_x_upsample = upsample_2d(v_x)
with tf.variable_scope('Gamma_Attention'):
gamma = tf.get_variable(shape=[],
initializer=tf.initializers.zeros(),
name='attention_gamma')
x = x + v_x_upsample * gamma
if architecture == 'resnet':
with tf.variable_scope('Skip'):
t = conv2d_layer(t,
fmaps=nf(res - 2),
kernel=1,
down=True,
resample_kernel=resample_kernel)
x = (x + t) * (1 / np.sqrt(2))
return x, v_x_upsample
# Main layers.
x = None
y = images_in
for res in range(resolution_log2, 2, -1):
with tf.variable_scope('%dx%d' % (2**res, 2**res)):
if architecture == 'skip' or res == resolution_log2:
x = fromrgb(x, y, res)
x, attention_map = block(x, res)
if res == 3:
fmap_out = x
current_attention_map = attention_map
if architecture == 'skip':
y = downsample(y)
# Final layers.
with tf.variable_scope('4x4'):
if architecture == 'skip':
x = fromrgb(x, y, 2)
if mbstd_group_size > 1:
with tf.variable_scope('MinibatchStddev'):
x = minibatch_stddev_layer(x, mbstd_group_size,
mbstd_num_features)
with tf.variable_scope('Conv'):
x = apply_bias_act(conv2d_layer(x, fmaps=nf(1), kernel=3), act=act)
with tf.variable_scope('Dense0'):
x = apply_bias_act(dense_layer(x, fmaps=nf(0)), act=act)
# Output layer with label conditioning from "Which Training Methods for GANs do actually Converge?"
with tf.variable_scope('Output'):
x = apply_bias_act(dense_layer(x, fmaps=max(labels_in.shape[1], 1)))
return x, fmap_out, current_attention_map
if labels_in.shape[1] > 0:
x = tf.reduce_sum(x * labels_in, axis=1, keepdims=True)
scores_out = x
# Output.
assert scores_out.dtype == tf.as_dtype(dtype)
scores_out = tf.identity(scores_out, name='scores_out')
return scores_out
#----------------------------------------------------------------------------
def D_basic(
images_in, # First input: Images [minibatch, channel, height, width].
labels_in, # Second input: Labels [minibatch, label_size].
num_channels=1, # Number of input color channels. Overridden based on dataset.
resolution=32, # Input resolution. Overridden based on dataset.
label_size=0, # Dimensionality of the labels, 0 if no labels. Overridden based on dataset.
aux_output_size=0, # Auxilliary output
fmap_base=8192, # Overall multiplier for the number of feature maps.
fmap_decay=1.0, # log2 feature map reduction when doubling the resolution.
fmap_max=512, # Maximum number of feature maps in any layer.
nonlinearity='lrelu', # Activation function: 'relu', 'lrelu',
use_wscale=True, # Enable equalized learning rate?
mbstd_group_size=4, # Group size for the minibatch standard deviation layer, 0 = disable.
mbstd_num_features=1, # Number of features for the minibatch standard deviation layer.
dtype='float32', # Data type to use for activations and outputs.
fused_scale='auto', # True = fused convolution + scaling, False = separate ops, 'auto' = decide automatically.
blur_filter=[
1, 2, 1
], # Low-pass filter to apply when resampling activations. None = no filtering.
structure='auto', # 'fixed' = no progressive growing, 'linear' = human-readable, 'recursive' = efficient, 'auto' = select automatically.
is_template_graph=False, # True = template graph constructed by the Network class, False = actual evaluation.
**_kwargs): # Ignore unrecognized keyword args.
resolution_log2 = int(np.log2(resolution))
assert resolution == 2**resolution_log2 and resolution >= 4
assert label_size == 0 or aux_output_size == 0
def nf(stage):
return min(int(fmap_base / (2.0**(stage * fmap_decay))), fmap_max)
def blur(x):
return blur2d(x, blur_filter) if blur_filter else x
if structure == 'auto':
structure = 'linear' if is_template_graph else 'recursive'
act, gain = {
'relu': (tf.nn.relu, np.sqrt(2)),
'lrelu': (leaky_relu, np.sqrt(2))
}[nonlinearity]
images_in.set_shape([None, num_channels, resolution, resolution])
labels_in.set_shape([None, label_size])
images_in = tf.cast(images_in, dtype)
labels_in = tf.cast(labels_in, dtype)
lod_in = tf.cast(
tf.get_variable('lod', initializer=np.float32(0.0), trainable=False),
dtype)
scores_out = None
# Building blocks.
def fromrgb(x, res): # res = 2..resolution_log2
with tf.variable_scope('FromRGB_lod%d' % (resolution_log2 - res)):
return act(
apply_bias(
conv2d(x,
fmaps=nf(res - 1),
kernel=1,
gain=gain,
use_wscale=use_wscale)))
def block(x, res): # res = 2..resolution_log2
with tf.variable_scope('%dx%d' % (2**res, 2**res)):
if res >= 3: # 8x8 and up
with tf.variable_scope('Conv0'):
x = act(
apply_bias(
conv2d(x,
fmaps=nf(res - 1),
kernel=3,
gain=gain,
use_wscale=use_wscale)))
with tf.variable_scope('Conv1_down'):
x = act(
apply_bias(
conv2d_downscale2d(blur(x),
fmaps=nf(res - 2),
kernel=3,
gain=gain,
use_wscale=use_wscale,
fused_scale=fused_scale)))
else: # 4x4
if mbstd_group_size > 1:
x = minibatch_stddev_layer(x, mbstd_group_size,
mbstd_num_features)
with tf.variable_scope('Conv'):
x = act(
apply_bias(
conv2d(x,
fmaps=nf(res - 1),
kernel=3,
gain=gain,
use_wscale=use_wscale)))
with tf.variable_scope('Dense0'):
x = act(
apply_bias(
dense(x,
fmaps=nf(res - 2),
gain=gain,
use_wscale=use_wscale)))
with tf.variable_scope('Dense1'):
x = apply_bias(
dense(x,
fmaps=max(label_size + aux_output_size, 1),
gain=1,
use_wscale=use_wscale))
return x
# Fixed structure: simple and efficient, but does not support progressive growing.
if structure == 'fixed':
x = fromrgb(images_in, resolution_log2)
for res in range(resolution_log2, 2, -1):
x = block(x, res)
scores_out = block(x, 2)
# Linear structure: simple but inefficient.
if structure == 'linear':
img = images_in
x = fromrgb(img, resolution_log2)
for res in range(resolution_log2, 2, -1):
lod = resolution_log2 - res
x = block(x, res)
img = downscale2d(img)
y = fromrgb(img, res - 1)
with tf.variable_scope('Grow_lod%d' % lod):
x = tflib.lerp_clip(x, y, lod_in - lod)
scores_out = block(x, 2)
# Recursive structure: complex but efficient.
if structure == 'recursive':
def cset(cur_lambda, new_cond, new_lambda):
return lambda: tf.cond(new_cond, new_lambda, cur_lambda)
def grow(res, lod):
x = lambda: fromrgb(downscale2d(images_in, 2**lod), res)
if lod > 0:
x = cset(x, (lod_in < lod), lambda: grow(res + 1, lod - 1))
x = block(x(), res)
y = lambda: x
if res > 2:
y = cset(
y, (lod_in > lod), lambda: tflib.lerp(
x,
fromrgb(downscale2d(images_in, 2**(lod + 1)), res - 1),
lod_in - lod))
return y()
scores_out = grow(2, resolution_log2 - 2)
# Label conditioning from "Which Training Methods for GANs do actually Converge?"
if label_size:
with tf.variable_scope('LabelSwitch'):
scores_out = tf.reduce_sum(scores_out * labels_in,
axis=1,
keepdims=True)
assert scores_out.dtype == tf.as_dtype(dtype)
scores_out = tf.identity(scores_out, name='scores_out')
return scores_out
def _setup_training(self):
"""Sets up graph, model and trainer."""
self._graph = tf.Graph()
self._graph.add_to_collection("IS_TRAINING", True)
with self._graph.as_default():
tf.set_random_seed(self.tf_random_seed)
self._global_step = tf.Variable(0,
name="global_step",
trainable=False)
# Setting up input and output placeholders.
input_shape = [None] + self._data_feeder.input_shape[1:]
output_shape = [None] + self._data_feeder.output_shape[1:]
self._inp = tf.placeholder(tf.as_dtype(
self._data_feeder.input_dtype),
input_shape,
name="input")
self._out = tf.placeholder(tf.as_dtype(
self._data_feeder.output_dtype),
output_shape,
name="output")
# If class weights are provided, add them to the graph.
# Different loss functions can use this tensor by name.
if self.class_weight:
self._class_weight_node = tf.constant(self.class_weight,
name='class_weight')
# Add histograms for X and y if they are floats.
if self._data_feeder.input_dtype in (np.float32, np.float64):
tf.histogram_summary("X", self._inp)
if self._data_feeder.output_dtype in (np.float32, np.float64):
tf.histogram_summary("y", self._out)
# Create model's graph.
self._model_predictions, self._model_loss = self.model_fn(
self._inp, self._out)
# Create summary to monitor loss
tf.scalar_summary("loss", self._model_loss)
# Set up a single operator to merge all the summaries
self._summaries = tf.merge_all_summaries()
# Create trainer and augment graph with gradients and optimizer.
# Additionally creates initialization ops.
self._trainer = TensorFlowTrainer(loss=self._model_loss,
global_step=self._global_step,
optimizer=self.optimizer,
learning_rate=self.learning_rate)
# Create model's saver capturing all the nodes created up until now.
self._saver = tf.train.Saver(max_to_keep=self.max_to_keep,
keep_checkpoint_every_n_hours=self.
keep_checkpoint_every_n_hours)
# Create session to run model with.
self._session = tf.Session(
self.tf_master,
config=tf.ConfigProto(
log_device_placement=self.verbose > 1,
inter_op_parallelism_threads=self.num_cores,
intra_op_parallelism_threads=self.num_cores))
def slice_dtype(self):
return tf.as_dtype(self._hparams.slice_dtype)
import argparse
import h5py
import numpy as np
import tensorflow as tf
import nobrainer
from nobrainer.io import read_mapping
from nobrainer.preprocessing import binarize, preprocess_aparcaseg
from nobrainer.util import iter_hdf5, input_fn_builder
# Data types of labels (x) and features (y).
DT_X = 'float32'
DT_Y = 'int32'
_DT_X_NP = np.dtype(DT_X)
_DT_X_TF = tf.as_dtype(DT_X)
_DT_Y_NP = np.dtype(DT_Y)
_DT_Y_TF = tf.as_dtype(DT_Y)
def train(params):
"""Train estimator."""
x_dataset = params['xdset']
y_dataset = params['ydset']
tf.logging.info('Using features dataset {x} and labels dataset {y}'.format(
x=x_dataset, y=y_dataset))
with h5py.File(params['hdf5path'], mode='r') as fp:
examples_per_epoch = fp[x_dataset].shape[0]
def D_stylegan2(
images_in, # First input: Images [minibatch, channel, height, width].
labels_in, # Second input: Labels [minibatch, label_size].
num_channels=3, # Number of input color channels. Overridden based on dataset.
resolution=1024, # Input resolution. Overridden based on dataset.
label_size=0, # Dimensionality of the labels, 0 if no labels. Overridden based on dataset.
fmap_base=16 << 10, # Overall multiplier for the number of feature maps.
fmap_decay=1.0, # log2 feature map reduction when doubling the resolution.
fmap_min=1, # Minimum number of feature maps in any layer.
fmap_max=512, # Maximum number of feature maps in any layer.
architecture='resnet', # Architecture: 'orig', 'skip', 'resnet'.
nonlinearity='lrelu', # Activation function: 'relu', 'lrelu', etc.
mbstd_group_size=4, # Group size for the minibatch standard deviation layer, 0 = disable.
mbstd_num_features=1, # Number of features for the minibatch standard deviation layer.
dtype='float32', # Data type to use for activations and outputs.
resample_kernel=[
1, 3, 3, 1
], # Low-pass filter to apply when resampling activations. None = no filtering.
**_kwargs): # Ignore unrecognized keyword args.
resolution_log2 = int(np.log2(resolution))
assert resolution == 2**resolution_log2 and resolution >= 4
def nf(stage):
return np.clip(int(fmap_base / (2.0**(stage * fmap_decay))), fmap_min,
fmap_max)
assert architecture in ['orig', 'skip', 'resnet']
act = nonlinearity
images_in.set_shape([None, num_channels, resolution, resolution])
labels_in.set_shape([None, label_size])
images_in = tf.cast(images_in, dtype)
labels_in = tf.cast(labels_in, dtype)
# Building blocks for main layers.
def fromrgb(x, y, res): # res = 2..resolution_log2
with tf.variable_scope('FromRGB'):
t = apply_bias_act(conv2d_layer(y, fmaps=nf(res - 1), kernel=1),
act=act)
return t if x is None else x + t
def block(x, res): # res = 2..resolution_log2
t = x
with tf.variable_scope('Conv0'):
x = apply_bias_act(conv2d_layer(x, fmaps=nf(res - 1), kernel=3),
act=act)
with tf.variable_scope('Conv1_down'):
x = apply_bias_act(conv2d_layer(x,
fmaps=nf(res - 2),
kernel=3,
down=True,
resample_kernel=resample_kernel),
act=act)
if architecture == 'resnet':
with tf.variable_scope('Skip'):
t = conv2d_layer(t,
fmaps=nf(res - 2),
kernel=1,
down=True,
resample_kernel=resample_kernel)
x = (x + t) * (1 / np.sqrt(2))
return x
def downsample(y):
with tf.variable_scope('Downsample'):
return downsample_2d(y, k=resample_kernel)
# Main layers.
x = None
y = images_in
for res in range(resolution_log2, 2, -1):
with tf.variable_scope('%dx%d' % (2**res, 2**res)):
if architecture == 'skip' or res == resolution_log2:
x = fromrgb(x, y, res)
x = block(x, res)
if architecture == 'skip':
y = downsample(y)
# Final layers.
with tf.variable_scope('4x4'):
if architecture == 'skip':
x = fromrgb(x, y, 2)
if mbstd_group_size > 1:
with tf.variable_scope('MinibatchStddev'):
x = minibatch_stddev_layer(x, mbstd_group_size,
mbstd_num_features)
with tf.variable_scope('Conv'):
x = apply_bias_act(conv2d_layer(x, fmaps=nf(1), kernel=3), act=act)
with tf.variable_scope('Dense0'):
x = apply_bias_act(dense_layer(x, fmaps=nf(0)), act=act)
# Output layer with label conditioning from "Which Training Methods for GANs do actually Converge?"
with tf.variable_scope('Output'):
x = apply_bias_act(dense_layer(x, fmaps=max(labels_in.shape[1], 1)))
if labels_in.shape[1] > 0:
x = tf.reduce_sum(x * labels_in, axis=1, keepdims=True)
scores_out = x
# Output.
assert scores_out.dtype == tf.as_dtype(dtype)
scores_out = tf.identity(scores_out, name='scores_out')
return scores_out
def sample_bounded_spec(spec, seed=None, outer_dims=None):
"""Samples uniformily the given bounded spec.
Args:
spec: A BoundedSpec to sample.
seed: A seed used for sampling ops
outer_dims: An optional `Tensor` specifying outer dimensions to add to the
spec shape before sampling.
Returns:
A Tensor sample of the requested spec.
"""
minval = spec.minimum
maxval = spec.maximum
dtype = tf.as_dtype(spec.dtype)
# To sample uint8 we will use int32 and cast later. This is needed for two
# reasons:
# - tf.random_uniform does not currently support uint8
# - if you want to sample [0, 255] range, there's no way to do this since
# tf.random_uniform has exclusive upper bound and 255 + 1 would overflow.
is_uint8 = dtype == tf.uint8
sampling_dtype = tf.int32 if is_uint8 else dtype
if dtype in [tf.float64, tf.float32]:
# Avoid under/over-flow as random_uniform can't sample over the full range
# for these types.
minval = np.maximum(dtype.min / 8, minval)
maxval = np.minimum(dtype.max / 8, maxval)
if outer_dims is None:
outer_dims = tf.constant([], dtype=tf.int32)
else:
outer_dims = tf.convert_to_tensor(outer_dims, dtype=tf.int32)
def _unique_vals(vals):
if vals.size > 0:
if vals.ndim > 0:
return np.all(vals == vals[0])
return True
if (minval.ndim != 0 or maxval.ndim != 0) and not (_unique_vals(minval) and
_unique_vals(maxval)):
# tf.random_uniform can only handle minval/maxval 0-d tensors.
res = _random_uniform_int(shape=spec.shape,
outer_dims=outer_dims,
minval=minval,
maxval=maxval,
dtype=sampling_dtype,
seed=seed)
else:
minval = minval.item(0) if minval.ndim != 0 else minval
maxval = maxval.item(0) if maxval.ndim != 0 else maxval
# BoundedTensorSpec are bounds inclusive.
# tf.random_uniform is upper bound exclusive, +1 to fix the sampling
# behavior.
# However +1 will cause overflow, in such cases we use the original maxval.
if sampling_dtype.is_integer and maxval < sampling_dtype.max:
maxval = maxval + 1
shape = tf.convert_to_tensor(spec.shape, dtype=tf.int32)
full_shape = tf.concat((outer_dims, shape), axis=0)
res = tf.random.uniform(full_shape,
minval=minval,
maxval=maxval,
dtype=sampling_dtype,
seed=seed)
if is_uint8:
res = tf.cast(res, dtype=dtype)
return res
def backward_tensor(self, y):
ys = tf.maximum(y - self._lower, tf.as_dtype(settings.float_type).min)
return ys + tf.log(-tf.expm1(-ys))
from cleverhansl2l import utils
from cleverhansl2l.attacks.attack import Attack
from cleverhansl2l.attacks.basic_iterative_method import BasicIterativeMethod
from cleverhansl2l.attacks.carlini_wagner_l2 import CarliniWagnerL2
from cleverhansl2l.attacks.deep_fool import DeepFool
from cleverhansl2l.attacks.elastic_net_method import ElasticNetMethod
from cleverhansl2l.attacks.fast_feature_adversaries import FastFeatureAdversaries
from cleverhansl2l.attacks.fast_gradient_method import FastGradientMethod, fgm, optimize_linear
from cleverhansl2l.attacks.lbfgs import LBFGS
from cleverhansl2l.attacks.madry_et_al import MadryEtAl
from cleverhansl2l.attacks.max_confidence import MaxConfidence
from cleverhansl2l.attacks.momentum_iterative_method import MomentumIterativeMethod
from cleverhansl2l.attacks.noise import Noise
from cleverhansl2l.attacks.projected_gradient_descent import ProjectedGradientDescent
from cleverhansl2l.attacks.saliency_map_method import SaliencyMapMethod
from cleverhansl2l.attacks.semantic import Semantic
from cleverhansl2l.attacks.spsa import SPSA, projected_optimization
from cleverhansl2l.attacks.spatial_transformation_method import SpatialTransformationMethod
from cleverhansl2l.attacks.virtual_adversarial_method import VirtualAdversarialMethod, vatm
from cleverhansl2l.model import Model, CallableModelWrapper
from cleverhansl2l.model import wrapper_warning, wrapper_warning_logits
from cleverhansl2l.compat import reduce_sum, reduce_mean
from cleverhansl2l.compat import reduce_max
from cleverhansl2l.compat import softmax_cross_entropy_with_logits
from cleverhansl2l.utils_tf import clip_eta
from cleverhansl2l import utils_tf
_logger = utils.create_logger("cleverhans.attacks")
tf_dtype = tf.as_dtype('float32')
def __init__(self, fan_in=None, a=5, seed=None, dtype=tf.float32):
self.fan_in = fan_in
self.a = a
self.seed = seed
self.dtype = tf.as_dtype(dtype)
def D_paper(
images_in, # Input: Images [minibatch, channel, height, width].
num_channels=1, # Number of input color channels. Overridden based on dataset.
resolution=32, # Input resolution. Overridden based on dataset.
label_size=0, # Dimensionality of the labels, 0 if no labels. Overridden based on dataset.
fmap_base=8192, # Overall multiplier for the number of feature maps.
fmap_decay=1.0, # log2 feature map reduction when doubling the resolution.
fmap_max=512, # Maximum number of feature maps in any layer.
use_wscale=True, # Enable equalized learning rate?
mbstd_group_size=4, # Group size for the minibatch standard deviation layer, 0 = disable.
dtype='float32', # Data type to use for activations and outputs.
fused_scale=True, # True = use fused conv2d + downscale2d, False = separate downscale2d layers.
structure=None, # 'linear' = human-readable, 'recursive' = efficient, None = select automatically
is_template_graph=False, # True = template graph constructed by the Network class, False = actual evaluation.
**kwargs): # Ignore unrecognized keyword args.
resolution_log2 = int(np.log2(resolution))
assert resolution == 2**resolution_log2 and resolution >= 4
def nf(stage):
return min(int(fmap_base / (2.0**(stage * fmap_decay))), fmap_max)
if structure is None:
structure = 'linear' if is_template_graph else 'recursive'
act = leaky_relu
images_in.set_shape([None, num_channels, resolution, resolution])
images_in = tf.cast(images_in, dtype)
lod_in = tf.cast(
tf.get_variable('lod', initializer=np.float32(0.0), trainable=False),
dtype)
# Building blocks.
def fromrgb(x, res): # res = 2..resolution_log2
with tf.variable_scope('FromRGB_lod%d' % (resolution_log2 - res)):
return act(
apply_bias(
conv2d(x,
fmaps=nf(res - 1),
kernel=1,
use_wscale=use_wscale)))
def block(x, res): # res = 2..resolution_log2
with tf.variable_scope('%dx%d' % (2**res, 2**res)):
if res >= 3: # 8x8 and up
with tf.variable_scope('Conv0'):
x = act(
apply_bias(
conv2d(x,
fmaps=nf(res - 1),
kernel=3,
use_wscale=use_wscale)))
if fused_scale:
with tf.variable_scope('Conv1_down'):
x = act(
apply_bias(
conv2d_downscale2d(x,
fmaps=nf(res - 2),
kernel=3,
use_wscale=use_wscale)))
else:
with tf.variable_scope('Conv1'):
x = act(
apply_bias(
conv2d(x,
fmaps=nf(res - 2),
kernel=3,
use_wscale=use_wscale)))
x = downscale2d(x)
else: # 4x4
if mbstd_group_size > 1:
x = minibatch_stddev_layer(x, mbstd_group_size)
with tf.variable_scope('Conv'):
x = act(
apply_bias(
conv2d(x,
fmaps=nf(res - 1),
kernel=3,
use_wscale=use_wscale)))
with tf.variable_scope('Dense0'):
x = act(
apply_bias(
dense(x, fmaps=nf(res - 2),
use_wscale=use_wscale)))
with tf.variable_scope('Dense1'):
x = apply_bias(
dense(x,
fmaps=1 + label_size,
gain=1,
use_wscale=use_wscale))
return x
# Linear structure: simple but inefficient.
if structure == 'linear':
img = images_in
x = fromrgb(img, resolution_log2)
for res in range(resolution_log2, 2, -1):
lod = resolution_log2 - res
x = block(x, res)
img = downscale2d(img)
y = fromrgb(img, res - 1)
with tf.variable_scope('Grow_lod%d' % lod):
x = lerp_clip(x, y, lod_in - lod)
combo_out = block(x, 2)
# Recursive structure: complex but efficient.
if structure == 'recursive':
def grow(res, lod):
x = lambda: fromrgb(downscale2d(images_in, 2**lod), res)
if lod > 0:
x = cset(x, (lod_in < lod), lambda: grow(res + 1, lod - 1))
x = block(x(), res)
y = lambda: x
if res > 2:
y = cset(
y, (lod_in > lod), lambda: lerp(
x,
fromrgb(downscale2d(images_in, 2**(lod + 1)), res - 1),
lod_in - lod))
return y()
combo_out = grow(2, resolution_log2 - 2)
assert combo_out.dtype == tf.as_dtype(dtype)
scores_out = tf.identity(combo_out[:, :1], name='scores_out')
labels_out = tf.identity(combo_out[:, 1:], name='labels_out')
return scores_out, labels_out
def __init__(self, shape, dtype, minimum=None, maximum=None, name=None):
"""Initializes a new `BoundedArraySpec`.
Args:
shape: An iterable specifying the array shape.
dtype: numpy dtype or string specifying the array dtype.
minimum: Number or sequence specifying the maximum element bounds
(inclusive). Must be broadcastable to `shape`.
maximum: Number or sequence specifying the maximum element bounds
(inclusive). Must be broadcastable to `shape`.
name: Optional string containing a semantic name for the corresponding
array. Defaults to `None`.
Raises:
ValueError: If `minimum` or `maximum` are not broadcastable to `shape` or
if the limits are outside of the range of the specified dtype.
TypeError: If the shape is not an iterable or if the `dtype` is an invalid
numpy dtype.
"""
super(BoundedArraySpec, self).__init__(shape, dtype, name)
try:
np.broadcast_to(minimum, shape=shape)
except ValueError as numpy_exception:
raise ValueError('minimum is not compatible with shape. '
'Message: {!r}.'.format(numpy_exception))
try:
np.broadcast_to(maximum, shape=shape)
except ValueError as numpy_exception:
raise ValueError('maximum is not compatible with shape. '
'Message: {!r}.'.format(numpy_exception))
tf_dtype = tf.as_dtype(self._dtype)
low = tf_dtype.min
high = tf_dtype.max
if minimum is None:
minimum = low
if maximum is None:
maximum = high
self._minimum = np.array(minimum)
self._maximum = np.array(maximum)
if tf_dtype.is_floating:
# Replacing infinities with extreme finite float values.
self._minimum[self._minimum == -np.inf] = low
self._minimum[self._minimum == np.inf] = high
self._maximum[self._maximum == -np.inf] = low
self._maximum[self._maximum == np.inf] = high
if np.any(self._minimum > self._maximum):
raise ValueError(
'Spec bounds min has values greater than max: [{},{}]'.format(
self._minimum, self._maximum))
if (np.any(self._minimum < low) or np.any(self._minimum > high)
or np.any(self._maximum < low)
or np.any(self._maximum > high)):
raise ValueError(
'Spec bounds [{},{}] not within the range [{}, {}] of the given '
'dtype ({})'.format(self._minimum, self._maximum, low, high,
self._dtype))
self._minimum = self._minimum.astype(self._dtype)
self._minimum.setflags(write=False)
self._maximum = self._maximum.astype(self._dtype)
self._maximum.setflags(write=False)
def do_testing(now_epoch, data_loader, best_value_history, indices_1, indices_2, list_train_loss, \
summary_writer, train_summary_datas, is_training):
timer = Timer()
timer.start()
test_avg = [0] * len(netG_test_summary_names)
#temporaly removed
#test_count = FLAGS['data_test_image_count'] if FLAGS['process_write_test_img_count'] == 0 or is_training == False else FLAGS['process_write_test_img_count']
test_count = FLAGS['data_test_image_count']
test_loss_list = np.zeros((len(netG_test_summary_names), test_count))
for i in range(test_count):
timer2 = Timer()
timer2.start()
label_img = data_loader.get_next_test_label()
input_img, data = data_loader.get_next_test_input_batch()
update_cache_dict(data['csr_ind1'], data['csr_val1'], data['csr_ind_r1'], data['csr_val_r1'], data['csr_ind_g1'], data['csr_val_g1'], data['csr_ind_b1'], data['csr_val_b1'], data['csr_names1'])
update_cache_dict(data['csr_ind2'], data['csr_val2'], data['csr_ind_r2'], data['csr_val_r2'], data['csr_ind_g2'], data['csr_val_g2'], data['csr_ind_b2'], data['csr_val_b2'], data['csr_names2'])
dict_d = [\
input_img, label_img] + \
data['rect1'] + data['rot1'] + \
data['rect2'] + data['rot2'] + \
data['csr_ind1'] + data['csr_val1'] + \
data['csr_ind_r1'] + data['csr_val_r1'] + \
data['csr_ind_g1'] + data['csr_val_g1'] + \
data['csr_ind_b1'] + data['csr_val_b1'] + data['csr_sha1'] + \
data['csr_ind2'] + data['csr_val2'] + \
data['csr_ind_r2'] + data['csr_val_r2'] + \
data['csr_ind_g2'] + data['csr_val_g2'] + \
data['csr_ind_b2'] + data['csr_val_b2'] + data['csr_sha2']
dict_t = [\
test_df.input1_src, test_df.input2_src] + \
test_df.mat1.rect + test_df.mat1.rot + \
test_df.mat2.rect + test_df.mat2.rot + \
test_df.mat1.csr_ind + test_df.mat1.csr_val + \
test_df.mat1.csr_ind_r + test_df.mat1.csr_val_r + \
test_df.mat1.csr_ind_g + test_df.mat1.csr_val_g + \
test_df.mat1.csr_ind_b + test_df.mat1.csr_val_b + test_df.mat1.csr_sha + \
test_df.mat2.csr_ind + test_df.mat2.csr_val + \
test_df.mat2.csr_ind_r + test_df.mat2.csr_val_r + \
test_df.mat2.csr_ind_g + test_df.mat2.csr_val_g + \
test_df.mat2.csr_ind_b + test_df.mat2.csr_val_b + test_df.mat2.csr_sha
if FLAGS['loss_constant_term_weight'] > 0:
test_s, enhance_test_img1, enhance_test_img1_inv, enhance_test_img2, enhance_test_img2_inv = sess.run([netG_test_summary, netG_test_output1_crop, netG_test_output1_inv_crop, netG_test_output2_crop, netG_test_output2_inv_crop], feed_dict={t:d for t, d in zip(dict_t, dict_d)})
enhance_test_img1 = safe_casting(enhance_test_img1 * tf.as_dtype(FLAGS['data_input_dtype']).max, FLAGS['data_input_dtype'])
enhance_test_img1_inv = safe_casting(enhance_test_img1_inv * tf.as_dtype(FLAGS['data_input_dtype']).max, FLAGS['data_input_dtype'])
enhance_test_img2 = safe_casting(enhance_test_img2 * tf.as_dtype(FLAGS['data_input_dtype']).max, FLAGS['data_input_dtype'])
enhance_test_img2_inv = safe_casting(enhance_test_img2_inv * tf.as_dtype(FLAGS['data_input_dtype']).max, FLAGS['data_input_dtype'])
if is_training:
cv2.imwrite(FLAGS['folder_test_img'] + 'A-' + test_image_name_list[i] + '-' + now_epoch + '.tif', enhance_test_img1)
cv2.imwrite(FLAGS['folder_test_img'] + 'A-' + test_image_name_list[i] + '-i' + now_epoch +'.tif', enhance_test_img1_inv)
cv2.imwrite(FLAGS['folder_test_img'] + 'B-' + test_image_name_list[i] + '-' + now_epoch + '.tif', enhance_test_img2)
cv2.imwrite(FLAGS['folder_test_img'] + 'B-' + test_image_name_list[i] + '-i' + now_epoch +'.tif', enhance_test_img2_inv)
else:
assert False, 'not yet'
cv2.imwrite(FLAGS['folder_test_img'] + test_image_name_list[i] + '.tif', enhance_test_img)
cv2.imwrite(FLAGS['folder_test_img'] + test_image_name_list[i] + '-i' + '.tif', enhance_test_img_inv)
else:
assert False, 'not yet'
test_s, enhance_test_img = sess.run([netG_test_summary, netG_test_output1_crop], feed_dict={t:d for t, d in zip(dict_t, dict_d)})
enhance_test_img = safe_casting(enhance_test_img * tf.as_dtype(FLAGS['data_input_dtype']).max, FLAGS['data_input_dtype'])
if is_training:
cv2.imwrite(FLAGS['folder_test_img'] + test_image_name_list[i] + '-' + now_epoch + '.tif', enhance_test_img)
else:
cv2.imwrite(FLAGS['folder_test_img'] + test_image_name_list[i] + '.tif', enhance_test_img)
if FLAGS['mode_use_debug']:
print("i = %d, %s" % (i, str(timer2.end())))
for j in range(len(netG_test_summary_names)):
test_loss_list[j][i] = test_s[j]
test_avg = [test_avg[j] + v for j, v in enumerate(test_s)]
test_avg = [v / test_count for v in test_avg]
best_value_history = [max(v, test_avg[i]) for i, v in enumerate(best_value_history)]
if is_training:
step = round(float(now_epoch)*FLAGS['process_test_log_interval_epoch'])
if step >= FLAGS['process_test_drop_summary_step']:
for i, v in enumerate(test_avg):
summary = sess.run(test_summary_list[i], feed_dict={test_summary_placeholders[i]: v})
summary_writer.add_summary(summary, step)
# weights_data = sess.run(main_net.weights)
# save_weights(weights_data, main_net.parameter_names, FLAGS['weight_folder'], now_epoch)
if is_training:
write_list_to_file(FLAGS['folder_test_netG_loss'] + now_epoch + ".txt", test_loss_list[ 0], test_image_name_list[:test_count])
write_list_to_file(FLAGS['folder_test_netG_psnr1'] + now_epoch + ".txt", test_loss_list[-2], test_image_name_list[:test_count])
write_list_to_file(FLAGS['folder_test_netG_psnr2'] + now_epoch + ".txt", test_loss_list[-1], test_image_name_list[:test_count])
write_list_to_file(FLAGS['folder_train_netG_loss'] + now_epoch + ".txt", list_train_loss, train_image_name_list_input)
write_list_to_file(FLAGS['folder_train_ind_input'] + now_epoch + ".txt", indices_1, train_image_name_list_input, True)
write_list_to_file(FLAGS['folder_train_ind_label'] + now_epoch + ".txt", indices_2, train_image_name_list_label, True)
save_model(saver, sess, FLAGS['folder_model'], now_epoch)
for train_c_summary, step in train_summary_datas:
summary_writer.add_summary(train_c_summary, step)
info = current_time() + ", epoch = " + now_epoch
for i, v in enumerate(test_avg):
info = info + ", %s = %s" % (netG_test_summary_names[i], FLAGS['format_log_value'].format(v).replace(' ', '*'))
info = info + ", (max:"
for i, v in enumerate(best_value_history):
info = info + "%s" % FLAGS['format_log_value'].format(v).replace(' ', '*')
info = info + (")" if i == len(best_value_history) - 1 else ", ")
info = info + ", tt = " + str(timer.end()).split(".")[0][2:]
print(info)
return best_value_history, []
