def _gan28_discr(experiment=None, X=None, reuse=False, **kwargs):
lrelu = partial(leaky_relu, leakiness=experiment.leak)
h_dim = experiment.h_dim
logger = experiment.logger
msg = "D_SHAPE {} {} [reuse={}]"
logger.debug(msg.format("in", X.shape, reuse))
with tf.variable_scope("discriminator", reuse=reuse):
with tf.variable_scope("hidden1"):
d_hidden1 = conv2d(
X, filters=h_dim, kernel_size=5, strides=2, padding="same",
activation=lrelu, kernel_initializer=xavier_init())
logger.debug(msg.format("dh1", d_hidden1.shape, reuse))
with tf.variable_scope("hidden2"):
d_hidden2 = conv2d(
d_hidden1, filters=h_dim * 2, kernel_size=5, strides=2,
padding="same", activation=lrelu,
kernel_initializer=xavier_init())
logger.debug(msg.format("dh2", d_hidden2.shape, reuse))
with tf.variable_scope("hidden3"):
d_hidden2 = tf.reshape(
d_hidden2, [-1, np.prod(d_hidden2.shape[1:], dtype=int)])
d_hidden3 = dense(d_hidden2, 1024, activation=lrelu,
kernel_initializer=xavier_init())
logger.debug(msg.format("dh3", d_hidden3.shape, reuse))
with tf.variable_scope("output"):
d_out = dense(d_hidden3, 1, kernel_initializer=xavier_init())
logger.debug(msg.format("out", d_out.shape, reuse))
# define summaries on the last layer
d_summaries = stats_summaries(d_out)
return d_out, d_summaries
def model_fn(features, labels, mode):
with ops.device("/device:IPU:0"):
with variable_scope.variable_scope("ascope", use_resource=True):
x = array_ops.reshape(features, [-1, 4])
x = layers.dense(inputs=x, units=10)
x = layers.dense(inputs=x, units=3)
if mode in [
model_fn_lib.ModeKeys.TRAIN, model_fn_lib.ModeKeys.EVAL
]:
labels = array_ops.stop_gradient(labels)
loss = math_ops.reduce_mean(
nn.softmax_cross_entropy_with_logits_v2(logits=x,
labels=labels))
else:
loss = None
if mode == model_fn_lib.ModeKeys.TRAIN:
opt = gradient_descent.GradientDescentOptimizer(0.01)
train = opt.minimize(loss, training_util.get_global_step())
else:
train = None
summary_ops.ipu_compile_summary("compile_summary", [train, loss])
return model_fn_lib.EstimatorSpec(mode=mode,
predictions=x,
loss=loss,
train_op=train)
def testDenseLayerJitScopeUndefinedShape(self):
"""Tests that the dense layer node is properly compiled in jit scope.
Dense layer uses shape op to get shape of input tensor if its shape is not
fully defined. XLA does not cluster shape op with other operators. But in
experimental_jit_scope, XLA is forced to compile shape op into its own
cluster, causing dense layer to be split into TWO XlaCompile/XlaRun op
pairs.
"""
with self.cached_session() as sess:
x = array_ops.placeholder(shape=[None, None, 3], dtype=np.float32)
with jit_scope():
y = layers.dense(x, 3)
self.evaluate(variables.global_variables_initializer())
run_metadata = config_pb2.RunMetadata()
test_utils.RunWithWarmup(
sess,
y, {
x: np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]
])
},
run_metadata=run_metadata,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE))
labels = GetRunMetadataLabels(run_metadata)
self.assertEqual(2, self.countXlaOps(labels))
self.assertFalse(InLabels(labels, "MatMult"))
def _model_fn(features, labels, mode):
predictions = layers.dense(
features['x'], 1, kernel_initializer=init_ops.zeros_initializer())
export_outputs = {
'predictions': export_output.RegressionOutput(predictions)
}
if mode == model_fn_lib.ModeKeys.PREDICT:
return model_fn_lib.EstimatorSpec(
mode, predictions=predictions, export_outputs=export_outputs)
loss = losses.mean_squared_error(labels, predictions)
train_op = training.GradientDescentOptimizer(learning_rate=0.5).minimize(
loss, training.get_global_step())
eval_metric_ops = {
'absolute_error': metrics_lib.mean_absolute_error(
labels, predictions)
}
return model_fn_lib.EstimatorSpec(
mode,
predictions=predictions,
loss=loss,
train_op=train_op,
eval_metric_ops=eval_metric_ops,
export_outputs=export_outputs)
def apply_constraints():
logger.debug("Using constraints: {}".format(
str(experiment.constraints)))
with tf.variable_scope("constrained_ll_out"):
constraints_out = dense(
constraints_features, 1, activation=lrelu,
kernel_initializer=xavier_init())
d_out_kernel = tf.get_variable(
"d_out_kernel", shape=[2, 1], initializer=xavier_init())
logger.debug(
msg.format("d_out_kernel", d_out_kernel.shape, reuse))
d_out_bias = tf.get_variable(
"d_out_bias", shape=[1, 1], initializer=xavier_init())
logger.debug(msg.format("d_out_bias", d_out_bias.shape, reuse))
input_concat = tf.concat(
[d_out, constraints_out],
axis=1, name="input_concat_{}".format(reuse))
logger.debug(msg.format(
"input_concat", input_concat.shape, reuse))
d_constrained_out = tf.add(
tf.matmul(input_concat, d_out_kernel), d_out_bias,
name="d_constrained_out_{}".format(reuse))
logger.debug(msg.format(
"constrained_out", d_constrained_out.shape, reuse))
# define summaries on the last layer
d_summaries = tf.summary.merge(
[stats_summaries(d_constrained_out), d_out_summaries])
return d_constrained_out, d_summaries
def testDenseLayerJitScopeUndefinedShape(self):
"""Tests that the dense layer node is properly compiled in jit scope.
Dense layer uses shape op to get shape of input tensor if its shape is not
fully defined. XLA does not cluster shape op with other operators. But in
experimental_jit_scope, XLA is forced to compile shape op into its own
cluster, causing dense layer to be split into TWO XlaCompile/XlaRun op
pairs.
"""
with self.cached_session() as sess:
x = array_ops.placeholder(shape=[None, None, 3], dtype=np.float32)
with jit_scope():
y = layers.dense(x, 3)
self.evaluate(variables.global_variables_initializer())
run_metadata = config_pb2.RunMetadata()
test_utils.RunWithWarmup(
sess,
y, {x: np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]])},
run_metadata=run_metadata,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE))
labels = GetRunMetadataLabels(run_metadata)
self.assertEqual(2, self.countXlaOps(labels))
self.assertFalse(InLabels(labels, "MatMult"))
def _model_fn_without_eval_metrics(self, features, labels, mode, params):
del params # unused.
predictions = layers.dense(
features['x'], 1, kernel_initializer=init_ops.zeros_initializer())
loss = losses.mean_squared_error(labels, predictions)
return self._create_head(mode, loss, None)
def _model_fn(features, labels, mode):
predictions = layers.dense(
features['x'], 1, kernel_initializer=init_ops.zeros_initializer())
export_outputs = {
'predictions': export.RegressionOutput(predictions)
}
if mode == model_fn_lib.ModeKeys.PREDICT:
return model_fn_lib.EstimatorSpec(
mode, predictions=predictions, export_outputs=export_outputs)
loss = losses.mean_squared_error(labels, predictions)
train_op = training.GradientDescentOptimizer(learning_rate=0.5).minimize(
loss, training.get_global_step())
eval_metric_ops = {
'absolute_error': metrics_lib.mean_absolute_error(
labels, predictions)
}
return model_fn_lib.EstimatorSpec(
mode,
predictions=predictions,
loss=loss,
train_op=train_op,
eval_metric_ops=eval_metric_ops,
export_outputs=export_outputs)
def testDenseLayerAutoJit(self):
"""Tests dense layer compilation in auto-jit mode.
Dense layer should be compiled into a single XlaCompile/XlaRun op pair in
auto-jit mode.
"""
os.environ["TF_XLA_FLAGS"] = (
"--tf_xla_cpu_global_jit " + os.environ.get("TF_XLA_FLAGS", ""))
config = config_pb2.ConfigProto()
config.graph_options.optimizer_options.global_jit_level = (
config_pb2.OptimizerOptions.ON_1)
with self.session(config=config) as sess:
x = array_ops.placeholder(shape=[None, None, 3], dtype=np.float32)
y = layers.dense(x, 3)
self.evaluate(variables.global_variables_initializer())
run_metadata = config_pb2.RunMetadata()
test_utils.RunWithWarmup(
sess,
y, {x: np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]])},
run_metadata=run_metadata,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE))
labels = GetRunMetadataLabels(run_metadata)
self.assertEqual(1, self.countXlaOps(labels))
self.assertFalse(InLabels(labels, "MatMult"))
def testDenseLayerJitScopeUndefinedShape(self):
"""Tests that the dense layer node is properly compiled in jit scope.
Dense layer uses shape op to get shape of input tensor if its shape is not
fully defined. XLA does not cluster shape op with other operators. But in
experimental_jit_scope, XLA is forced to compile shape op into its own
cluster, causing dense layer to be split into TWO XlaLaunch ops.
"""
with self.test_session() as sess:
x = array_ops.placeholder(shape=[None, None, 3], dtype=np.float32)
with jit_scope():
y = layers.dense(x, 3)
sess.run(variables.initialize_all_variables())
run_metadata = config_pb2.RunMetadata()
sess.run(y, {
x:
np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]])
},
run_metadata=run_metadata,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE))
labels = GetRunMetadataLabels(run_metadata)
self.assertEqual(2, XlaLaunchOpCount(labels))
self.assertFalse(InLabels(labels, "ListDiff"))
def testDenseLayerAutoJit(self):
"""Tests dense layer compilation in auto-jit mode.
Dense layer should be compiled into a single XlaLaunch op in auto-jit mode.
"""
os.environ["TF_XLA_FLAGS"] = ("--tf_xla_cpu_global_jit")
config = config_pb2.ConfigProto()
config.graph_options.optimizer_options.global_jit_level = (
config_pb2.OptimizerOptions.ON_1)
with self.test_session(config=config) as sess:
x = array_ops.placeholder(shape=[None, None, 3], dtype=np.float32)
y = layers.dense(x, 3)
sess.run(variables.initialize_all_variables())
run_metadata = config_pb2.RunMetadata()
sess.run(y, {
x:
np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]])
},
run_metadata=run_metadata,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE))
labels = GetRunMetadataLabels(run_metadata)
self.assertEqual(1, XlaLaunchOpCount(labels))
self.assertFalse(InLabels(labels, "ListDiff"))
def testDenseLayerJitScopeUndefinedShape(self):
"""Tests that the dense layer node is properly compiled in jit scope.
Dense layer uses shape op to get shape of input tensor if its shape is not
fully defined. XLA does not cluster shape op with other operators. But in
experimental_jit_scope, XLA is forced to compile shape op into its own
cluster, causing dense layer to be split into TWO XlaLaunch ops.
"""
with self.test_session() as sess:
x = array_ops.placeholder(shape=[None, None, 3], dtype=np.float32)
with jit_scope():
y = layers.dense(x, 3)
sess.run(variables.initialize_all_variables())
run_metadata = config_pb2.RunMetadata()
sess.run(
y, {x: np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]])},
run_metadata=run_metadata,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE))
labels = GetRunMetadataLabels(run_metadata)
self.assertEqual(2, XlaLaunchOpCount(labels))
self.assertFalse(InLabels(labels, "ListDiff"))
def testDenseLayerJitScopeDefinedShape(self):
"""Tests that the dense layer node is properly compiled in jit scope.
Dense layer with static shape input tensor should be compiled into a single
XlaCompile/XlaRun op pair by XLA.
"""
with self.session() as sess:
x = array_ops.placeholder(shape=[2, 2, 3], dtype=np.float32)
with jit_scope():
y = layers.dense(x, 3)
self.evaluate(variables.global_variables_initializer())
run_metadata = config_pb2.RunMetadata()
test_utils.RunWithWarmup(
sess,
y, {
x: np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]
])
},
run_metadata=run_metadata,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE))
labels = GetRunMetadataLabels(run_metadata)
self.assertEqual(1, self.countXlaOps(labels))
def testDenseLayerAutoJit(self):
"""Tests dense layer compilation in auto-jit mode.
Dense layer should be compiled into a single XlaCompile/XlaRun op pair in
auto-jit mode.
"""
os.environ["TF_XLA_FLAGS"] = (
"--tf_xla_cpu_global_jit " + os.environ.get("TF_XLA_FLAGS", ""))
config = config_pb2.ConfigProto()
config.graph_options.optimizer_options.global_jit_level = (
config_pb2.OptimizerOptions.ON_1)
with self.session(config=config) as sess:
x = array_ops.placeholder(shape=[None, None, 3], dtype=np.float32)
y = layers.dense(x, 3)
self.evaluate(variables.global_variables_initializer())
run_metadata = config_pb2.RunMetadata()
test_utils.RunWithWarmup(
sess,
y, {x: np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]])},
run_metadata=run_metadata,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE))
labels = GetRunMetadataLabels(run_metadata)
self.assertEqual(1, self.countXlaOps(labels))
self.assertFalse(InLabels(labels, "MatMult"))
def testDenseLayerAutoJit(self):
"""Tests dense layer compilation in auto-jit mode.
Dense layer should be compiled into a single XlaLaunch op in auto-jit mode.
"""
os.environ["TF_XLA_FLAGS"] = ("--tf_xla_cpu_global_jit")
config = config_pb2.ConfigProto()
config.graph_options.optimizer_options.global_jit_level = (
config_pb2.OptimizerOptions.ON_1)
with self.test_session(config=config) as sess:
x = array_ops.placeholder(shape=[None, None, 3], dtype=np.float32)
y = layers.dense(x, 3)
sess.run(variables.initialize_all_variables())
run_metadata = config_pb2.RunMetadata()
sess.run(
y, {x: np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]])},
run_metadata=run_metadata,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE))
labels = GetRunMetadataLabels(run_metadata)
self.assertEqual(1, XlaLaunchOpCount(labels))
self.assertFalse(InLabels(labels, "ListDiff"))
def _gan60_gen(experiment=None, z=None, **kwargs):
# Batch normalization is applied after the layer's activation function and
# without removing its bias.
h_dim = experiment.h_dim
logger = experiment.logger
msg = "G_SHAPE {}: {}"
logger.debug(msg.format("in", z.shape))
with tf.variable_scope("generator"):
with tf.variable_scope("hidden1"):
g_hidden1 = batch_norm(dense(z, 1024, activation=relu))
logger.debug(msg.format("gh1", g_hidden1.shape))
with tf.variable_scope("hidden2"):
g_hidden2 = batch_norm(dense(
g_hidden1, h_dim * 2 * 5 * 5, activation=relu))
logger.debug(msg.format("gh2", g_hidden2.shape))
g_hidden2 = tf.reshape(g_hidden2, [-1, h_dim * 2, 5, 5])
logger.debug(msg.format("gh2", g_hidden2.shape))
with tf.variable_scope("hidden3"):
# deconv2d only supports nhwc channels. Transposing nchw to nhwc
g_hidden2 = tf.transpose(g_hidden2, [0, 3, 2, 1])
logger.debug(msg.format("gh3", g_hidden2.shape))
g_hidden3 = batch_norm(deconv2d(
g_hidden2, filters=h_dim, kernel_size=5, strides=2,
padding="same", activation=relu, kernel_initializer=ort_init(),
bias_initializer=zeros_init()))
logger.debug(msg.format("gh3", g_hidden3.shape))
with tf.variable_scope("hidden4"):
g_hidden4 = batch_norm(deconv2d(
g_hidden3, filters=h_dim, kernel_size=5, strides=2,
padding="same", activation=relu, kernel_initializer=ort_init(),
bias_initializer=zeros_init()))
logger.debug(msg.format("gh4", g_hidden4.shape))
with tf.variable_scope("output"):
g_out = deconv2d(
g_hidden4, filters=1, kernel_size=5,
strides=3, padding="same", kernel_initializer=ort_init(),
bias_initializer=zeros_init())
logger.debug(msg.format("out", g_out.shape))
# define summaries on the last layer
g_summaries, images_summaries = generator_summaries(
g_out, experiment)
return g_out, g_summaries, images_summaries
def _model_fn_with_eval_dict(self, features, labels, mode, params):
del params # unused.
predictions = layers.dense(
features['x'], 1, kernel_initializer=init_ops.zeros_initializer())
loss = losses.mean_squared_error(labels, predictions)
return self._create_head(mode,
loss,
eval_metrics=(self._metric_fn_on_cpu, {
'labels': labels,
'predictions': predictions
}))
def testDenseLayer(self):
"""Tests that the dense layer node is properly compiled."""
with self.test_session(config=NoRewriteSessionConfig()) as sess:
x = array_ops.placeholder(shape=[2, 3], dtype=np.float32)
with jit_scope():
y = layers.dense(x, 3)
sess.run(variables.initialize_all_variables())
run_metadata = config_pb2.RunMetadata()
sess.run(y, {x: np.array([[1, 2, 3], [4, 5, 6]])},
run_metadata=run_metadata,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE))
self.assert_(MetadataHasXlaLaunch(run_metadata))
def testDenseLayer(self):
"""Tests that the dense layer node is properly compiled."""
with self.test_session(config=NoRewriteSessionConfig()) as sess:
x = array_ops.placeholder(shape=[2, 3], dtype=np.float32)
with jit_scope():
y = layers.dense(x, 3)
sess.run(variables.initialize_all_variables())
run_metadata = config_pb2.RunMetadata()
sess.run(y, {x: np.array([[1, 2, 3], [4, 5, 6]])},
run_metadata=run_metadata,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE))
self.assert_(MetadataHasXlaLaunch(run_metadata))
def _model_fn(features, labels, mode, params):
if not self._export_mode:
# Always check batch size in params
self.assertEqual(batch_size_dict[mode], params['batch_size'])
else:
self.assertNotIn('batch_size', params)
# Check the input feeds correct shape for train and eval. When eval on CPU
# or predict, it is allowed to have dynamic shape. So, here only validates
# the fully known shape (which covers the TPU train).
if features['x'].shape.is_fully_defined():
self.assertEqual(batch_size_dict[mode], features['x'].shape[0])
predictions = layers.dense(
features['x'],
1,
kernel_initializer=init_ops.ones_initializer())
export_outputs = {
'predictions': export_output.RegressionOutput(predictions)
}
if mode == _PREDICT:
return _create_estimator_spec(
mode=mode,
predictions={'predictions': predictions},
export_outputs=export_outputs)
loss = losses.mean_squared_error(labels, predictions)
optimizer = tf.tpu.CrossShardOptimizer(
training.GradientDescentOptimizer(learning_rate=0.5))
train_op = optimizer.minimize(
loss, global_step=training.get_global_step())
eval_metrics = (
lambda labels, predictions: { # pylint: disable=g-long-lambda
'absolute_error':
metrics_lib.mean_absolute_error(labels, predictions)
},
[labels, predictions])
return _create_estimator_spec(
mode=mode,
loss=loss,
predictions={'predictions': predictions},
export_outputs=export_outputs,
train_op=train_op,
eval_metrics=eval_metrics)
def testDenseLayerJitScopeUndefinedShape(self):
"""Tests that the dense layer node is properly compiled in jit scope.
"""
with self.cached_session() as sess:
x = array_ops.placeholder(shape=[None, None, 3], dtype=np.float32)
with jit_scope():
y = layers.dense(x, 3)
self.evaluate(variables.global_variables_initializer())
run_metadata = config_pb2.RunMetadata()
test_utils.RunWithWarmup(
sess,
y, {x: np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]])},
run_metadata=run_metadata,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE))
labels = GetRunMetadataLabels(run_metadata)
self.assertEqual(1, self.countXlaOps(labels))
self.assertFalse(InLabels(labels, "MatMult"))
def testDenseLayerJitScopeUndefinedShape(self):
"""Tests that the dense layer node is properly compiled in jit scope.
"""
with self.session() as sess:
x = array_ops.placeholder(shape=[None, None, 3], dtype=np.float32)
with jit_scope():
y = layers.dense(x, 3)
self.evaluate(variables.global_variables_initializer())
run_metadata = config_pb2.RunMetadata()
test_utils.RunWithWarmup(
sess,
y, {x: np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]])},
run_metadata=run_metadata,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE))
labels = GetRunMetadataLabels(run_metadata)
self.assertEqual(1, self.countXlaOps(labels))
self.assertFalse(InLabels(labels, "MatMult"))
def testDenseLayerJitScopeDefinedShape(self):
"""Tests that the dense layer node is properly compiled in jit scope.
Dense layer with static shape input tensor should be compiled into a single
XlaLaunch op by XLA.
"""
with self.cached_session() as sess:
x = array_ops.placeholder(shape=[2, 2, 3], dtype=np.float32)
with jit_scope():
y = layers.dense(x, 3)
sess.run(variables.initialize_all_variables())
run_metadata = config_pb2.RunMetadata()
sess.run(
y, {x: np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]])},
run_metadata=run_metadata,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE))
labels = GetRunMetadataLabels(run_metadata)
self.assertEqual(1, XlaLaunchOpCount(labels))
def testDenseLayerJitScopeDefinedShape(self):
"""Tests that the dense layer node is properly compiled in jit scope.
Dense layer with static shape input tensor should be compiled into a single
XlaLaunch op by XLA.
"""
with self.test_session() as sess:
x = array_ops.placeholder(shape=[2, 2, 3], dtype=np.float32)
with jit_scope():
y = layers.dense(x, 3)
sess.run(variables.initialize_all_variables())
run_metadata = config_pb2.RunMetadata()
sess.run(
y, {x: np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]])},
run_metadata=run_metadata,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE))
labels = GetRunMetadataLabels(run_metadata)
self.assertEqual(1, XlaLaunchOpCount(labels))
def testDenseLayerJitScopeDefinedShape(self):
"""Tests that the dense layer node is properly compiled in jit scope.
Dense layer with static shape input tensor should be compiled into a single
XlaCompile/XlaRun op pair by XLA.
"""
with self.cached_session() as sess:
x = array_ops.placeholder(shape=[2, 2, 3], dtype=np.float32)
with jit_scope():
y = layers.dense(x, 3)
self.evaluate(variables.global_variables_initializer())
run_metadata = config_pb2.RunMetadata()
test_utils.RunWithWarmup(
sess,
y, {x: np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]])},
run_metadata=run_metadata,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE))
labels = GetRunMetadataLabels(run_metadata)
self.assertEqual(1, self.countXlaOps(labels))
def testTrainingLoop(self):
random_seed.set_random_seed(1)
# Model
with ops.device("/device:IPU:0"):
with variable_scope.variable_scope("vs", use_resource=True):
x = array_ops.placeholder(np.float32, [4, 1, 4], name="a")
l = array_ops.placeholder(np.float32, [4, 1, 1], name="b")
y = layers.dense(x, 1, activation=nn.sigmoid)
loss = losses.log_loss(l, y)
train_op = gradient_descent.GradientDescentOptimizer(0.1) \
.minimize(loss)
init = variables.global_variables_initializer()
# Test data
image_data = [[[1, 1, 1, 1]], [[2, 2, 2, 2]], [[3, 3, 3, 3]],
[[4, 4, 4, 4]]]
label_data = [[[1]], [[2]], [[3]], [[4]]]
# Run training.
with ms.MonitoredTrainingSession(is_chief=True,
chief_only_hooks=None,
save_summaries_steps=None,
save_summaries_secs=None) as sess:
sess.run(init)
previous_loss = float("inf")
for _ in range(5):
measured_loss, _ = sess.run([loss, train_op],
feed_dict={
x: image_data,
l: label_data
})
self.assertTrue(measured_loss < previous_loss)
previous_loss = measured_loss
def _can60_discr_32layer_auto(
experiment=None, X=None, reuse=False, use_constraints=None,
constraints_features=None, **kwargs):
lrelu = partial(leaky_relu, leakiness=experiment.leak)
h_dim = experiment.h_dim
logger = experiment.logger
msg = "D_SHAPE {} {} [reuse={}]"
logger.debug(msg.format("in", X.shape, reuse))
with tf.variable_scope("discriminator", reuse=reuse):
with tf.variable_scope("hidden1"):
d_hidden1 = conv2d(
X, filters=h_dim, kernel_size=5, strides=2, padding="same",
activation=lrelu, kernel_initializer=xavier_init())
logger.debug(msg.format("dh1", d_hidden1.shape, reuse))
with tf.variable_scope("hidden15"):
d_hidden15 = conv2d(
d_hidden1, filters=h_dim, kernel_size=5, strides=2,
padding="same", activation=lrelu,
kernel_initializer=xavier_init())
logger.debug(msg.format("dh15", d_hidden15.shape, reuse))
with tf.variable_scope("hidden2"):
d_hidden2 = conv2d(
d_hidden15, filters=h_dim * 2, kernel_size=5, strides=2,
padding="same", activation=lrelu,
kernel_initializer=xavier_init())
logger.debug(msg.format("dh2", d_hidden2.shape, reuse))
d_hidden2 = tf.reshape(
d_hidden2, [-1, np.prod(d_hidden2.shape[1:], dtype=int)])
logger.debug(msg.format("dh2", d_hidden2.shape, reuse))
with tf.variable_scope("hidden3"):
d_hidden3 = dense(d_hidden2, 1024, activation=lrelu,
kernel_initializer=xavier_init())
logger.debug(msg.format("dh3", d_hidden3.shape, reuse))
with tf.variable_scope("hidden4"):
d_hidden4 = dense(d_hidden3, 32, activation=lrelu,
kernel_initializer=xavier_init())
logger.debug(msg.format("dh4", d_hidden4.shape, reuse))
d_summaries_dh4 = stats_summaries(d_hidden4, "dh4_pre_cond")
with tf.variable_scope("shared_weights"):
d_out_kernel = tf.get_variable(
"d_out_kernel", shape=[32, 1], initializer=xavier_init())
logger.debug(msg.format("d_out_kernel", d_out_kernel.shape, reuse))
d_out_bias = tf.get_variable(
"d_out_bias", shape=[1, 1], initializer=xavier_init())
logger.debug(msg.format("d_out_bias", d_out_bias.shape, reuse))
def skip_constraints():
with tf.variable_scope("output"):
d_out = tf.add(tf.matmul(d_hidden4, d_out_kernel),
d_out_bias, name="d_out_{}".format(reuse))
logger.debug(msg.format("out", d_out.shape, reuse))
# define summaries on the last layer
d_summaries = tf.summary.merge(
[stats_summaries(d_out), d_summaries_dh4])
return d_out, d_summaries
def apply_constraints():
logger.debug("Using constraints: {}".format(
str(experiment.constraints)))
with tf.variable_scope("constrained_out"):
d_constraints_kernel = tf.get_variable(
"d_constraints_kernel",
shape=[constraints_features.shape[1], 1],
initializer=xavier_init())
logger.debug(msg.format(
"d_constraints_kernel", d_constraints_kernel.shape, reuse))
input_concat = tf.concat(
[d_hidden4, constraints_features],
axis=1, name="input_concat_{}".format(reuse))
logger.debug(msg.format(
"input_concat", input_concat.shape, reuse))
weight_concat = tf.concat(
[d_out_kernel, d_constraints_kernel],
axis=0, name="weight_concat_{}".format(reuse))
logger.debug(msg.format(
"weight_concat", weight_concat.shape, reuse))
d_constrained_out = tf.add(
tf.matmul(input_concat, weight_concat), d_out_bias,
name="d_constrained_out_{}".format(reuse))
logger.debug(msg.format(
"constrained_out", d_constrained_out.shape, reuse))
# define summaries on the last layer
d_summaries = tf.summary.merge(
[stats_summaries(d_constrained_out),
d_summaries_dh4])
return d_constrained_out, d_summaries
return tf.cond(tf.cast(use_constraints, tf.bool),
skip_constraints, apply_constraints)
def logistic_classifier(inputs):
return layers.dense(inputs, 1, activation=math_ops.sigmoid)
def inception_v3(inputs,
num_classes=12,
is_training=True,
dropout_keep_prob=0.8,
prediction_fn=layers_lib.softmax,
reuse=None,
hparam_string=None,
global_pool=True,
scope='InceptionV3'):
"""Inception model from http://arxiv.org/abs/1512.00567.
"Rethinking the Inception Architecture for Computer Vision"
Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens,
Zbigniew Wojna.
With the default arguments this method constructs the exact model defined in
the paper. However, one can experiment with variations of the inception_v3
network by changing arguments dropout_keep_prob, min_depth and
depth_multiplier.
The default image size used to train this network is 299x299.
Args:
inputs: a tensor of size [batch_size, height, width, channels].
num_classes: number of predicted classes.
is_training: whether is training or not.
dropout_keep_prob: the percentage of activation values that are retained.
min_depth: Minimum depth value (number of channels) for all convolution ops.
Enforced when depth_multiplier < 1, and not an active constraint when
depth_multiplier >= 1.
depth_multiplier: Float multiplier for the depth (number of channels)
for all convolution ops. The value must be greater than zero. Typical
usage will be to set this value in (0, 1) to reduce the number of
parameters or computation cost of the model.
prediction_fn: a function to get predictions out of logits.
spatial_squeeze: if True, logits is of shape is [B, C], if false logits is
of shape [B, 1, 1, C], where B is batch_size and C is number of classes.
reuse: whether or not the network and its variables should be reused. To be
able to reuse 'scope' must be given.
scope: Optional variable_scope.
Returns:
logits: the pre-softmax activations, a tensor of size
[batch_size, num_classes]
end_points: a dictionary from components of the network to the corresponding
activation.
Raises:
ValueError: if 'depth_multiplier' is less than or equal to zero.
"""
hparams = create_hparams(hparam_string=hparam_string)
if hparams.depth_multiplier <= 0:
raise ValueError('depth_multiplier is not greater than zero.')
depth = lambda d: max(int(d * hparams.depth_multiplier), hparams.min_depth)
with variable_scope.variable_scope(scope,
'InceptionV3', [inputs, num_classes],
reuse=reuse) as scope:
with arg_scope([layers_lib.batch_norm, layers_lib.dropout],
is_training=is_training):
net, end_points = inception_v3_base(inputs,
scope=scope,
**hparams.values())
# Final pooling and prediction
with variable_scope.variable_scope('Logits'):
if global_pool:
# Global average pooling.
net_avg = math_ops.reduce_mean(net, [1, 2],
keep_dims=True,
name='global_avg_pool')
net_max = math_ops.reduce_max(net, [1, 2],
keep_dims=True,
name='global_max_pool')
net = array_ops.concat([net_avg, net_max], -1)
else:
kernel_size = _reduced_kernel_size_for_small_input(
net, [8, 8])
net = layers_lib.avg_pool2d(
net,
kernel_size,
padding='VALID',
scope='AvgPool_1a_{}x{}'.format(*kernel_size))
# 1 x 1 x 2048
net = layers_lib.dropout(net,
keep_prob=dropout_keep_prob,
scope='Dropout_1b')
end_points['PreLogits'] = net
# 2048
logits = layers.conv2d(net,
num_classes, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='Conv2d_1c_1x1')
logits = layers_core.dense(array_ops.squeeze(
logits, [1, 2], name='SpatialSqueeze'),
num_classes,
name='logits')
end_points['Logits'] = logits
end_points['Predictions'] = prediction_fn(logits,
scope='Predictions')
return logits, end_points
def dense_computation(features):
return layers.dense(features['x'],
1,
kernel_initializer=init_ops.zeros_initializer())
def computation(input_tensor):
return layers.dense(
input_tensor,
1,
kernel_initializer=init_ops.zeros_initializer())
def logistic_classifier(inputs): return layers.dense(inputs, 1, activation=math_ops.sigmoid)