def test_model(self, strategy_fn, use_operator=False, use_regularizer=False,
cloning=True):
regularizer = IdentityRegularizer() if use_regularizer else None
with strategy_fn().scope():
with policy.policy_scope('infer_float32_vars'):
x = layers.Input(shape=(1,), batch_size=2, dtype=dtypes.float16)
layer = AddLayer(assert_type=dtypes.float16, use_operator=use_operator,
regularizer=regularizer)
y = layer(x)
y = math_ops.cast(y, dtypes.float32)
model = models.Model(inputs=x, outputs=y)
def loss_fn(y_true, y_pred):
del y_true
return math_ops.reduce_mean(y_pred)
# Learning rate is small enough that if applied to a float16 variable,
# the variable will not change. So this tests the learning rate not
# applied to a float16 value, but instead the float32 variable.
opt = gradient_descent.SGD(2 ** -14)
model.compile(opt, loss=loss_fn, cloning=cloning)
self.assertEqual(backend.eval(layer.v), 1)
x = np.ones((2, 1))
y = np.ones((2, 1))
dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).batch(2)
model.fit(dataset)
# Variable starts at 1, and should have gradient of 2 ** -14 subtracted
# from it.
expected = 1 - 2 ** -14
if use_regularizer:
# Regularizer adds another 2 ** -14 to the gradient.
expected -= 2 ** -14
self.assertEqual(backend.eval(layer.v), expected)
def test_fixed_loss_scaling(self, strategy_fn, cloning=True):
# Note: We do not test mixed precision in this method, only loss scaling.
loss_scale = 8.
batch_size = 4
with strategy_fn().scope():
x = layers.Input(shape=(1,), batch_size=batch_size)
layer = AddLayer()
y = layer(x)
# The gradient of 'y' at this point is 1. With loss scaling, the gradient
# is 'loss_scale'. We divide by the batch size since the loss is averaged
# across batch elements.
expected_gradient = loss_scale / batch_size
identity_with_grad_check_fn = (
mp_test_util.create_identity_with_grad_check_fn([expected_gradient]))
y = core.Lambda(identity_with_grad_check_fn)(y)
model = models.Model(inputs=x, outputs=y)
def loss_fn(y_true, y_pred):
del y_true
return math_ops.reduce_mean(y_pred)
opt = gradient_descent.SGD(1.)
opt = loss_scale_optimizer.LossScaleOptimizer(opt, loss_scale)
model.compile(opt, loss=loss_fn, cloning=cloning)
self.assertEqual(backend.eval(layer.v), 1)
x = np.ones((batch_size, 1))
y = np.ones((batch_size, 1))
dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).batch(batch_size)
model.fit(dataset)
# Variable starts at 1, and should have gradient of 1 subtracted from it.
expected = 0
self.assertEqual(backend.eval(layer.v), expected)
def test_top_k_categorical_accuracy(self):
with self.test_session():
y_pred = K.variable(np.array([[0.3, 0.2, 0.1], [0.1, 0.2, 0.7]]))
y_true = K.variable(np.array([[0, 1, 0], [1, 0, 0]]))
result = K.eval(metrics.top_k_categorical_accuracy(y_true, y_pred, k=3))
self.assertEqual(result, 1)
result = K.eval(metrics.top_k_categorical_accuracy(y_true, y_pred, k=2))
self.assertEqual(result, 0.5)
result = K.eval(metrics.top_k_categorical_accuracy(y_true, y_pred, k=1))
self.assertEqual(result, 0.)
def test_advanced_model(self, strategy_fn):
# The advanced model tests mixed-precision-related features that would occur
# in a resnet50 model. It tests a model that has:
# * Multiple layers, some which use auto-cast variables and some which do
# not
# * Regularization on some variables and not others.
strategy = strategy_fn()
learning_rate = 2 ** -14
with strategy.scope():
with policy.policy_scope(policy.Policy('infer_float32_vars')):
x = layers.Input(shape=(), batch_size=2, dtype=dtypes.float16)
layer1 = AddLayer(assert_type=dtypes.float16,
regularizer=IdentityRegularizer(), use_operator=True)
layer2 = AddLayerWithoutAutoCast(assert_type=dtypes.float16,
use_operator=True)
layer3 = AddLayer(assert_type=dtypes.float16, use_operator=False)
layer4 = AddLayerWithoutAutoCast(assert_type=dtypes.float16,
regularizer=IdentityRegularizer(),
use_operator=False)
y = layer1(x)
y = layer2(y)
y = layer3(y)
y = layer4(y)
y = math_ops.cast(y, dtypes.float32)
model = models.Model(inputs=x, outputs=y)
def loss_fn(y_true, y_pred):
self.assertEqual(y_true.dtype, dtypes.float32)
self.assertEqual(y_pred.dtype, dtypes.float32)
return math_ops.reduce_mean(y_pred)
opt = gradient_descent.SGD(learning_rate)
model.compile(opt, loss=loss_fn)
x = np.ones((2, 1))
y = np.ones((2, 1))
dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).batch(2)
model.fit(dataset)
for layer in (layer1, layer2, layer3, layer4):
if layer.losses:
# Layer has weight regularizer
self.assertEqual(backend.eval(layer.v), 1 - 2 * learning_rate)
else:
# Layer does not have weight regularizer
self.assertEqual(backend.eval(layer.v), 1 - learning_rate)
def test_merge_concatenate(self):
i1 = keras.layers.Input(shape=(4, 5))
i2 = keras.layers.Input(shape=(4, 5))
concat_layer = keras.layers.Concatenate(axis=1)
o = concat_layer([i1, i2])
self.assertListEqual(o.shape.as_list(), [None, 8, 5])
model = keras.models.Model([i1, i2], o)
model.run_eagerly = testing_utils.should_run_eagerly()
x1 = np.random.random((2, 4, 5))
x2 = np.random.random((2, 4, 5))
out = model.predict([x1, x2])
self.assertEqual(out.shape, (2, 8, 5))
self.assertAllClose(out, np.concatenate([x1, x2], axis=1), atol=1e-4)
self.assertEqual(concat_layer.compute_mask([i1, i2], [None, None]), None)
self.assertTrue(
np.all(
K.eval(
concat_layer.compute_mask(
[i1, i2], [K.variable(x1), K.variable(x2)]))))
with self.assertRaisesRegexp(ValueError, "`mask` should be a list."):
concat_layer.compute_mask([i1, i2], x1)
with self.assertRaisesRegexp(ValueError, "`inputs` should be a list."):
concat_layer.compute_mask(i1, [None, None])
with self.assertRaisesRegexp(ValueError, "should have the same length"):
concat_layer.compute_mask([i1, i2], [None])
with self.assertRaisesRegexp(ValueError,
"layer should be called on a list of inputs"):
concat_layer(i1)
def test_metrics(self):
with self.test_session():
y_a = K.variable(np.random.random((6, 7)))
y_b = K.variable(np.random.random((6, 7)))
for metric in [metrics.binary_accuracy, metrics.categorical_accuracy]:
output = metric(y_a, y_b)
self.assertEqual(K.eval(output).shape, (6,))
def test_merge_subtract(self):
i1 = keras.layers.Input(shape=(4, 5))
i2 = keras.layers.Input(shape=(4, 5))
i3 = keras.layers.Input(shape=(4, 5))
subtract_layer = keras.layers.Subtract()
o = subtract_layer([i1, i2])
self.assertListEqual(o.shape.as_list(), [None, 4, 5])
model = keras.models.Model([i1, i2], o)
model.run_eagerly = testing_utils.should_run_eagerly()
x1 = np.random.random((2, 4, 5))
x2 = np.random.random((2, 4, 5))
out = model.predict([x1, x2])
self.assertEqual(out.shape, (2, 4, 5))
self.assertAllClose(out, x1 - x2, atol=1e-4)
self.assertEqual(subtract_layer.compute_mask([i1, i2], [None, None]), None)
self.assertTrue(
np.all(
K.eval(
subtract_layer.compute_mask(
[i1, i2], [K.variable(x1), K.variable(x2)]))))
with self.assertRaisesRegexp(ValueError, "`mask` should be a list."):
subtract_layer.compute_mask([i1, i2], x1)
with self.assertRaisesRegexp(ValueError, "`inputs` should be a list."):
subtract_layer.compute_mask(i1, [None, None])
with self.assertRaisesRegexp(ValueError,
"layer should be called on exactly 2 inputs"):
subtract_layer([i1, i2, i3])
with self.assertRaisesRegexp(ValueError,
"layer should be called on exactly 2 inputs"):
subtract_layer([i1])
def test_merge_add(self):
i1 = keras.layers.Input(shape=(4, 5))
i2 = keras.layers.Input(shape=(4, 5))
i3 = keras.layers.Input(shape=(4, 5))
add_layer = keras.layers.Add()
o = add_layer([i1, i2, i3])
self.assertListEqual(o.shape.as_list(), [None, 4, 5])
model = keras.models.Model([i1, i2, i3], o)
model.run_eagerly = testing_utils.should_run_eagerly()
x1 = np.random.random((2, 4, 5))
x2 = np.random.random((2, 4, 5))
x3 = np.random.random((2, 4, 5))
out = model.predict([x1, x2, x3])
self.assertEqual(out.shape, (2, 4, 5))
self.assertAllClose(out, x1 + x2 + x3, atol=1e-4)
self.assertEqual(
add_layer.compute_mask([i1, i2, i3], [None, None, None]), None)
self.assertTrue(
np.all(
K.eval(
add_layer.compute_mask(
[i1, i2], [K.variable(x1), K.variable(x2)]))))
with self.assertRaisesRegexp(ValueError, "`mask` should be a list."):
add_layer.compute_mask([i1, i2, i3], x1)
with self.assertRaisesRegexp(ValueError, "`inputs` should be a list."):
add_layer.compute_mask(i1, [None, None, None])
with self.assertRaisesRegexp(ValueError, " should have the same length."):
add_layer.compute_mask([i1, i2, i3], [None, None])
def test_sparse_categorical_accuracy(self):
with self.cached_session():
metric = metrics.sparse_categorical_accuracy
y_true = K.variable(np.random.randint(0, 7, (6,)))
y_pred = K.variable(np.random.random((6, 7)))
self.assertEqual(K.eval(metric(y_true, y_pred)).shape, (6,))
# Test correctness if the shape of y_true is (num_samples,)
y_true = K.variable([1., 0., 0., 0.])
y_pred = K.variable([[0.8, 0.2], [0.6, 0.4], [0.7, 0.3], [0.9, 0.1]])
print(K.eval(metric(y_true, y_pred)))
self.assertAllEqual(K.eval(metric(y_true, y_pred)), [0., 1., 1., 1.])
# Test correctness if the shape of y_true is (num_samples, 1)
y_true = K.variable([[1.], [0.], [0.], [0.]])
y_pred = K.variable([[0.8, 0.2], [0.6, 0.4], [0.7, 0.3], [0.9, 0.1]])
print(K.eval(metric(y_true, y_pred)))
self.assertAllEqual(K.eval(metric(y_true, y_pred)), [0., 1., 1., 1.])
def check_operation_offset(depth, eval_f, padding):
"""Check if backend used an offset while placing the filter
e.g. during a convolution.
TensorFlow is inconsistent in doing so depending
on the type of operation, the used device (CPU/GPU) and the input depth.
"""
in_arr = np.array([[[[i] * depth for i in range(6)]]])
input_data = K.variable(value=in_arr, dtype='float32')
output = eval_f(depth, padding, input_data)
result = K.eval(output).flatten().tolist()
assert result in [[0, 3], [1, 4]]
return result == [1, 4]
def test_sparse_top_k_categorical_accuracy(self):
with self.cached_session():
# Test correctness if the shape of y_true is (num_samples, 1)
y_pred = K.variable(np.array([[0.3, 0.2, 0.1], [0.1, 0.2, 0.7]]))
y_true = K.variable(np.array([[1], [0]]))
result = K.eval(
metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=3))
self.assertEqual(result, 1)
result = K.eval(
metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=2))
self.assertEqual(result, 0.5)
result = K.eval(
metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=1))
self.assertEqual(result, 0.)
# Test correctness if the shape of y_true is (num_samples,)
y_pred = K.variable(np.array([[0.3, 0.2, 0.1], [0.1, 0.2, 0.7]]))
y_true = K.variable(np.array([1, 0]))
result = K.eval(
metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=3))
self.assertEqual(result, 1)
result = K.eval(
metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=2))
self.assertEqual(result, 0.5)
result = K.eval(
metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=1))
self.assertEqual(result, 0.)
def assert_optimizer_iterations_increases(self, optimizer):
model = _get_model()
model.compile(
optimizer, 'mse', metrics=['acc', metrics.categorical_accuracy],
run_eagerly=testing_utils.should_run_eagerly())
global_step = keras.backend.variable(123, dtype=dtypes.int64)
clone_model = models.clone_and_build_model(
model, compile_clone=True, optimizer_iterations=global_step,
in_place_reset=(testing_utils.get_model_type() == 'subclass'))
inp = np.random.random((10, 4))
out = np.random.random((10, 4))
clone_model.train_on_batch(inp, out)
self.assertEqual(K.eval(global_step), 124)
def assert_optimizer_iterations_increases(self, optimizer):
model = _get_model()
model.compile(optimizer,
'mse',
metrics=['acc', metrics.categorical_accuracy],
run_eagerly=testing_utils.should_run_eagerly())
global_step = keras.backend.variable(123, dtype=dtypes.int64)
clone_model = models.clone_and_build_model(
model,
compile_clone=True,
optimizer_iterations=global_step,
in_place_reset=(testing_utils.get_model_type() == 'subclass'))
inp = np.random.random((10, 4))
out = np.random.random((10, 4))
clone_model.train_on_batch(inp, out)
self.assertEqual(K.eval(global_step), 124)
def test_merge_subtract(self):
if testing_utils.should_run_distributed():
self.skipTest("b/137397816")
i1 = keras.layers.Input(shape=(4, 5))
i2 = keras.layers.Input(shape=(4, 5))
i3 = keras.layers.Input(shape=(4, 5))
subtract_layer = keras.layers.Subtract()
o = subtract_layer([i1, i2])
self.assertListEqual(o.shape.as_list(), [None, 4, 5])
model = keras.models.Model([i1, i2], o)
model.run_eagerly = testing_utils.should_run_eagerly()
model._run_distributed = testing_utils.should_run_distributed()
x1 = np.random.random((2, 4, 5))
x2 = np.random.random((2, 4, 5))
out = model.predict([x1, x2])
self.assertEqual(out.shape, (2, 4, 5))
self.assertAllClose(out, x1 - x2, atol=1e-4)
self.assertEqual(subtract_layer.compute_mask([i1, i2], [None, None]),
None)
self.assertTrue(
np.all(
K.eval(
subtract_layer.compute_mask(
[i1, i2],
[K.variable(x1), K.variable(x2)]))))
with self.assertRaisesRegexp(ValueError, "`mask` should be a list."):
subtract_layer.compute_mask([i1, i2], x1)
with self.assertRaisesRegexp(ValueError, "`inputs` should be a list."):
subtract_layer.compute_mask(i1, [None, None])
with self.assertRaisesRegexp(
ValueError, "layer should be called on exactly 2 inputs"):
subtract_layer([i1, i2, i3])
with self.assertRaisesRegexp(
ValueError, "layer should be called on exactly 2 inputs"):
subtract_layer([i1])
def maybe_load_initial_epoch_from_ckpt(self, initial_epoch, mode):
"""Maybe load initial epoch from ckpt considering possible worker recovery.
When `_ckpt_saved_epoch` attribute exists and is not
`CKPT_SAVED_EPOCH_UNUSED_VALUE`, this is under multi-worker training setting
and indicates the worker is recovering from previous failure. In this case,
infer `initial_epoch` from `self._ckpt_saved_epoch` to continue previous
unfinished training from certain epoch.
Arguments:
initial_epoch: The original initial_epoch user passes in in `fit()`.
mode: The mode for running `model.fit()`.
Returns:
If the training is recovering from previous failure under multi-worker
training setting, return the epoch the training is supposed to continue
at. Otherwise, return the `initial_epoch` the user passes in.
"""
epoch = K.eval(self._ckpt_saved_epoch)
if mode == mode_keys.ModeKeys.TRAIN and epoch >= 0:
# The most recently saved epoch is one epoch prior to the epoch it
# failed at, so return the value of 'self._ckpt_saved_epoch' plus one.
return epoch + 1
return initial_epoch
def assert_optimizer_iterations_increases(self, optimizer):
with self.cached_session():
input_a = keras.Input(shape=(4,))
dense_1 = keras.layers.Dense(4,)
dense_2 = keras.layers.Dense(4,)
x_a = dense_1(input_a)
x_a = keras.layers.Dropout(0.5)(x_a)
x_a = keras.layers.BatchNormalization()(x_a)
x_a = dense_2(x_a)
model = keras.models.Model(input_a, x_a)
model.compile(optimizer, 'mse',
metrics=['acc', metrics.categorical_accuracy])
global_step = keras.backend.variable(123, dtype=dtypes.int64)
clone_model = models.clone_and_build_model(
model, compile_clone=True, optimizer_iterations=global_step)
inp = np.random.random((10, 4))
out = np.random.random((10, 4))
clone_model.train_on_batch(inp, out)
self.assertEqual(K.eval(global_step), 124)
def assert_optimizer_iterations_increases(self, optimizer):
with self.cached_session():
input_a = keras.Input(shape=(4,))
dense_1 = keras.layers.Dense(4,)
dense_2 = keras.layers.Dense(4,)
x_a = dense_1(input_a)
x_a = keras.layers.Dropout(0.5)(x_a)
x_a = keras.layers.BatchNormalization()(x_a)
x_a = dense_2(x_a)
model = keras.models.Model(input_a, x_a)
model.compile(optimizer, 'mse',
metrics=['acc', metrics.categorical_accuracy])
global_step = keras.backend.variable(123, dtype=dtypes.int64)
clone_model = models.clone_and_build_model(
model, compile_clone=True, optimizer_iterations=global_step)
inp = np.random.random((10, 4))
out = np.random.random((10, 4))
clone_model.train_on_batch(inp, out)
self.assertEqual(K.eval(global_step), 124)
def test_merge_concatenate(self):
if testing_utils.should_run_distributed():
self.skipTest("b/137397816")
i1 = keras.layers.Input(shape=(4, 5))
i2 = keras.layers.Input(shape=(4, 5))
concat_layer = keras.layers.Concatenate(axis=1)
o = concat_layer([i1, i2])
self.assertListEqual(o.shape.as_list(), [None, 8, 5])
model = keras.models.Model([i1, i2], o)
model.run_eagerly = testing_utils.should_run_eagerly()
model._run_distributed = testing_utils.should_run_distributed()
x1 = np.random.random((2, 4, 5))
x2 = np.random.random((2, 4, 5))
out = model.predict([x1, x2])
self.assertEqual(out.shape, (2, 8, 5))
self.assertAllClose(out, np.concatenate([x1, x2], axis=1), atol=1e-4)
self.assertEqual(concat_layer.compute_mask([i1, i2], [None, None]),
None)
self.assertTrue(
np.all(
K.eval(
concat_layer.compute_mask(
[i1, i2],
[K.variable(x1), K.variable(x2)]))))
with self.assertRaisesRegexp(ValueError, "`mask` should be a list."):
concat_layer.compute_mask([i1, i2], x1)
with self.assertRaisesRegexp(ValueError, "`inputs` should be a list."):
concat_layer.compute_mask(i1, [None, None])
with self.assertRaisesRegexp(ValueError,
"should have the same length"):
concat_layer.compute_mask([i1, i2], [None])
with self.assertRaisesRegexp(
ValueError, "layer should be called on a list of inputs"):
concat_layer(i1)
def test_merge_concatenate(self):
i1 = keras.layers.Input(shape=(4, 5))
i2 = keras.layers.Input(shape=(4, 5))
concat_layer = keras.layers.Concatenate(axis=1)
o = concat_layer([i1, i2])
self.assertListEqual(o.shape.as_list(), [None, 8, 5])
model = keras.models.Model([i1, i2], o)
model.run_eagerly = testing_utils.should_run_eagerly()
x1 = np.random.random((2, 4, 5))
x2 = np.random.random((2, 4, 5))
out = model.predict([x1, x2])
self.assertEqual(out.shape, (2, 8, 5))
self.assertAllClose(out, np.concatenate([x1, x2], axis=1), atol=1e-4)
self.assertEqual(concat_layer.compute_mask([i1, i2], [None, None]),
None)
self.assertTrue(
np.all(
K.eval(
concat_layer.compute_mask(
[i1, i2],
[K.variable(x1), K.variable(x2)]))))
# Should work with unit-length input.
unit_length_o = concat_layer([i1])
self.assertListEqual(unit_length_o.shape.as_list(), i1.shape.as_list())
with self.assertRaisesRegex(ValueError, '`mask` should be a list.'):
concat_layer.compute_mask([i1, i2], x1)
with self.assertRaisesRegex(ValueError, '`inputs` should be a list.'):
concat_layer.compute_mask(i1, [None, None])
with self.assertRaisesRegex(ValueError, 'should have the same length'):
concat_layer.compute_mask([i1, i2], [None])
with self.assertRaisesRegex(
ValueError, 'layer should be called on a list of inputs'):
concat_layer(i1)
def on_epoch_end(self, epoch, logs=None):
X_dev, y_dev = self.validation_data
preds = self.model.predict(X_dev) # [B, T, N]
X, Y, Z = 0, 0, 0
trans_matrix = K.eval(self.crf.trans)
for i, score in enumerate(preds):
rel_len = len([j for j in y_dev[i]
if j != -1]) # real sequence length
pred_path = viterbi_decode(score[:rel_len], trans_matrix)
rel_path = y_dev[i][1:rel_len - 1] # exclude [CLS], [SEP]
pred_path = pred_path[1:-1]
assert len(pred_path) == len(rel_path)
pred = set([tag for tag in pred_path if tag > 0]) # exclude 'O'
rel = set([tag for tag in rel_path if tag > 0])
X += len(pred & rel)
Y += len(pred)
Z += len(rel)
precision = X / Y
recall = X / Z
f1 = 2 * X / (Y + Z)
if f1 >= self.best_val_f1:
self.best_val_f1 = f1
self.model.save_weights(
os.path.join(opts.save_dir, 'best_model.weights'))
# self.model.save(os.path.join(self.opts.save_dir, 'best_model.h5'))
print('test: f1: %.5f, precision: %.5f, recall: %.5f\n' %
(f1, precision, recall))
def test_merge_add(self):
i1 = keras.layers.Input(shape=(4, 5))
i2 = keras.layers.Input(shape=(4, 5))
i3 = keras.layers.Input(shape=(4, 5))
add_layer = keras.layers.Add()
o = add_layer([i1, i2, i3])
self.assertListEqual(o.shape.as_list(), [None, 4, 5])
model = keras.models.Model([i1, i2, i3], o)
model.run_eagerly = testing_utils.should_run_eagerly()
model._experimental_run_tf_function = testing_utils.should_run_tf_function(
)
x1 = np.random.random((2, 4, 5))
x2 = np.random.random((2, 4, 5))
x3 = np.random.random((2, 4, 5))
out = model.predict([x1, x2, x3])
self.assertEqual(out.shape, (2, 4, 5))
self.assertAllClose(out, x1 + x2 + x3, atol=1e-4)
self.assertEqual(
add_layer.compute_mask([i1, i2, i3], [None, None, None]), None)
self.assertTrue(
np.all(
K.eval(
add_layer.compute_mask(
[i1, i2],
[K.variable(x1), K.variable(x2)]))))
with self.assertRaisesRegexp(ValueError, "`mask` should be a list."):
add_layer.compute_mask([i1, i2, i3], x1)
with self.assertRaisesRegexp(ValueError, "`inputs` should be a list."):
add_layer.compute_mask(i1, [None, None, None])
with self.assertRaisesRegexp(ValueError,
" should have the same length."):
add_layer.compute_mask([i1, i2, i3], [None, None])
def test(opts):
tokenizer = MODEL_ZOOS[opts.model_name][1].from_pretrained(
opts.pretrained_path)
dataset = TASK_NAMES[opts.task_name](opts.data_dir, tokenizer,
opts.max_seq_len)
X_test, y_test = dataset.get_test_datasets()
model, crf = get_ner_model(opts, len(dataset.get_labels()))
model.load_weights(opts.save_dir)
trans_matrix = K.eval(crf.trans)
preds = model.predict(X_test)
X, Y, Z = 0, 0, 0
for i, score in enumerate(preds):
rel_len = len([j for j in y_test[i]
if j != -1]) # real sequence length
pred_path = viterbi_decode(score[:rel_len], trans_matrix)
rel_path = y_test[i][1:rel_len - 1] # exclude [CLS], [SEP]
pred_path = pred_path[1:-1]
assert len(pred_path) == len(rel_path)
pred = set([tag for tag in pred_path if tag > 0]) # exclude 'O'
rel = set([tag for tag in rel_path if tag > 0])
X += len(pred & rel)
Y += len(pred)
Z += len(rel)
precision = X / Y
recall = X / Z
f1 = 2 * X / (Y + Z)
print('test: f1: %.5f, precision: %.5f, recall: %.5f\n' %
(f1, precision, recall))
def test_sparse_categorical_accuracy_float(self):
with self.cached_session():
metric = metrics.sparse_categorical_accuracy
y_true = K.variable(np.random.random((6,)))
y_pred = K.variable(np.random.random((6, 7)))
self.assertEqual(K.eval(metric(y_true, y_pred)).shape, (6,))
def read_dataset(type='twitter', mode='train', embedding_dim=100, max_seq_len=40, max_aspect_len=3, polarities_dim=3):
print("preparing data...")
fname = {
'twitter': {
'train': './datasets/acl-14-short-data/train.raw',
'test': './datasets/acl-14-short-data/test.raw'
},
'restaurant': {
'train': './datasets/semeval14/Restaurants_Train.xml.seg',
'test': './datasets/semeval14/Restaurants_Test_Gold.xml.seg'
},
'laptop': {
'train': './datasets/semeval14/Laptops_Train.xml.seg',
'test': './datasets/semeval14/Laptops_Test_Gold.xml.seg'
}
}
fin = open(fname[type][mode], 'r', encoding='utf-8', newline='\n', errors='ignore')
lines = fin.readlines()
fin.close()
print("number of {0} {1} data: {2}".format(type, mode ,len(lines)/3))
text = ''
texts_raw = []
texts_raw_without_aspects = []
texts_left = []
texts_left_with_aspects = []
texts_right = []
texts_right_with_aspects = []
aspects = []
polarities = []
for i in range(0, len(lines), 3):
text_left, _, text_right = [s.lower().strip() for s in lines[i].partition("$T$")]
aspect = lines[i+1].lower().strip()
polarity = lines[i+2].strip()
text_raw = text_left + " " + aspect + " " + text_right
text += text_raw
texts_raw.append(text_raw)
texts_raw_without_aspects.append(text_left + " " + text_right)
texts_left.append(text_left)
texts_left_with_aspects.append(text_left + " " + aspect)
texts_right.append(text_right)
texts_right_with_aspects.append(aspect + " " + text_right)
aspects.append(aspect)
polarities.append(int(polarity))
text_words = text.strip().split()
print('tokenizing...')
tokenizer = Tokenizer(lower=False)
tokenizer.fit_on_texts(text_words)
word_index = tokenizer.word_index
texts_raw_indices = tokenizer.texts_to_sequences(texts_raw)
texts_raw_indices = pad_sequences(texts_raw_indices, maxlen=max_seq_len)
texts_raw_without_aspects_indices = tokenizer.texts_to_sequences(texts_raw_without_aspects)
texts_raw_without_aspects_indices = pad_sequences(texts_raw_without_aspects_indices, maxlen=max_seq_len)
texts_left_indices = tokenizer.texts_to_sequences(texts_left)
texts_left_indices = pad_sequences(texts_left_indices, maxlen=max_seq_len)
texts_left_with_aspects_indices = tokenizer.texts_to_sequences(texts_left_with_aspects)
texts_left_with_aspects_indices = pad_sequences(texts_left_with_aspects_indices, maxlen=max_seq_len)
texts_right_indices = tokenizer.texts_to_sequences(texts_right)
texts_right_indices = pad_sequences(texts_right_indices, maxlen=max_seq_len, padding='post', truncating='post')
texts_right_with_aspects_indices = tokenizer.texts_to_sequences(texts_right_with_aspects)
texts_right_with_aspects_indices = pad_sequences(texts_right_with_aspects_indices, maxlen=max_seq_len, padding='post', truncating='post')
aspects_indices = tokenizer.texts_to_sequences(aspects)
aspects_indices = pad_sequences(aspects_indices, maxlen=max_aspect_len)
polarities_matrix = K.eval(tf.one_hot(indices=polarities, depth=polarities_dim))
if mode == 'test':
return texts_raw_indices, texts_raw_without_aspects_indices, texts_left_indices, texts_left_with_aspects_indices, \
aspects_indices, texts_right_indices, texts_right_with_aspects_indices, polarities_matrix
embedding_matrix_file_name = '{0}_{1}_embedding_matrix.dat'.format(str(embedding_dim), type)
if os.path.exists(embedding_matrix_file_name):
print('loading embedding_matrix:', embedding_matrix_file_name)
embedding_matrix = pickle.load(open(embedding_matrix_file_name, 'rb'))
else:
print('loading word vectors...')
word_vec = load_word_vec(word_index, embedding_dim)
embedding_matrix = np.zeros((len(word_index) + 1, embedding_dim))
print('building embedding_matrix:', embedding_matrix_file_name)
for word, i in word_index.items():
vec = word_vec.get(word)
if vec is not None:
# words not found in embedding index will be all-zeros.
embedding_matrix[i] = vec
pickle.dump(embedding_matrix, open(embedding_matrix_file_name, 'wb'))
return texts_raw_indices, texts_raw_without_aspects_indices, texts_left_indices, texts_left_with_aspects_indices, \
aspects_indices, texts_right_indices, texts_right_with_aspects_indices, \
polarities_matrix, \
embedding_matrix, \
tokenizer
def test_sparse_categorical_accuracy_float(self):
with self.cached_session():
metric = metrics.sparse_categorical_accuracy
y_true = K.variable(np.random.random((6, )))
y_pred = K.variable(np.random.random((6, 7)))
self.assertEqual(K.eval(metric(y_true, y_pred)).shape, (6, ))
def test_sparse_categorical_accuracy(self):
with self.test_session():
metric = metrics.sparse_categorical_accuracy
y_a = K.variable(np.random.randint(0, 7, (6,)))
y_b = K.variable(np.random.random((6, 7)))
self.assertEqual(K.eval(metric(y_a, y_b)).shape, (6,))
def test_dynamic_loss_scaling(self, strategy_fn, cloning=True):
if testing_utils.should_run_distributed():
self.skipTest('b/137397816')
if not self._is_strategy_supported(strategy_fn):
return
strategy = strategy_fn()
initial_loss_scale = 2.
batch_size = 4
expected_gradient = backend.variable([initial_loss_scale / batch_size],
dtype=dtypes.float16)
# If this variable is set to True, the model below will have NaN gradients
have_nan_gradients = backend.variable(False, dtype=dtypes.bool)
with strategy.scope():
with policy.policy_scope(policy.Policy('infer_float32_vars')):
x = layers.Input(shape=(1, ),
batch_size=batch_size,
dtype=dtypes.float16)
layer = AddLayer(assert_type=dtypes.float16)
y = layer(x)
identity_with_nan_grads = (
mp_test_util.create_identity_with_nan_gradients_fn(
have_nan_gradients))
y = core.Lambda(identity_with_nan_grads)(y)
identity_with_grad_check_fn = (
mp_test_util.create_identity_with_grad_check_fn(
expected_dtype=dtypes.float16,
expected_gradient=expected_gradient))
y = core.Lambda(identity_with_grad_check_fn)(y)
y = math_ops.cast(y, dtypes.float32)
model = models.Model(inputs=x, outputs=y)
def loss_fn(y_true, y_pred):
del y_true
return math_ops.reduce_mean(y_pred)
opt = gradient_descent.SGD(1.)
loss_scale = loss_scale_module.DynamicLossScale(
initial_loss_scale=initial_loss_scale, increment_period=2)
opt = loss_scale_optimizer.LossScaleOptimizer(opt, loss_scale)
model.compile(
opt,
loss=loss_fn,
cloning=cloning,
run_eagerly=testing_utils.should_run_eagerly(),
run_distributed=testing_utils.should_run_distributed())
self.assertEqual(backend.eval(layer.v), 1)
x = np.ones((batch_size, 1))
y = np.ones((batch_size, 1))
dataset = dataset_ops.Dataset.from_tensor_slices(
(x, y)).batch(batch_size)
model.fit(dataset)
# The variables starts with 1 and has a gradient of 1, so will go down by 1
# each step.
self.assertEqual(backend.eval(layer.v), 0)
model.fit(dataset)
self.assertEqual(backend.eval(layer.v), -1)
# There have been two steps without NaNs, so the loss scale will double
backend.set_value(expected_gradient,
backend.get_value(expected_gradient * 2))
model.fit(dataset)
self.assertEqual(backend.eval(layer.v), -2)
# Next test with NaN gradients.
backend.set_value(have_nan_gradients, True)
model.fit(dataset)
# Variable should not be updated
self.assertEqual(backend.eval(layer.v), -2)
# Test with finite gradients again
backend.set_value(have_nan_gradients, False)
# The loss scale will be halved due to the NaNs, so the gradient will also
# be halved
backend.set_value(expected_gradient,
backend.get_value(expected_gradient / 2))
model.fit(dataset)
self.assertEqual(backend.eval(layer.v), -3)
def test_dynamic_loss_scaling(self, strategy_fn, cloning=True):
strategy = strategy_fn()
initial_loss_scale = 2.
batch_size = 4
expected_gradient = backend.variable([initial_loss_scale / batch_size],
dtype=dtypes.float16)
# If this variable is set to True, the model below will have NaN gradients
have_nan_gradients = backend.variable(False, dtype=dtypes.bool)
with strategy.scope():
with policy.policy_scope(policy.Policy('infer_float32_vars')):
x = layers.Input(shape=(1,), batch_size=batch_size,
dtype=dtypes.float16)
layer = AddLayer(assert_type=dtypes.float16)
y = layer(x)
identity_with_nan_grads = (
mp_test_util.create_identity_with_nan_gradients_fn(
have_nan_gradients))
y = core.Lambda(identity_with_nan_grads)(y)
identity_with_grad_check_fn = (
mp_test_util.create_identity_with_grad_check_fn(
expected_dtype=dtypes.float16,
expected_gradient=expected_gradient))
y = core.Lambda(identity_with_grad_check_fn)(y)
y = math_ops.cast(y, dtypes.float32)
model = models.Model(inputs=x, outputs=y)
def loss_fn(y_true, y_pred):
del y_true
return math_ops.reduce_mean(y_pred)
opt = gradient_descent.SGD(1.)
loss_scale = loss_scale_module.DynamicLossScale(
initial_loss_scale=initial_loss_scale, increment_period=2)
opt = loss_scale_optimizer.LossScaleOptimizer(opt, loss_scale)
model.compile(opt, loss=loss_fn, cloning=cloning)
self.assertEqual(backend.eval(layer.v), 1)
x = np.ones((batch_size, 1))
y = np.ones((batch_size, 1))
dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).batch(batch_size)
model.fit(dataset)
# The variables starts with 1 and has a gradient of 1, so will go down by 1
# each step.
self.assertEqual(backend.eval(layer.v), 0)
model.fit(dataset)
self.assertEqual(backend.eval(layer.v), -1)
# There have been two steps without NaNs, so the loss scale will double
backend.set_value(expected_gradient,
backend.get_value(expected_gradient * 2))
model.fit(dataset)
self.assertEqual(backend.eval(layer.v), -2)
# Next test with NaN gradients.
backend.set_value(have_nan_gradients, True)
model.fit(dataset)
# Variable should not be updated
self.assertEqual(backend.eval(layer.v), -2)
# Test with finite gradients again
backend.set_value(have_nan_gradients, False)
# The loss scale will be halved due to the NaNs, so the gradient will also
# be halved
backend.set_value(expected_gradient,
backend.get_value(expected_gradient / 2))
model.fit(dataset)
self.assertEqual(backend.eval(layer.v), -3)
test_accuracy = tf.keras.metrics.Accuracy(name='test_accuracy')
# m0, m1 = modelaki.call(image_batch)
EPOCHS = 5
CHECKEPOCHS = 1
# train_loss = tf.keras.metrics.Mean(name='train_loss')
# image_batch, label_batch = next(iter(DS))
for epoch in range(EPOCHS):
train_loss_results = []
train_accuracy_results = []
for images, labels in DS:
#if images.shape[0] == 4:
#m0, m1, mask0, mask1, croped0, croped1, newmask0, newmask1 = modelaki(images)
croped0, newmask0 = train_step(modelaki, images, loss_fun, opt_fun, batch_size=4)
tf.print(croped0.shape, newmask0.shape)
plt.imshow(K.eval(croped0)[0])#.astype(np.float32) #, interpolation='nearest')
plt.show()
plt.imshow(images[0].numpy().astype(np.float32))
plt.show()
#plt.imshow(K.eval(newmask0)[0].astype(np.uint8))
plt.imshow(K.eval(newmask0)[0])
plt.show()
tf.print('Epoch {}, train_Loss: {}\n'.format(epoch + 1, train_loss.result()))
# train_loss_results.append(train_loss.result())
# train_accuracy_results.append(train_accuracy.result())
# print('Epoch {}, Loss: {}'.format(epoch + 1, train_loss.result()))
def fit(self, train_generator, val_generator, y):
"""
Fit predictor on training data.
Parameters
----------
#TODO
Returns
-------
Notes
-----
"""
# multi gpu
if self.n_gpu > 1:
# cpu_merge used for nv_link : https://keras.io/utils/#multi_gpu_model
# otherwise use : model = multi_gpu_model(model, cpu_relocation=True)
# to train models with weights merge on CPU using cpu_relocation
self.model = multi_gpu_model(self.model,
gpus=self.n_gpu,
cpu_merge=False)
self.model.compile(optimizer=self.optimizer,
loss='categorical_crossentropy',
metrics=self.metrics)
# sample weights
y_decoded = np.argmax(y, axis=1)
class_weights = class_weight.compute_class_weight(
'balanced', np.unique(y_decoded), y_decoded)
class_weights = dict(enumerate(class_weights))
print(class_weights)
# callbacks
reduce_lr_plateau = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=10,
min_lr=1e-7,
epsilon=0.05,
verbose=1)
callbacks_fit = []
if "plateau" in self.callbacks:
callbacks_fit.append(reduce_lr_plateau)
fit = self.model.fit_generator(
generator=train_generator,
steps_per_epoch=self.n_passes_by_epoch *
len(train_generator.indexes) / self.batch_size,
epochs=int(self.epochs / self.n_passes_by_epoch),
class_weight=class_weights,
validation_data=val_generator,
callbacks=callbacks_fit,
)
#use_multiprocessing=True, workers=4)
self.last_history = {
key:
[float(x)
for x in array] # To make it serializable contrary to np.float64
for key, array in fit.history.items()
}
if self.epochs_reduced_lr > 0:
K.set_value(self.model.optimizer.lr, self.lr / 10)
print('The LR is now at:', K.eval(self.model.optimizer.lr))
fit = self.model.fit_generator(
generator=train_generator,
steps_per_epoch=self.n_passes_by_epoch *
len(train_generator.indexes) / self.batch_size,
epochs=self.epochs_reduced_lr,
class_weight=class_weights,
validation_data=val_generator,
callbacks=callbacks_fit,
)
#use_multiprocessing=True, workers=4)
for key, array in fit.history.items():
self.last_history[key].extend([float(x) for x in array])
if "samples_per_sec" in self.callbacks:
samples_per_sec.print_results()
def test_jaccrad_loss(case):
gt, pr, res = case
gt = _to_4d(gt)
pr = _to_4d(pr)
score = K.eval(jaccard_loss(gt, pr))
assert np.allclose(score, 1 - res)
def get_config(self):
config = {}
for k, v in six.iteritems(self._fn_kwargs):
config[k] = K.eval(v) if tf_utils.is_tensor_or_variable(v) else v
base_config = super(LossFunctionWrapper, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
original = adj.copy()
train = adj.copy()
print('\nCompiling autoencoder model...\n')
encoder, ae = autoencoder(sparse_net=False, adj=adj)
print(ae.summary())
# Specify some hyperparameters
epochs = 50
print('\nFitting autoencoder model...\n')
generator = generate_data(original, train)
for e in range(epochs):
print('\nEpoch {:d}/{:d}'.format(e + 1, epochs))
print('Learning rate: {:6f}'.format(K.eval(ae.optimizer.lr)))
curr_iter = 0
train_loss = []
batch_adj, batch_train = next(generator)
# Each iteration/loop is a batch of train_batch_size samples
res = ae.train_on_batch([batch_adj], [batch_train])
train_loss.append(res)
train_loss = np.asarray(train_loss)
train_loss = np.mean(train_loss, axis=0)
print('Avg. training loss: {:6f}'.format(train_loss))
encoder.save_weights(f'data/cora/graph_encoder_epoch_{e}.h5')
print('\nEvaluating reconstruction performance...')
reconstruction = ae.predict([original])
print('Computing precision@k...')
def read_dataset(types=['twitter', 'restaurant'],
mode='train',
embedding_dim=100,
max_seq_len=40,
max_aspect_len=3,
polarities_dim=3):
#print("preparing data...")
fname = {
'twitter': {
'train': './data/twitter/all.raw',
'test': './data/twitter/test.raw',
'validate': './data/twitter/test.raw'
},
'restaurant': {
'train': './data/restaurant_1/all.raw',
'test': './data/restaurant_1/rest2014test.raw',
# 'train': './data/restaurant/train.raw',
# 'test': './data/restaurant/test.raw',
'validate': './data/restaurant/test.raw'
},
'laptop': {
'train': './data/laptop_1/all.raw',
'test': './data/laptop_1/test.raw',
# 'train': './data/laptop/train.raw',
# 'test': './data/laptop/test_short.raw',
'validate': './data/laptop/test_short.raw'
},
'hotel': {
'train': './data/laptop_1/train.raw',
'test': './data/hotel/hotel.raw',
# 'train': './data/laptop/train.raw',
# 'test': './data/laptop/test_short.raw',
'validate': './data/laptop/test_short.raw'
}
}
texts_raw = []
texts_raw_without_aspects = []
texts_left = []
texts_left_with_aspects = []
texts_right = []
texts_right_with_aspects = []
aspects = []
polarities = []
dataset_index = []
word_dict, word_vec = load_word_vec(embedding_dim)
for j in range(len(types)):
fin = open(fname[types[j]][mode],
'r',
encoding='utf8',
newline='\n',
errors='ignore')
lines = fin.readlines()
fin.close()
print("number of {0} {1} data: {2}".format(types[j], mode,
len(lines) / 3))
for i in range(0, len(lines), 3):
text_left, _, text_right = [
s.lower().strip().split()
for s in lines[i].lower().partition("$t$")
]
if (len(text_left) < max_seq_len
and len(text_right) < max_seq_len):
text_left = textCleaner(text_left)
text_right = textCleaner(text_right)
text_left = " ".join(text_left)
text_right = " ".join(text_right)
aspect = lines[i + 1].lower().strip()
aspect = " ".join(textCleaner(aspect))
polarity = lines[i + 2].strip()
text_left = ". " + text_left
text_right = text_right + " ."
text_raw = text_left + " " + aspect + " " + text_right
texts_raw.append(text_raw)
texts_raw_without_aspects.append(text_left + " " + text_right)
texts_left.append(text_left)
texts_left_with_aspects.append(text_left + " " + aspect)
texts_right.append(text_right)
texts_right_with_aspects.append(aspect + " " + text_right)
aspects.append(aspect)
polarities.append(int(polarity))
if (types[j] == 'twitter'):
dataset_index.append([1001])
elif (types[j] == 'restaurant'):
dataset_index.append([1002])
elif (types[j] == 'laptop'):
dataset_index.append([1003])
else:
dataset_index.append([1004])
polarities_k = np.array(polarities)
polarities_k[polarities_k == -1] = 2
polarities_matrix = K.eval(
tf.one_hot(indices=polarities_k, depth=polarities_dim + 1))
polarities = K.eval(tf.one_hot(indices=polarities_k, depth=polarities_dim))
text_words = word_dict.strip().split()
#print('tokenizing...')
tokenizer = Tokenizer(filters="\t\n")
tokenizer.fit_on_texts(text_words)
texts_raw_indices = tokenizer.texts_to_sequences(texts_raw)
texts_raw_indices = pad_sequences(texts_raw_indices,
maxlen=max_seq_len,
padding='post')
texts_raw_without_aspects_indices = tokenizer.texts_to_sequences(
texts_raw_without_aspects)
texts_raw_without_aspects_indices = pad_sequences(
texts_raw_without_aspects_indices, maxlen=max_seq_len, padding='post')
texts_left_indices = tokenizer.texts_to_sequences(texts_left)
texts_left_indices = pad_sequences(texts_left_indices,
maxlen=max_seq_len,
padding='post')
texts_left_with_aspects_indices = tokenizer.texts_to_sequences(
texts_left_with_aspects)
texts_left_with_aspects_indices = pad_sequences(
texts_left_with_aspects_indices, maxlen=max_seq_len, padding='post')
texts_right_indices = tokenizer.texts_to_sequences(texts_right)
texts_right_indices = pad_sequences(texts_right_indices,
maxlen=max_seq_len,
truncating='post')
texts_right_with_aspects_indices = tokenizer.texts_to_sequences(
texts_right_with_aspects)
texts_right_with_aspects_indices = pad_sequences(
texts_right_with_aspects_indices,
maxlen=max_seq_len,
truncating='post')
aspects_indices = tokenizer.texts_to_sequences(aspects)
aspects_indices = pad_sequences(aspects_indices,
maxlen=max_aspect_len,
padding='post')
for i in range(len(polarities_matrix)):
polarities_matrix[i][3] = dataset_index[i][0]
dataset_index = np.array(dataset_index)
dataset_index = dataset_index.astype('float64')
if mode == 'validate' or mode == 'test':
return texts_raw_indices, texts_raw_without_aspects_indices, texts_left_indices, texts_left_with_aspects_indices, \
aspects_indices, texts_right_indices, texts_right_with_aspects_indices,dataset_index, polarities_matrix,polarities_k
#print('loading word vectors...')
embedding_matrix = word_vec
return texts_raw_indices, texts_raw_without_aspects_indices, texts_left_indices, texts_left_with_aspects_indices, \
aspects_indices, texts_right_indices, texts_right_with_aspects_indices, \
dataset_index,polarities_matrix,polarities_k, \
embedding_matrix, \
tokenizer
def train(self):
# start training
flip = False
variance = 1 / 127.5
start_time = time.time()
for epoch in range(1, self.epochs + 1):
# create batch generator at each epoch
batch_generator = DataGenerator(image_size=self.image_size,
batch_size=self.batch_size)
batch_end = len(batch_generator)
print('Epoch {}'.format(epoch))
# start training for each batch
for idx, (photo, cartoon, smooth_cartoon,
index) in enumerate(batch_generator):
# these two tensors measure the output of generator and discriminator
real = np.ones((self.batch_size, ) + (64, 64, 1))
fake = np.zeros((self.batch_size, ) + (64, 64, 1))
# check if it is the end of an epoch
if index + 1 == batch_end:
break
# initial training or start training
if epoch < self.init_epoch:
g_loss = self.train_generator.train_on_batch(
photo, [photo, real])
generated_img = self.generator.predict(photo)
print(
"Batch %d (initial training for generator), g_loss: %.5f, with time: %4.4f"
% (idx, g_loss[2], time.time() - start_time))
start_time = time.time()
write_log(self.callback2, 'g_loss', g_loss[2],
idx + (epoch + 1) * len(batch_generator))
if idx % 20 == 0:
write_images(self.callback3, generated_img,
'generated_imgs',
idx + (epoch + 1) * len(batch_generator))
if epoch % 20 == 0 and K.eval(
self.train_generator.optimizer.lr) > 0.0001:
K.set_value(
self.train_generator.optimizer.lr,
K.eval(self.train_generator.optimizer.lr) * 0.99)
else:
# add noise to the input of discriminator
if variance > 0.00001:
variance = variance * 0.9999
gaussian = np.random.normal(
0, variance, (cartoon.shape[1], cartoon.shape[2]))
cartoon[:, :, :, 0] = cartoon[:, :, :, 0] + gaussian
cartoon[:, :, :, 1] = cartoon[:, :, :, 1] + gaussian
cartoon[:, :, :, 2] = cartoon[:, :, :, 2] + gaussian
gaussian = np.random.normal(
0, variance, (cartoon.shape[1], cartoon.shape[2]))
smooth_cartoon[:, :, :,
0] = smooth_cartoon[:, :, :,
0] + gaussian
smooth_cartoon[:, :, :,
1] = smooth_cartoon[:, :, :,
1] + gaussian
smooth_cartoon[:, :, :,
2] = smooth_cartoon[:, :, :,
2] + gaussian
# generate cartoonized images
generated_img = self.generator.predict(photo)
# to certain probability: flip the label of discriminator
if idx % 9 == 0 or np.random.uniform(0, 1) < 0.05:
real = fake
fake = fake + 1
flip = True
# train discriminator and adversarial loss
real_loss = self.discriminator.train_on_batch(
cartoon, real)
smooth_loss = self.discriminator.train_on_batch(
smooth_cartoon, fake)
fake_loss = self.discriminator.train_on_batch(
generated_img, fake)
d_loss = (real_loss + smooth_loss + fake_loss) / 3
# train generator
if flip:
real = fake
fake = fake - 1
flip = False
g_loss = self.train_generator.train_on_batch(
photo, [photo, real])
print(
"Batch %d, d_loss: %.5f, g_loss: %.5f, with time: %4.4f"
% (idx, d_loss, g_loss[2], time.time() - start_time))
start_time = time.time()
# add losses to writer
write_log(self.callback1, 'd_loss', d_loss,
idx + (epoch + 1) * len(batch_generator))
write_log(self.callback2, 'g_loss', g_loss[2],
idx + (epoch + 1) * len(batch_generator))
if idx % 20 == 0:
write_images(self.callback3, generated_img,
'generated_imgs',
idx + (epoch + 1) * len(batch_generator))
# change learning rate
if epoch % 20 == 0 and K.eval(
self.discriminator.optimizer.lr) > 0.0001:
K.set_value(
self.discriminator.optimizer.lr,
K.eval(self.discriminator.optimizer.lr) * 0.95)
if epoch % 20 == 0 and K.eval(
self.train_generator.optimizer.lr) > 0.0001:
K.set_value(
self.train_generator.optimizer.lr,
K.eval(self.train_generator.optimizer.lr) * 0.95)
# save model
if epoch % 50 == 0:
self.generator.save_weights(
self.model_dir + '/' +
'CartoonGan_generator_epoch_{}.h5'.format(epoch))
self.discriminator.save_weights(
self.model_dir + '/' +
'CartoonGan_discriminator_epoch_{}.h5'.format(epoch))
self.train_generator.save_weights(
self.model_dir + '/' +
'CartoonGan_train_generator_epoch_{}.h5'.format(epoch))
print('Done!')
self.generator.save('CartoonGan_generator.h5')
def on_epoch_begin(self, epoch, logs=None):
# `_ckpt_saved_epoch` attribute is set at the end of every epoch.
self.test_obj.assertEqual(
K.eval(self.model._ckpt_saved_epoch) ==
training_state.CKPT_SAVED_EPOCH_UNUSED_VALUE,
epoch == 0)
def test_advanced_model(self, strategy_fn, use_loss_scaling=False):
if testing_utils.should_run_distributed():
self.skipTest('b/137397816')
# The advanced model tests mixed-precision-related features that would occur
# in a resnet50 model. It tests a model that has:
# * Multiple layers, some which use auto-cast variables and some which do
# not
# * Regularization on some variables and not others.
# * A fixed loss scale (if use_loss_scaling is True)
if not self._is_strategy_supported(strategy_fn):
return
strategy = strategy_fn()
if use_loss_scaling:
loss_scale = 8.
learning_rate = 2**-14
with strategy.scope():
with policy.policy_scope(policy.Policy('infer_float32_vars')):
x = layers.Input(shape=(1, ),
batch_size=2,
dtype=dtypes.float16)
layer1 = AddLayer(assert_type=dtypes.float16,
regularizer=IdentityRegularizer(),
use_operator=True)
layer2 = AddLayerWithoutAutoCast(assert_type=dtypes.float16,
use_operator=True)
layer3 = AddLayer(assert_type=dtypes.float16,
use_operator=False)
layer4 = AddLayerWithoutAutoCast(
assert_type=dtypes.float16,
regularizer=IdentityRegularizer(),
use_operator=False)
y = layer1(x)
y = layer2(y)
y = layer3(y)
y = layer4(y)
if use_loss_scaling:
# The gradient of 'y' at this point is 1. With loss scaling, the
# gradient is 'loss_scale'. We divide by the batch size of 2 since the
# loss is averaged across batch elements.
expected_gradient = loss_scale / 2
identity_with_grad_check_fn = (
mp_test_util.create_identity_with_grad_check_fn(
expected_dtype=dtypes.float16,
expected_gradient=[expected_gradient]))
y = core.Lambda(identity_with_grad_check_fn)(y)
y = math_ops.cast(y, dtypes.float32)
model = models.Model(inputs=x, outputs=y)
def loss_fn(y_true, y_pred):
self.assertEqual(y_true.dtype, dtypes.float32)
self.assertEqual(y_pred.dtype, dtypes.float32)
return math_ops.reduce_mean(y_pred)
opt = gradient_descent.SGD(learning_rate)
if use_loss_scaling:
opt = loss_scale_optimizer.LossScaleOptimizer(
opt, loss_scale)
model.compile(
opt,
loss=loss_fn,
run_eagerly=testing_utils.should_run_eagerly(),
run_distributed=testing_utils.should_run_distributed())
x = np.ones((2, 1))
y = np.ones((2, 1))
dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).batch(2)
model.fit(dataset)
for layer in (layer1, layer2, layer3, layer4):
if layer.losses:
# Layer has weight regularizer
self.assertEqual(backend.eval(layer.v), 1 - 2 * learning_rate)
else:
# Layer does not have weight regularizer
self.assertEqual(backend.eval(layer.v), 1 - learning_rate)
from deep_bottleneck.mi_estimator import kde
from tensorflow.python.keras import backend as K
import numpy as np
Y_samples = K.placeholder(ndim=2)
noise_variance = 0.05
entropy_func_upper = K.function([Y_samples], [kde.entropy_estimator_kl(Y_samples, noise_variance)])
entropy_func_lower = K.function([Y_samples], [kde.entropy_estimator_bd(Y_samples, noise_variance)])
data = np.random.random(size=(1000, 20)) # N x dims
H_Y_given_X = kde.kde_condentropy(data, noise_variance)
H_Y_upper = entropy_func_upper([data])[0]
H_Y_lower = entropy_func_lower([data])[0]
print("Upper bound: %0.3f nats" % (H_Y_upper - H_Y_given_X))
print("Lower bound: %0.3f nats" % (H_Y_lower - H_Y_given_X))
# Alternative calculation, direct from distance matrices
dims, N = kde.get_shape(K.variable(data))
dists = kde.Kget_dists(K.variable(data))
dists2 = dists / (2 * noise_variance)
mi2 = K.eval(-K.mean(K.logsumexp(-dists2, axis=1) - K.log(N)))
print("Upper bound2: %0.3f nats" % mi2)
dims, N = kde.get_shape(K.variable(data))
dists = kde.Kget_dists(K.variable(data))
dists2 = dists / (2 * 4 * noise_variance)
mi2 = K.eval(-K.mean(K.logsumexp(-dists2, axis=1) - K.log(N)))
print("Lower bound2: %0.3f nats" % mi2)
def test_advanced_model(self, strategy_fn, use_loss_scaling=False):
# The advanced model tests mixed-precision-related features that would occur
# in a resnet50 model. It tests a model that has:
# * Multiple layers, some which use auto-cast variables and some which do
# not
# * Regularization on some variables and not others.
# * A fixed loss scale (if use_loss_scaling is True)
strategy = strategy_fn()
if use_loss_scaling:
loss_scale = 8.
learning_rate = 2 ** -14
with strategy.scope():
with policy.policy_scope(policy.Policy('infer_float32_vars')):
x = layers.Input(shape=(1,), batch_size=2, dtype=dtypes.float16)
layer1 = AddLayer(assert_type=dtypes.float16,
regularizer=IdentityRegularizer(), use_operator=True)
layer2 = AddLayerWithoutAutoCast(assert_type=dtypes.float16,
use_operator=True)
layer3 = AddLayer(assert_type=dtypes.float16, use_operator=False)
layer4 = AddLayerWithoutAutoCast(assert_type=dtypes.float16,
regularizer=IdentityRegularizer(),
use_operator=False)
y = layer1(x)
y = layer2(y)
y = layer3(y)
y = layer4(y)
if use_loss_scaling:
# The gradient of 'y' at this point is 1. With loss scaling, the
# gradient is 'loss_scale'. We divide by the batch size of 2 since the
# loss is averaged across batch elements.
expected_gradient = loss_scale / 2
identity_with_grad_check_fn = (
mp_test_util.create_identity_with_grad_check_fn(
expected_dtype=dtypes.float16,
expected_gradient=[expected_gradient]))
y = core.Lambda(identity_with_grad_check_fn)(y)
y = math_ops.cast(y, dtypes.float32)
model = models.Model(inputs=x, outputs=y)
def loss_fn(y_true, y_pred):
self.assertEqual(y_true.dtype, dtypes.float32)
self.assertEqual(y_pred.dtype, dtypes.float32)
return math_ops.reduce_mean(y_pred)
opt = gradient_descent.SGD(learning_rate)
if use_loss_scaling:
opt = loss_scale_optimizer.LossScaleOptimizer(opt, loss_scale)
model.compile(opt, loss=loss_fn)
x = np.ones((2, 1))
y = np.ones((2, 1))
dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).batch(2)
model.fit(dataset)
for layer in (layer1, layer2, layer3, layer4):
if layer.losses:
# Layer has weight regularizer
self.assertEqual(backend.eval(layer.v), 1 - 2 * learning_rate)
else:
# Layer does not have weight regularizer
self.assertEqual(backend.eval(layer.v), 1 - learning_rate)
def test_sparse_categorical_accuracy(self):
with self.test_session():
metric = metrics.sparse_categorical_accuracy
y_a = K.variable(np.random.randint(0, 7, (6, )))
y_b = K.variable(np.random.random((6, 7)))
self.assertEqual(K.eval(metric(y_a, y_b)).shape, (6, ))
def test_dice_loss(case):
gt, pr, res = case
gt = _to_4d(gt)
pr = _to_4d(pr)
score = K.eval(dice_loss(gt, pr))
assert np.allclose(score, 1 - res)
def fit(self, X, y, validation_data=None, tensorboard=None):
"""
Fit predictor on training data.
Parameters
----------
X : array, shape = [n, img_width, img_height,3]
Array of n pictures.
y : array, shape = [n]
Array of pictures label
Returns
-------
Notes
-----
"""
# multi gpu
if self.n_gpu > 1:
# cpu_merge used for nv_link : https://keras.io/utils/#multi_gpu_model
# otherwise use : model = multi_gpu_model(model, cpu_relocation=True)
# to train models with weights merge on CPU using cpu_relocation
self.model = multi_gpu_model(self.model,
gpus=self.n_gpu,
cpu_merge=False)
self.model.compile(optimizer=self.optimizer,
loss='categorical_crossentropy',
metrics=self.metrics)
# sample weights
y_decoded = np.argmax(y, axis=1)
class_weights = class_weight.compute_class_weight(
'balanced', np.unique(y_decoded), y_decoded)
class_weights = dict(enumerate(class_weights))
print(class_weights)
# callbacks
reduce_lr_plateau = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=10,
min_lr=1e-7,
epsilon=0.05,
verbose=1)
callbacks_fit = []
if "plateau" in self.callbacks:
callbacks_fit.append(reduce_lr_plateau)
if tensorboard is not None:
callbacks_fit.append(tensorboard)
# fit
if self.datagen is None:
fit = self.model.fit(X,
y,
batch_size=self.batch_size,
epochs=self.epochs,
shuffle=True,
class_weight=class_weights,
validation_data=validation_data,
callbacks=callbacks_fit)
else:
fit = self.model.fit_generator(
self.datagen.flow(X,
y,
batch_size=self.batch_size,
shuffle=True),
steps_per_epoch=self.n_passes_by_epoch * len(X) /
self.batch_size,
epochs=int(self.epochs / self.n_passes_by_epoch),
class_weight=class_weights,
validation_data=validation_data,
callbacks=callbacks_fit)
self.last_history = {
key:
[float(x)
for x in array] # To make it serializable contrary to np.float64
for key, array in fit.history.items()
}
if self.epochs_reduced_lr > 0:
K.set_value(self.model.optimizer.lr, self.lr / 10)
print('The LR is now at:', K.eval(self.model.optimizer.lr))
if self.datagen is None:
fit = self.model.fit(X,
y,
batch_size=self.batch_size,
shuffle=True,
epochs=self.epochs_reduced_lr,
class_weight=class_weights,
validation_data=validation_data,
callbacks=callbacks_fit)
else:
fit = self.model.fit_generator(
self.datagen.flow(X,
y,
batch_size=self.batch_size,
shuffle=True),
steps_per_epoch=len(X) / self.batch_size,
epochs=self.epochs_reduced_lr,
class_weight=class_weights,
validation_data=validation_data,
callbacks=callbacks_fit)
for key, array in fit.history.items():
self.last_history[key].extend([float(x) for x in array])
if "samples_per_sec" in self.callbacks:
samples_per_sec.print_results()
def test_non_neg(self):
non_neg_instance = constraints.non_neg()
normed = non_neg_instance(backend.variable(get_example_array()))
assert np.all(np.min(backend.eval(normed), axis=1) == 0.)
def get_config(self):
config = {}
for k, v in six.iteritems(self._fn_kwargs):
config[k] = K.eval(v) if is_tensor_or_variable(v) else v
base_config = super(LossFunctionWrapper, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
use_class_probabilities_pca=use_class_probabilities_pca)
batchLossHistory = LossHistory()
tf.config.experimental_run_functions_eagerly(True) # 加入快速图
print("进行训练！！！！！！！！！！！！！！！！！！！！！！！！！！！！！")
# for epoch in range(0, no_epochs):
for epoch in range(0, no_epochs):
history = model.fit_generator(train_generator,
initial_epoch=epoch,
epochs=epoch + 1,
callbacks=[batchLossHistory, learning_rate],
max_queue_size=10,
workers=4,
use_multiprocessing=False)
epoch_loss = history.history['loss'][0]
learning_rate_hist = K.eval(model.optimizer.lr)
# Saving weights
if len(weights_filename) > 0:
logger.info(" saving model weights ...")
model.save(weights_filename)
print("模型保存至", weights_filename)
# Evaluation on test data
logger.info(" Evaluation on test data ...")
model_outputs = model.predict_generator(validation_generator,
max_queue_size=10,
workers=4,
use_multiprocessing=False,
verbose=1)
def test_model(self,
strategy_fn,
use_operator=False,
use_regularizer=False,
policy_name='mixed_float16',
get_config=False,
save_format=None,
use_input_spec=False):
self._skip_if_strategy_unsupported(strategy_fn)
self._skip_if_save_format_unsupported(save_format)
regularizer = (mp_test_util.IdentityRegularizer()
if use_regularizer else None)
with strategy_fn().scope():
# Pass loss_scale=None, as this test will fail if the DynamicLossScale
# skips applying gradients for a step
with policy.policy_scope(
policy.Policy(policy_name, loss_scale=None)):
layer = mp_test_util.MultiplyLayer(assert_type=dtypes.float16,
use_operator=use_operator,
regularizer=regularizer,
input_shape=(1, ))
if use_input_spec:
layer.input_spec = input_spec.InputSpec(shape=(2, 1))
model = testing_utils.get_model_from_layers(
[layer], input_shape=(1, ), input_dtype=dtypes.float16)
if get_config:
config = model.get_config()
model = model.__class__.from_config(
config,
custom_objects={
'MultiplyLayer': mp_test_util.MultiplyLayer
})
(layer, ) = (
layer for layer in model.layers
if isinstance(layer, mp_test_util.MultiplyLayer))
def loss_fn(y_true, y_pred):
del y_true
return math_ops.reduce_mean(y_pred)
# Learning rate is small enough that if applied to a float16 variable,
# the variable will not change. So this tests the learning rate not
# applied to a float16 value, but instead the float32 variable.
opt = gradient_descent.SGD(2**-14)
model.compile(opt,
loss=loss_fn,
run_eagerly=testing_utils.should_run_eagerly())
x = np.ones((2, 1))
y = np.ones((2, 1))
dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).batch(2)
model.fit(dataset)
# Variable starts at 1, and should have gradient of 2 ** -14 subtracted
# from it.
expected = 1 - 2**-14
if use_regularizer:
# Regularizer adds another 2 ** -14 to the gradient.
expected -= 2**-14
self.assertEqual(backend.eval(layer.v), expected)
if save_format:
with generic_utils.CustomObjectScope({
'MultiplyLayer':
mp_test_util.MultiplyLayer,
'loss_fn':
loss_fn
}):
self._test_saving(model, dataset, save_format, use_regularizer)
def test_dynamic_loss_scaling(self,
strategy_fn,
pass_loss_scale_to_policy=False,
get_config=False):
strategy = strategy_fn()
initial_loss_scale = 2.
batch_size = 4
loss_scale = loss_scale_module.DynamicLossScale(
initial_loss_scale=initial_loss_scale, increment_period=2)
expected_gradient = backend.variable([initial_loss_scale / batch_size],
dtype=dtypes.float16)
# If this variable is set to True, the model below will have NaN gradients
have_nan_gradients = backend.variable(False, dtype=dtypes.bool)
with strategy.scope():
opt = gradient_descent.SGD(1.)
if pass_loss_scale_to_policy:
p = policy.Policy('mixed_float16', loss_scale=loss_scale)
else:
p = policy.Policy('mixed_float16', loss_scale=None)
opt = loss_scale_optimizer.LossScaleOptimizer(opt, loss_scale)
with policy.policy_scope(p):
x = layers.Input(shape=(1, ),
batch_size=batch_size,
dtype=dtypes.float16)
layer = mp_test_util.MultiplyLayer(assert_type=dtypes.float16)
y = layer(x)
identity_with_nan_grads = (
mp_test_util.create_identity_with_nan_gradients_fn(
have_nan_gradients))
y = core.Lambda(identity_with_nan_grads)(y)
identity_with_grad_check_fn = (
mp_test_util.create_identity_with_grad_check_fn(
expected_dtype=dtypes.float16,
expected_gradient=expected_gradient))
y = core.Lambda(identity_with_grad_check_fn)(y)
model = models.Model(inputs=x, outputs=y)
if get_config:
config = model.get_config()
model = model.__class__.from_config(
config,
custom_objects={
'MultiplyLayer': mp_test_util.MultiplyLayer
})
(layer, ) = (
layer for layer in model.layers
if isinstance(layer, mp_test_util.MultiplyLayer))
def loss_fn(y_true, y_pred):
del y_true
return math_ops.reduce_mean(y_pred)
model.compile(opt,
loss=loss_fn,
run_eagerly=testing_utils.should_run_eagerly())
self.assertEqual(backend.eval(layer.v), 1)
x = np.ones((batch_size, 1))
y = np.ones((batch_size, 1))
dataset = dataset_ops.Dataset.from_tensor_slices(
(x, y)).batch(batch_size)
model.fit(dataset)
# The variables starts with 1 and has a gradient of 1, so will go down by 1
# each step.
self.assertEqual(backend.eval(layer.v), 0)
model.fit(dataset)
self.assertEqual(backend.eval(layer.v), -1)
# There have been two steps without NaNs, so the loss scale will double
backend.set_value(expected_gradient,
backend.get_value(expected_gradient * 2))
model.fit(dataset)
self.assertEqual(backend.eval(layer.v), -2)
# Next test with NaN gradients.
backend.set_value(have_nan_gradients, True)
model.fit(dataset)
# Variable should not be updated
self.assertEqual(backend.eval(layer.v), -2)
# Test with finite gradients again
backend.set_value(have_nan_gradients, False)
# The loss scale will be halved due to the NaNs, so the gradient will also
# be halved
backend.set_value(expected_gradient,
backend.get_value(expected_gradient / 2))
model.fit(dataset)
self.assertEqual(backend.eval(layer.v), -3)
def test_iou_metric(case):
gt, pr, res = case
gt = _to_4d(gt)
pr = _to_4d(pr)
score = K.eval(iou_score(gt, pr))
assert np.allclose(score, res)