def evaluate_error(model: training.Model) -> np.float64:
pred = model.predict(x_test, batch_size=32)
pred = np.argmax(pred, axis=1)
pred = np.expand_dims(pred, axis=1) # make same shape as y_test
error = np.sum(np.not_equal(pred, y_test)) / y_test.shape[0]
return error
def model_predict(model, embedded_ip, raw_text, label_layer_name):
total_items = embedded_ip.shape[0]
label_model = Model(inputs=model.input,
outputs=model.get_layer(label_layer_name).output)
y_prob = label_model.predict(embedded_ip).reshape(-1)
y_pred = y_prob > 0.5
return y_prob, y_pred
def get_embedded_input(model, encoded_text):
"""
Get embedding layer output from a CNN model as the input for CNN_DCNN model
"""
embedding_layer_model = Model(
inputs=model.input, outputs=model.get_layer('word_embedding').output)
return embedding_layer_model.predict(encoded_text)
def test_stateful_metrics(self):
with self.cached_session():
np.random.seed(1334)
class BinaryTruePositives(layers.Layer):
"""Stateful Metric to count the total true positives over all batches.
Assumes predictions and targets of shape `(samples, 1)`.
Arguments:
threshold: Float, lower limit on prediction value that counts as a
positive class prediction.
name: String, name for the metric.
"""
def __init__(self, name='true_positives', **kwargs):
super(BinaryTruePositives, self).__init__(name=name,
**kwargs)
self.true_positives = K.variable(value=0, dtype='int32')
self.stateful = True
def reset_states(self):
K.set_value(self.true_positives, 0)
def __call__(self, y_true, y_pred):
"""Computes the number of true positives in a batch.
Args:
y_true: Tensor, batch_wise labels
y_pred: Tensor, batch_wise predictions
Returns:
The total number of true positives seen this epoch at the
completion of the batch.
"""
y_true = math_ops.cast(y_true, 'int32')
y_pred = math_ops.cast(math_ops.round(y_pred), 'int32')
correct_preds = math_ops.cast(
math_ops.equal(y_pred, y_true), 'int32')
true_pos = math_ops.cast(
math_ops.reduce_sum(correct_preds * y_true), 'int32')
current_true_pos = self.true_positives * 1
self.add_update(state_ops.assign_add(
self.true_positives, true_pos),
inputs=[y_true, y_pred])
return current_true_pos + true_pos
metric_fn = BinaryTruePositives()
config = metrics.serialize(metric_fn)
metric_fn = metrics.deserialize(
config,
custom_objects={'BinaryTruePositives': BinaryTruePositives})
# Test on simple model
inputs = layers.Input(shape=(2, ))
outputs = layers.Dense(1, activation='sigmoid')(inputs)
model = Model(inputs, outputs)
model.compile(optimizer='sgd',
loss='binary_crossentropy',
metrics=['acc', metric_fn])
# Test fit, evaluate
samples = 100
x = np.random.random((samples, 2))
y = np.random.randint(2, size=(samples, 1))
val_samples = 10
val_x = np.random.random((val_samples, 2))
val_y = np.random.randint(2, size=(val_samples, 1))
history = model.fit(x,
y,
epochs=1,
batch_size=10,
validation_data=(val_x, val_y))
outs = model.evaluate(x, y, batch_size=10)
preds = model.predict(x)
def ref_true_pos(y_true, y_pred):
return np.sum(np.logical_and(y_pred > 0.5, y_true == 1))
# Test correctness (e.g. updates should have been run)
self.assertAllClose(outs[2], ref_true_pos(y, preds), atol=1e-5)
# Test correctness of the validation metric computation
val_preds = model.predict(val_x)
val_outs = model.evaluate(val_x, val_y, batch_size=10)
self.assertAllClose(val_outs[2],
ref_true_pos(val_y, val_preds),
atol=1e-5)
self.assertAllClose(val_outs[2],
history.history['val_true_positives'][-1],
atol=1e-5)
# Test with generators
gen = [(np.array([x0]), np.array([y0])) for x0, y0 in zip(x, y)]
val_gen = [(np.array([x0]), np.array([y0]))
for x0, y0 in zip(val_x, val_y)]
history = model.fit_generator(iter(gen),
epochs=1,
steps_per_epoch=samples,
validation_data=iter(val_gen),
validation_steps=val_samples)
outs = model.evaluate_generator(iter(gen), steps=samples)
preds = model.predict_generator(iter(gen), steps=samples)
# Test correctness of the metric results
self.assertAllClose(outs[2], ref_true_pos(y, preds), atol=1e-5)
# Test correctness of the validation metric computation
val_preds = model.predict_generator(iter(val_gen),
steps=val_samples)
val_outs = model.evaluate_generator(iter(val_gen),
steps=val_samples)
self.assertAllClose(val_outs[2],
ref_true_pos(val_y, val_preds),
atol=1e-5)
self.assertAllClose(val_outs[2],
history.history['val_true_positives'][-1],
atol=1e-5)
def test_stateful_metrics(self):
with self.test_session():
np.random.seed(1334)
class BinaryTruePositives(layers.Layer):
"""Stateful Metric to count the total true positives over all batches.
Assumes predictions and targets of shape `(samples, 1)`.
Arguments:
threshold: Float, lower limit on prediction value that counts as a
positive class prediction.
name: String, name for the metric.
"""
def __init__(self, name='true_positives', **kwargs):
super(BinaryTruePositives, self).__init__(name=name, **kwargs)
self.true_positives = K.variable(value=0, dtype='int32')
self.stateful = True
def reset_states(self):
K.set_value(self.true_positives, 0)
def __call__(self, y_true, y_pred):
"""Computes the number of true positives in a batch.
Args:
y_true: Tensor, batch_wise labels
y_pred: Tensor, batch_wise predictions
Returns:
The total number of true positives seen this epoch at the
completion of the batch.
"""
y_true = math_ops.cast(y_true, 'int32')
y_pred = math_ops.cast(math_ops.round(y_pred), 'int32')
correct_preds = math_ops.cast(math_ops.equal(y_pred, y_true), 'int32')
true_pos = math_ops.cast(
math_ops.reduce_sum(correct_preds * y_true), 'int32')
current_true_pos = self.true_positives * 1
self.add_update(
state_ops.assign_add(self.true_positives, true_pos),
inputs=[y_true, y_pred])
return current_true_pos + true_pos
metric_fn = BinaryTruePositives()
config = metrics.serialize(metric_fn)
metric_fn = metrics.deserialize(
config, custom_objects={'BinaryTruePositives': BinaryTruePositives})
# Test on simple model
inputs = layers.Input(shape=(2,))
outputs = layers.Dense(1, activation='sigmoid')(inputs)
model = Model(inputs, outputs)
model.compile(optimizer='sgd',
loss='binary_crossentropy',
metrics=['acc', metric_fn])
# Test fit, evaluate
samples = 100
x = np.random.random((samples, 2))
y = np.random.randint(2, size=(samples, 1))
val_samples = 10
val_x = np.random.random((val_samples, 2))
val_y = np.random.randint(2, size=(val_samples, 1))
history = model.fit(x, y,
epochs=1,
batch_size=10,
validation_data=(val_x, val_y))
outs = model.evaluate(x, y, batch_size=10)
preds = model.predict(x)
def ref_true_pos(y_true, y_pred):
return np.sum(np.logical_and(y_pred > 0.5, y_true == 1))
# Test correctness (e.g. updates should have been run)
self.assertAllClose(outs[2], ref_true_pos(y, preds), atol=1e-5)
# Test correctness of the validation metric computation
val_preds = model.predict(val_x)
val_outs = model.evaluate(val_x, val_y, batch_size=10)
self.assertAllClose(
val_outs[2], ref_true_pos(val_y, val_preds), atol=1e-5)
self.assertAllClose(
val_outs[2], history.history['val_true_positives'][-1], atol=1e-5)
# Test with generators
gen = [(np.array([x0]), np.array([y0])) for x0, y0 in zip(x, y)]
val_gen = [(np.array([x0]), np.array([y0]))
for x0, y0 in zip(val_x, val_y)]
history = model.fit_generator(iter(gen),
epochs=1,
steps_per_epoch=samples,
validation_data=iter(val_gen),
validation_steps=val_samples)
outs = model.evaluate_generator(iter(gen), steps=samples)
preds = model.predict_generator(iter(gen), steps=samples)
# Test correctness of the metric results
self.assertAllClose(outs[2], ref_true_pos(y, preds), atol=1e-5)
# Test correctness of the validation metric computation
val_preds = model.predict_generator(iter(val_gen), steps=val_samples)
val_outs = model.evaluate_generator(iter(val_gen), steps=val_samples)
self.assertAllClose(
val_outs[2], ref_true_pos(val_y, val_preds), atol=1e-5)
self.assertAllClose(
val_outs[2], history.history['val_true_positives'][-1], atol=1e-5)
class ChessModel:
"""
The model which can be trained to take observations of a game of chess
and return value and policy predictions. Inpired by https://github.com/Zeta36/chess-alpha-zero/blob/master/src/chess_zero/agent/model_chess.py
Attributes:
:ivar Config config: configuration to use
:ivar Model model: the Keras model to use for predictions
"""
def __init__(self, config):
self.config = config
self.model = None # type: Model
self.digest = None
self.api = None
def build(self):
"""
Builds the full Keras model and stores it in self.model.
"""
mc = self.config
in_x = x = Input((12, 8, 8))
# (batch, channels, height, width)
x = Conv2D(filters=mc.cnn_filter_num,
kernel_size=mc.cnn_first_filter_size,
padding="same",
data_format="channels_first",
use_bias=False,
kernel_regularizer=l2(mc.l2_reg),
name="input_conv-" + str(mc.cnn_first_filter_size) + "-" +
str(mc.cnn_filter_num))(x)
x = BatchNormalization(axis=1, name="input_batchnorm")(x)
x = Activation("relu", name="input_relu")(x)
for i in range(mc.res_layer_num):
x = self._build_residual_block(x, i + 1)
res_out = x
# for policy output
x = Conv2D(filters=2,
kernel_size=1,
data_format="channels_first",
use_bias=False,
kernel_regularizer=l2(mc.l2_reg),
name="policy_conv-1-2")(res_out)
x = BatchNormalization(axis=1, name="policy_batchnorm")(x)
x = Activation("relu", name="policy_relu")(x)
x = Flatten(name="policy_flatten")(x)
policy_out = Dense(self.config.n_labels,
kernel_regularizer=l2(mc.l2_reg),
activation="softmax",
name="policy_out")(x)
# for value output
x = Conv2D(filters=4,
kernel_size=1,
data_format="channels_first",
use_bias=False,
kernel_regularizer=l2(mc.l2_reg),
name="value_conv-1-4")(res_out)
x = BatchNormalization(axis=1, name="value_batchnorm")(x)
x = Activation("relu", name="value_relu")(x)
x = Flatten(name="value_flatten")(x)
x = Dense(mc.value_fc_size,
kernel_regularizer=l2(mc.l2_reg),
activation="relu",
name="value_dense")(x)
value_out = Dense(1,
kernel_regularizer=l2(mc.l2_reg),
activation="tanh",
name="value_out")(x)
self.model = Model(in_x, [policy_out, value_out], name="chess_model")
def _build_residual_block(self, x, index):
# mc = self.config.model
mc = self.config
in_x = x
res_name = "res" + str(index)
x = Conv2D(filters=mc.cnn_filter_num,
kernel_size=mc.cnn_filter_size,
padding="same",
data_format="channels_first",
use_bias=False,
kernel_regularizer=l2(mc.l2_reg),
name=res_name + "_conv1-" + str(mc.cnn_filter_size) + "-" +
str(mc.cnn_filter_num))(x)
x = BatchNormalization(axis=1, name=res_name + "_batchnorm1")(x)
x = Activation("relu", name=res_name + "_relu1")(x)
x = Conv2D(filters=mc.cnn_filter_num,
kernel_size=mc.cnn_filter_size,
padding="same",
data_format="channels_first",
use_bias=False,
kernel_regularizer=l2(mc.l2_reg),
name=res_name + "_conv2-" + str(mc.cnn_filter_size) + "-" +
str(mc.cnn_filter_num))(x)
x = BatchNormalization(axis=1,
name="res" + str(index) + "_batchnorm2")(x)
x = Add(name=res_name + "_add")([in_x, x])
x = Activation("relu", name=res_name + "_relu2")(x)
return x
def compile(self, optimizer, loss, metrics, loss_weights=None):
self.model.compile(optimizer=optimizer,
loss=loss,
metrics=metrics,
loss_weights=loss_weights)
return self.model
def fit(self,
dataset,
y=None,
validation_data=None,
batch_size=None,
epochs=10,
shuffle=True,
val_split=None,
callbacks=None):
self.model.fit(x=dataset,
y=y,
batch_size=batch_size,
epochs=epochs,
shuffle=True,
validation_split=val_split,
validation_data=validation_data,
callbacks=callbacks)
return self.model
def predict(self,
x,
batch_size=None,
steps=None,
callbacks=None,
max_queue_size=10,
workers=1,
use_multiprocessing=False):
value = self.model.predict(x=x,
batch_size=batch_size,
steps=steps,
callbacks=callbacks,
max_queue_size=max_queue_size,
workers=workers,
use_multiprocessing=use_multiprocessing)
return value
def summary(self):
self.model.summary(line_length=None, positions=None, print_fn=None)
def run(self, ctx, exa, train: bool):
session_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
session = tf.Session(config=session_config)
tf.keras.backend.set_session(session)
config = self.read_config(exa)
batch_size = config["batch_size"]
epochs = config["epochs"]
steps_per_epoch = ctx.size() // batch_size
use_cache = config["use_cache"]
load_path = None
if "model_load_bucketfs_path" in config:
load_path = config["model_load_bucketfs_path"]
save_url = None
if "model_save_bucketfs_url" in config:
save_url = config["model_save_bucketfs_url"]
save_path = config["model_temporary_save_path"]
dataset = DatasetUtils().create_generator_dataset(
ctx, epochs, batch_size, use_cache, exa.meta.input_columns)
with tf.device(config["device"]):
input_columns, keras_inputs, preprocessed_keras_inputs = \
ColumnEncoder().generate_inputs(
exa.meta.input_columns, config["columns"])
table_network = self.create_table_network(
preprocessed_keras_inputs)
output_columns, keras_outputs, losses, loss_weights, output_metrics = \
ColumnEncoder().generate_outputs(
exa.meta.input_columns, table_network, config["columns"])
session.run(tf.tables_initializer())
dataset = DatasetUtils().create_dataset(dataset, input_columns,
output_columns, batch_size,
use_cache)
session.run(tf.global_variables_initializer())
session.run(tf.local_variables_initializer())
dataset_iterator = dataset.make_initializable_iterator()
session.run(dataset_iterator.initializer)
saver = tf.train.Saver(max_to_keep=1, save_relative_paths=True)
print("load_path", load_path, flush=True)
if load_path is not None and load_path != "":
initial_epoch = Utils().restore_model_and_get_inital_epoch(
session, saver, load_path + "/checkpoints/tmp/save")
else:
initial_epoch = 0
callbacks = Utils().create_callbacks(session, saver, save_path)
model = Model(inputs=keras_inputs, outputs=keras_outputs)
profile = config["profile"]
profile_model_options = Utils().add_profiler(
callbacks, profile, session, save_path)
print(output_metrics, flush=True)
model.compile(optimizer='rmsprop',
loss=losses,
loss_weights=loss_weights,
metrics=output_metrics,
**profile_model_options)
print(model.summary(), flush=True)
if train:
print("Starting training", flush=True)
history = model.fit(dataset_iterator,
steps_per_epoch=steps_per_epoch,
epochs=initial_epoch + epochs,
verbose=2,
callbacks=callbacks,
initial_epoch=initial_epoch)
ctx.emit(str(history.history))
print("save_url", save_url, flush=True)
if save_url != "" and save_url is not None:
tarfile = f"/tmp/save"
os.makedirs(tarfile, exist_ok=True)
self.tar_save(save_path, tarfile)
self.upload_save(save_url, tarfile)
else:
print("Starting prediction", flush=True)
for i in range(steps_per_epoch):
print(f"Predicting Batch {i}/steps_per_epoch", flush=True)
output = model.predict(dataset_iterator, steps=1)
ctx.emit(output)
class ChessModel:
"""
The model which can be trained to take observations of a game of chess and return value and policy
predictions.
Attributes:
:ivar Config config: configuration to use
:ivar Model model: the Keras model to use for predictions
:ivar digest: basically just a hash of the file containing the weights being used by this model
:ivar ChessModelAPI api: the api to use to listen for and then return this models predictions (on a pipe).
"""
def __init__(self, config):
self.config = config
self.model = None # type: Model
self.digest = None
self.api = None
def build(self):
"""
Builds the full Keras model and stores it in self.model.
"""
mc = self.config
in_x = x = Input((12, 8, 8))
# (batch, channels, height, width)
x = Conv2D(filters=mc.cnn_filter_num,
kernel_size=mc.cnn_first_filter_size,
padding="same",
data_format="channels_first",
use_bias=False,
kernel_regularizer=l2(mc.l2_reg),
name="input_conv-" + str(mc.cnn_first_filter_size) + "-" +
str(mc.cnn_filter_num))(x)
x = BatchNormalization(axis=1, name="input_batchnorm")(x)
x = Activation("relu", name="input_relu")(x)
for i in range(mc.res_layer_num):
x = self._build_residual_block(x, i + 1)
res_out = x
# for value output
x = Conv2D(filters=4,
kernel_size=1,
data_format="channels_first",
use_bias=False,
kernel_regularizer=l2(mc.l2_reg),
name="value_conv-1-4")(res_out)
x = BatchNormalization(axis=1, name="value_batchnorm")(x)
x = Activation("relu", name="value_relu")(x)
x = Flatten(name="value_flatten")(x)
x = Dense(mc.value_fc_size,
kernel_regularizer=l2(mc.l2_reg),
activation="relu",
name="value_dense")(x)
value_out = Dense(1,
kernel_regularizer=l2(mc.l2_reg),
activation="tanh",
name="value_out")(x)
self.model = Model(in_x, [value_out], name="chess_model")
def _build_residual_block(self, x, index):
# mc = self.config.model
mc = self.config
in_x = x
res_name = "res" + str(index)
x = Conv2D(filters=mc.cnn_filter_num,
kernel_size=mc.cnn_filter_size,
padding="same",
data_format="channels_first",
use_bias=False,
kernel_regularizer=l2(mc.l2_reg),
name=res_name + "_conv1-" + str(mc.cnn_filter_size) + "-" +
str(mc.cnn_filter_num))(x)
x = BatchNormalization(axis=1, name=res_name + "_batchnorm1")(x)
x = Activation("relu", name=res_name + "_relu1")(x)
x = Conv2D(filters=mc.cnn_filter_num,
kernel_size=mc.cnn_filter_size,
padding="same",
data_format="channels_first",
use_bias=False,
kernel_regularizer=l2(mc.l2_reg),
name=res_name + "_conv2-" + str(mc.cnn_filter_size) + "-" +
str(mc.cnn_filter_num))(x)
x = BatchNormalization(axis=1,
name="res" + str(index) + "_batchnorm2")(x)
x = Add(name=res_name + "_add")([in_x, x])
x = Activation("relu", name=res_name + "_relu2")(x)
return x
def compile(self, optimizer, loss, metrics):
self.model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
return self.model
def fit(self, dataset, batch_size, epochs, shuffle, validation_split,
validation_data, callbacks):
self.model.fit(x=dataset,
batch_size=batch_size,
epochs=epochs,
shuffle=True,
validation_split=validation_split,
validation_data=validation_data,
callbacks=callbacks)
return self.model
def predict(self,
x,
batch_size=None,
steps=None,
callbacks=None,
max_queue_size=10,
workers=1,
use_multiprocessing=False):
value = self.model.predict(x=x,
batch_size=batch_size,
steps=steps,
callbacks=callbacks,
max_queue_size=max_queue_size,
workers=workers,
use_multiprocessing=use_multiprocessing)
return value