def evaluate_error(model: training.Model) -> np.float64: pred = model.predict(x_test, batch_size=32) pred = np.argmax(pred, axis=1) pred = np.expand_dims(pred, axis=1) # make same shape as y_test error = np.sum(np.not_equal(pred, y_test)) / y_test.shape[0] return error
def model_predict(model, embedded_ip, raw_text, label_layer_name): total_items = embedded_ip.shape[0] label_model = Model(inputs=model.input, outputs=model.get_layer(label_layer_name).output) y_prob = label_model.predict(embedded_ip).reshape(-1) y_pred = y_prob > 0.5 return y_prob, y_pred
def get_embedded_input(model, encoded_text): """ Get embedding layer output from a CNN model as the input for CNN_DCNN model """ embedding_layer_model = Model( inputs=model.input, outputs=model.get_layer('word_embedding').output) return embedding_layer_model.predict(encoded_text)
def test_stateful_metrics(self): with self.cached_session(): np.random.seed(1334) class BinaryTruePositives(layers.Layer): """Stateful Metric to count the total true positives over all batches. Assumes predictions and targets of shape `(samples, 1)`. Arguments: threshold: Float, lower limit on prediction value that counts as a positive class prediction. name: String, name for the metric. """ def __init__(self, name='true_positives', **kwargs): super(BinaryTruePositives, self).__init__(name=name, **kwargs) self.true_positives = K.variable(value=0, dtype='int32') self.stateful = True def reset_states(self): K.set_value(self.true_positives, 0) def __call__(self, y_true, y_pred): """Computes the number of true positives in a batch. Args: y_true: Tensor, batch_wise labels y_pred: Tensor, batch_wise predictions Returns: The total number of true positives seen this epoch at the completion of the batch. """ y_true = math_ops.cast(y_true, 'int32') y_pred = math_ops.cast(math_ops.round(y_pred), 'int32') correct_preds = math_ops.cast( math_ops.equal(y_pred, y_true), 'int32') true_pos = math_ops.cast( math_ops.reduce_sum(correct_preds * y_true), 'int32') current_true_pos = self.true_positives * 1 self.add_update(state_ops.assign_add( self.true_positives, true_pos), inputs=[y_true, y_pred]) return current_true_pos + true_pos metric_fn = BinaryTruePositives() config = metrics.serialize(metric_fn) metric_fn = metrics.deserialize( config, custom_objects={'BinaryTruePositives': BinaryTruePositives}) # Test on simple model inputs = layers.Input(shape=(2, )) outputs = layers.Dense(1, activation='sigmoid')(inputs) model = Model(inputs, outputs) model.compile(optimizer='sgd', loss='binary_crossentropy', metrics=['acc', metric_fn]) # Test fit, evaluate samples = 100 x = np.random.random((samples, 2)) y = np.random.randint(2, size=(samples, 1)) val_samples = 10 val_x = np.random.random((val_samples, 2)) val_y = np.random.randint(2, size=(val_samples, 1)) history = model.fit(x, y, epochs=1, batch_size=10, validation_data=(val_x, val_y)) outs = model.evaluate(x, y, batch_size=10) preds = model.predict(x) def ref_true_pos(y_true, y_pred): return np.sum(np.logical_and(y_pred > 0.5, y_true == 1)) # Test correctness (e.g. updates should have been run) self.assertAllClose(outs[2], ref_true_pos(y, preds), atol=1e-5) # Test correctness of the validation metric computation val_preds = model.predict(val_x) val_outs = model.evaluate(val_x, val_y, batch_size=10) self.assertAllClose(val_outs[2], ref_true_pos(val_y, val_preds), atol=1e-5) self.assertAllClose(val_outs[2], history.history['val_true_positives'][-1], atol=1e-5) # Test with generators gen = [(np.array([x0]), np.array([y0])) for x0, y0 in zip(x, y)] val_gen = [(np.array([x0]), np.array([y0])) for x0, y0 in zip(val_x, val_y)] history = model.fit_generator(iter(gen), epochs=1, steps_per_epoch=samples, validation_data=iter(val_gen), validation_steps=val_samples) outs = model.evaluate_generator(iter(gen), steps=samples) preds = model.predict_generator(iter(gen), steps=samples) # Test correctness of the metric results self.assertAllClose(outs[2], ref_true_pos(y, preds), atol=1e-5) # Test correctness of the validation metric computation val_preds = model.predict_generator(iter(val_gen), steps=val_samples) val_outs = model.evaluate_generator(iter(val_gen), steps=val_samples) self.assertAllClose(val_outs[2], ref_true_pos(val_y, val_preds), atol=1e-5) self.assertAllClose(val_outs[2], history.history['val_true_positives'][-1], atol=1e-5)
def test_stateful_metrics(self): with self.test_session(): np.random.seed(1334) class BinaryTruePositives(layers.Layer): """Stateful Metric to count the total true positives over all batches. Assumes predictions and targets of shape `(samples, 1)`. Arguments: threshold: Float, lower limit on prediction value that counts as a positive class prediction. name: String, name for the metric. """ def __init__(self, name='true_positives', **kwargs): super(BinaryTruePositives, self).__init__(name=name, **kwargs) self.true_positives = K.variable(value=0, dtype='int32') self.stateful = True def reset_states(self): K.set_value(self.true_positives, 0) def __call__(self, y_true, y_pred): """Computes the number of true positives in a batch. Args: y_true: Tensor, batch_wise labels y_pred: Tensor, batch_wise predictions Returns: The total number of true positives seen this epoch at the completion of the batch. """ y_true = math_ops.cast(y_true, 'int32') y_pred = math_ops.cast(math_ops.round(y_pred), 'int32') correct_preds = math_ops.cast(math_ops.equal(y_pred, y_true), 'int32') true_pos = math_ops.cast( math_ops.reduce_sum(correct_preds * y_true), 'int32') current_true_pos = self.true_positives * 1 self.add_update( state_ops.assign_add(self.true_positives, true_pos), inputs=[y_true, y_pred]) return current_true_pos + true_pos metric_fn = BinaryTruePositives() config = metrics.serialize(metric_fn) metric_fn = metrics.deserialize( config, custom_objects={'BinaryTruePositives': BinaryTruePositives}) # Test on simple model inputs = layers.Input(shape=(2,)) outputs = layers.Dense(1, activation='sigmoid')(inputs) model = Model(inputs, outputs) model.compile(optimizer='sgd', loss='binary_crossentropy', metrics=['acc', metric_fn]) # Test fit, evaluate samples = 100 x = np.random.random((samples, 2)) y = np.random.randint(2, size=(samples, 1)) val_samples = 10 val_x = np.random.random((val_samples, 2)) val_y = np.random.randint(2, size=(val_samples, 1)) history = model.fit(x, y, epochs=1, batch_size=10, validation_data=(val_x, val_y)) outs = model.evaluate(x, y, batch_size=10) preds = model.predict(x) def ref_true_pos(y_true, y_pred): return np.sum(np.logical_and(y_pred > 0.5, y_true == 1)) # Test correctness (e.g. updates should have been run) self.assertAllClose(outs[2], ref_true_pos(y, preds), atol=1e-5) # Test correctness of the validation metric computation val_preds = model.predict(val_x) val_outs = model.evaluate(val_x, val_y, batch_size=10) self.assertAllClose( val_outs[2], ref_true_pos(val_y, val_preds), atol=1e-5) self.assertAllClose( val_outs[2], history.history['val_true_positives'][-1], atol=1e-5) # Test with generators gen = [(np.array([x0]), np.array([y0])) for x0, y0 in zip(x, y)] val_gen = [(np.array([x0]), np.array([y0])) for x0, y0 in zip(val_x, val_y)] history = model.fit_generator(iter(gen), epochs=1, steps_per_epoch=samples, validation_data=iter(val_gen), validation_steps=val_samples) outs = model.evaluate_generator(iter(gen), steps=samples) preds = model.predict_generator(iter(gen), steps=samples) # Test correctness of the metric results self.assertAllClose(outs[2], ref_true_pos(y, preds), atol=1e-5) # Test correctness of the validation metric computation val_preds = model.predict_generator(iter(val_gen), steps=val_samples) val_outs = model.evaluate_generator(iter(val_gen), steps=val_samples) self.assertAllClose( val_outs[2], ref_true_pos(val_y, val_preds), atol=1e-5) self.assertAllClose( val_outs[2], history.history['val_true_positives'][-1], atol=1e-5)
class ChessModel: """ The model which can be trained to take observations of a game of chess and return value and policy predictions. Inpired by https://github.com/Zeta36/chess-alpha-zero/blob/master/src/chess_zero/agent/model_chess.py Attributes: :ivar Config config: configuration to use :ivar Model model: the Keras model to use for predictions """ def __init__(self, config): self.config = config self.model = None # type: Model self.digest = None self.api = None def build(self): """ Builds the full Keras model and stores it in self.model. """ mc = self.config in_x = x = Input((12, 8, 8)) # (batch, channels, height, width) x = Conv2D(filters=mc.cnn_filter_num, kernel_size=mc.cnn_first_filter_size, padding="same", data_format="channels_first", use_bias=False, kernel_regularizer=l2(mc.l2_reg), name="input_conv-" + str(mc.cnn_first_filter_size) + "-" + str(mc.cnn_filter_num))(x) x = BatchNormalization(axis=1, name="input_batchnorm")(x) x = Activation("relu", name="input_relu")(x) for i in range(mc.res_layer_num): x = self._build_residual_block(x, i + 1) res_out = x # for policy output x = Conv2D(filters=2, kernel_size=1, data_format="channels_first", use_bias=False, kernel_regularizer=l2(mc.l2_reg), name="policy_conv-1-2")(res_out) x = BatchNormalization(axis=1, name="policy_batchnorm")(x) x = Activation("relu", name="policy_relu")(x) x = Flatten(name="policy_flatten")(x) policy_out = Dense(self.config.n_labels, kernel_regularizer=l2(mc.l2_reg), activation="softmax", name="policy_out")(x) # for value output x = Conv2D(filters=4, kernel_size=1, data_format="channels_first", use_bias=False, kernel_regularizer=l2(mc.l2_reg), name="value_conv-1-4")(res_out) x = BatchNormalization(axis=1, name="value_batchnorm")(x) x = Activation("relu", name="value_relu")(x) x = Flatten(name="value_flatten")(x) x = Dense(mc.value_fc_size, kernel_regularizer=l2(mc.l2_reg), activation="relu", name="value_dense")(x) value_out = Dense(1, kernel_regularizer=l2(mc.l2_reg), activation="tanh", name="value_out")(x) self.model = Model(in_x, [policy_out, value_out], name="chess_model") def _build_residual_block(self, x, index): # mc = self.config.model mc = self.config in_x = x res_name = "res" + str(index) x = Conv2D(filters=mc.cnn_filter_num, kernel_size=mc.cnn_filter_size, padding="same", data_format="channels_first", use_bias=False, kernel_regularizer=l2(mc.l2_reg), name=res_name + "_conv1-" + str(mc.cnn_filter_size) + "-" + str(mc.cnn_filter_num))(x) x = BatchNormalization(axis=1, name=res_name + "_batchnorm1")(x) x = Activation("relu", name=res_name + "_relu1")(x) x = Conv2D(filters=mc.cnn_filter_num, kernel_size=mc.cnn_filter_size, padding="same", data_format="channels_first", use_bias=False, kernel_regularizer=l2(mc.l2_reg), name=res_name + "_conv2-" + str(mc.cnn_filter_size) + "-" + str(mc.cnn_filter_num))(x) x = BatchNormalization(axis=1, name="res" + str(index) + "_batchnorm2")(x) x = Add(name=res_name + "_add")([in_x, x]) x = Activation("relu", name=res_name + "_relu2")(x) return x def compile(self, optimizer, loss, metrics, loss_weights=None): self.model.compile(optimizer=optimizer, loss=loss, metrics=metrics, loss_weights=loss_weights) return self.model def fit(self, dataset, y=None, validation_data=None, batch_size=None, epochs=10, shuffle=True, val_split=None, callbacks=None): self.model.fit(x=dataset, y=y, batch_size=batch_size, epochs=epochs, shuffle=True, validation_split=val_split, validation_data=validation_data, callbacks=callbacks) return self.model def predict(self, x, batch_size=None, steps=None, callbacks=None, max_queue_size=10, workers=1, use_multiprocessing=False): value = self.model.predict(x=x, batch_size=batch_size, steps=steps, callbacks=callbacks, max_queue_size=max_queue_size, workers=workers, use_multiprocessing=use_multiprocessing) return value def summary(self): self.model.summary(line_length=None, positions=None, print_fn=None)
def run(self, ctx, exa, train: bool): session_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) session = tf.Session(config=session_config) tf.keras.backend.set_session(session) config = self.read_config(exa) batch_size = config["batch_size"] epochs = config["epochs"] steps_per_epoch = ctx.size() // batch_size use_cache = config["use_cache"] load_path = None if "model_load_bucketfs_path" in config: load_path = config["model_load_bucketfs_path"] save_url = None if "model_save_bucketfs_url" in config: save_url = config["model_save_bucketfs_url"] save_path = config["model_temporary_save_path"] dataset = DatasetUtils().create_generator_dataset( ctx, epochs, batch_size, use_cache, exa.meta.input_columns) with tf.device(config["device"]): input_columns, keras_inputs, preprocessed_keras_inputs = \ ColumnEncoder().generate_inputs( exa.meta.input_columns, config["columns"]) table_network = self.create_table_network( preprocessed_keras_inputs) output_columns, keras_outputs, losses, loss_weights, output_metrics = \ ColumnEncoder().generate_outputs( exa.meta.input_columns, table_network, config["columns"]) session.run(tf.tables_initializer()) dataset = DatasetUtils().create_dataset(dataset, input_columns, output_columns, batch_size, use_cache) session.run(tf.global_variables_initializer()) session.run(tf.local_variables_initializer()) dataset_iterator = dataset.make_initializable_iterator() session.run(dataset_iterator.initializer) saver = tf.train.Saver(max_to_keep=1, save_relative_paths=True) print("load_path", load_path, flush=True) if load_path is not None and load_path != "": initial_epoch = Utils().restore_model_and_get_inital_epoch( session, saver, load_path + "/checkpoints/tmp/save") else: initial_epoch = 0 callbacks = Utils().create_callbacks(session, saver, save_path) model = Model(inputs=keras_inputs, outputs=keras_outputs) profile = config["profile"] profile_model_options = Utils().add_profiler( callbacks, profile, session, save_path) print(output_metrics, flush=True) model.compile(optimizer='rmsprop', loss=losses, loss_weights=loss_weights, metrics=output_metrics, **profile_model_options) print(model.summary(), flush=True) if train: print("Starting training", flush=True) history = model.fit(dataset_iterator, steps_per_epoch=steps_per_epoch, epochs=initial_epoch + epochs, verbose=2, callbacks=callbacks, initial_epoch=initial_epoch) ctx.emit(str(history.history)) print("save_url", save_url, flush=True) if save_url != "" and save_url is not None: tarfile = f"/tmp/save" os.makedirs(tarfile, exist_ok=True) self.tar_save(save_path, tarfile) self.upload_save(save_url, tarfile) else: print("Starting prediction", flush=True) for i in range(steps_per_epoch): print(f"Predicting Batch {i}/steps_per_epoch", flush=True) output = model.predict(dataset_iterator, steps=1) ctx.emit(output)
class ChessModel: """ The model which can be trained to take observations of a game of chess and return value and policy predictions. Attributes: :ivar Config config: configuration to use :ivar Model model: the Keras model to use for predictions :ivar digest: basically just a hash of the file containing the weights being used by this model :ivar ChessModelAPI api: the api to use to listen for and then return this models predictions (on a pipe). """ def __init__(self, config): self.config = config self.model = None # type: Model self.digest = None self.api = None def build(self): """ Builds the full Keras model and stores it in self.model. """ mc = self.config in_x = x = Input((12, 8, 8)) # (batch, channels, height, width) x = Conv2D(filters=mc.cnn_filter_num, kernel_size=mc.cnn_first_filter_size, padding="same", data_format="channels_first", use_bias=False, kernel_regularizer=l2(mc.l2_reg), name="input_conv-" + str(mc.cnn_first_filter_size) + "-" + str(mc.cnn_filter_num))(x) x = BatchNormalization(axis=1, name="input_batchnorm")(x) x = Activation("relu", name="input_relu")(x) for i in range(mc.res_layer_num): x = self._build_residual_block(x, i + 1) res_out = x # for value output x = Conv2D(filters=4, kernel_size=1, data_format="channels_first", use_bias=False, kernel_regularizer=l2(mc.l2_reg), name="value_conv-1-4")(res_out) x = BatchNormalization(axis=1, name="value_batchnorm")(x) x = Activation("relu", name="value_relu")(x) x = Flatten(name="value_flatten")(x) x = Dense(mc.value_fc_size, kernel_regularizer=l2(mc.l2_reg), activation="relu", name="value_dense")(x) value_out = Dense(1, kernel_regularizer=l2(mc.l2_reg), activation="tanh", name="value_out")(x) self.model = Model(in_x, [value_out], name="chess_model") def _build_residual_block(self, x, index): # mc = self.config.model mc = self.config in_x = x res_name = "res" + str(index) x = Conv2D(filters=mc.cnn_filter_num, kernel_size=mc.cnn_filter_size, padding="same", data_format="channels_first", use_bias=False, kernel_regularizer=l2(mc.l2_reg), name=res_name + "_conv1-" + str(mc.cnn_filter_size) + "-" + str(mc.cnn_filter_num))(x) x = BatchNormalization(axis=1, name=res_name + "_batchnorm1")(x) x = Activation("relu", name=res_name + "_relu1")(x) x = Conv2D(filters=mc.cnn_filter_num, kernel_size=mc.cnn_filter_size, padding="same", data_format="channels_first", use_bias=False, kernel_regularizer=l2(mc.l2_reg), name=res_name + "_conv2-" + str(mc.cnn_filter_size) + "-" + str(mc.cnn_filter_num))(x) x = BatchNormalization(axis=1, name="res" + str(index) + "_batchnorm2")(x) x = Add(name=res_name + "_add")([in_x, x]) x = Activation("relu", name=res_name + "_relu2")(x) return x def compile(self, optimizer, loss, metrics): self.model.compile(optimizer=optimizer, loss=loss, metrics=metrics) return self.model def fit(self, dataset, batch_size, epochs, shuffle, validation_split, validation_data, callbacks): self.model.fit(x=dataset, batch_size=batch_size, epochs=epochs, shuffle=True, validation_split=validation_split, validation_data=validation_data, callbacks=callbacks) return self.model def predict(self, x, batch_size=None, steps=None, callbacks=None, max_queue_size=10, workers=1, use_multiprocessing=False): value = self.model.predict(x=x, batch_size=batch_size, steps=steps, callbacks=callbacks, max_queue_size=max_queue_size, workers=workers, use_multiprocessing=use_multiprocessing) return value