def train(mnist):
x = tf.placeholder(tf.float32, [None, INPUT_NODE],name = 'x-input')
y_= tf.placeholder(tf.float32, [None, OUTPUT_NODE], name= 'y-input')
weights1 = tf.Variable(
tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1)
)
biases1 = tf.Variable(tf.constant(0,1, shape=[LAYER1_NODE]))
weights2 = tf.Variable(
tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1)
)
biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))
y = inference(x, None, weights1, biases1, weights2, biases2)
global_step = tf.Variable(0, trainable=False)
variable_averages = tf.train.ExponentialMovingAverage(
MOVING_AVERAGE_DECY, global_step
)
variable_averages_op = variable_averages.apply(
tf.trainable_variables()
)
average_y = inference(
x, variable_averages, weights1, biases1, weights2, biases2
)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits = y, labels = tf.argmax(y_, 1))
cross_entropy_mean = tf.reduce_mean(cross_entropy)
# regularizere = tf.contrib.l
regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)
regularization = regularizer(weights1) + regularizer(weights2)
loss = cross_entropy_mean + regularization
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples / BATCH_SIZE,
LEARNING_RATE_DECAY
)
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
with tf.control_dependencies([train_step, variable_averages_op]):
train_op = tf.no_op(name='train')
correct_prection = tf.equal(tf.argmax(average_y, 1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prection, tf.float32))
with tf.Session() as sess:
tf.tables_initializer().run()
validate_feed = {x: mnist.validation.images, y_:mnist.validation.labels}
test_feed = {x:mnist.test.images, y_:mnist.test.labels}
for i in range(TRAINING_STEPS):
if i % 100 ==0:
validate_acc = sess.run(accuracy, feed_dict=validate_feed)
print("After %d training step(s), validation accuracy using avarage model is %g " %(i , validate_acc))
xs, ys = mnist.train.next_batch(BATCH_SIZE)
sess.run(train_op, feed_dict={x:xs, y_:ys})
test_acc =sess.run(accuracy, feed_dict=test_feed)
print('After %d training step(s), test accuracy usering average model is %g' %(TRAINING_STEPS, test_acc))
def test_text_corresponds_to_ids(self):
charset = create_fake_charset(36)
ids = tf.constant(
[[17, 14, 21, 21, 24], [32, 24, 27, 21, 13]], dtype=tf.int64)
charset_mapper = model.CharsetMapper(charset)
with self.test_session() as sess:
tf.tables_initializer().run()
text = sess.run(charset_mapper.get_text(ids))
self.assertAllEqual(text, ['hello', 'world'])
def test_predicted_text_has_correct_shape_w_charset(self):
charset = create_fake_charset(self.num_char_classes)
ocr_model = self.create_model(charset=charset)
with self.test_session() as sess:
endpoints_tf = ocr_model.create_base(
images=self.fake_images, labels_one_hot=None)
sess.run(tf.global_variables_initializer())
tf.tables_initializer().run()
endpoints = sess.run(endpoints_tf)
self.assertEqual(endpoints.predicted_text.shape, (self.batch_size,))
self.assertEqual(len(endpoints.predicted_text[0]), self.seq_length)
def main(_):
images_placeholder, endpoints, init_fn = load_model(FLAGS.checkpoint,
FLAGS.batch_size,
FLAGS.dataset_name)
images_data = load_images(FLAGS.image_path_pattern, FLAGS.batch_size,
FLAGS.dataset_name)
with tf.Session() as sess:
tf.tables_initializer().run() # required by the CharsetMapper
init_fn(sess)
predictions = sess.run(endpoints.predicted_text,
feed_dict={images_placeholder: images_data})
print("Predicted strings:")
for line in predictions:
print(line)
def main(args):
if not os.path.exists(FLAGS.checkpoint):
tf.logging.fatal(
'Checkpoint %s does not exist. Have you download it? See tools/download_data.sh',
FLAGS.checkpoint)
g = tf.Graph()
with g.as_default():
input_image = PreprocessImage(FLAGS.image_path[0])
with slim.arg_scope(inception.inception_v3_arg_scope()):
logits, end_points = inception.inception_v3(
input_image, num_classes=FLAGS.num_classes, is_training=False)
bottleneck = end_points['PreLogits']
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer(),
tf.tables_initializer())
saver = tf_saver.Saver()
sess = tf.Session()
saver.restore(sess, FLAGS.checkpoint)
# Run the evaluation on the image
bottleneck_eval = np.squeeze(sess.run(bottleneck))
first = True
for val in bottleneck_eval:
if not first:
sys.stdout.write(",")
first = False
sys.stdout.write('{:.3f}'.format(val))
sys.stdout.write('\n')
def testWordEmbedder(self):
with open(vocab_file, "w") as vocab:
vocab.write("the\n"
"world\n"
"hello\n"
"toto\n")
with open(data_file, "w") as data:
data.write("hello world !\n")
embedder = text_inputter.WordEmbedder(
"vocabulary_file", embedding_size=10)
data, transformed = _first_element(
embedder, data_file, {"vocabulary_file": vocab_file})
input_receiver = embedder.get_serving_input_receiver()
self.assertAllEqual(
[None, None],
input_receiver.features["ids"].get_shape().as_list())
self.assertAllEqual(
[None],
input_receiver.features["length"].get_shape().as_list())
with self.test_session() as sess:
sess.run(tf.tables_initializer())
sess.run(tf.global_variables_initializer())
data, transformed = sess.run([data, transformed])
self.assertNotIn("raw", data)
self.assertNotIn("tokens", data)
self.assertAllEqual([3], data["length"])
self.assertAllEqual([[2, 1, 4]], data["ids"])
self.assertAllEqual([1, 3, 10], transformed.shape)
def testCharConvEmbedder(self):
with open(vocab_file, "w") as vocab:
vocab.write("h\n"
"e\n"
"l\n"
"w\n"
"o\n")
with open(data_file, "w") as data:
data.write("hello world !\n")
embedder = text_inputter.CharConvEmbedder("vocabulary_file", 10, 5)
data, transformed = _first_element(
embedder, data_file, {"vocabulary_file": vocab_file})
input_receiver = embedder.get_serving_input_receiver()
self.assertAllEqual(
[None, None, None],
input_receiver.features["char_ids"].get_shape().as_list())
self.assertAllEqual(
[None],
input_receiver.features["length"].get_shape().as_list())
with self.test_session() as sess:
sess.run(tf.tables_initializer())
sess.run(tf.global_variables_initializer())
data, transformed = sess.run([data, transformed])
self.assertNotIn("raw", data)
self.assertNotIn("tokens", data)
self.assertAllEqual([3], data["length"])
self.assertAllEqual(
[[[0, 1, 2, 2, 4], [3, 4, 5, 2, 5], [5, 5, 5, 5, 5]]],
data["char_ids"])
self.assertAllEqual([1, 3, 5], transformed.shape)
def export(self, last_checkpoint, output_dir):
"""Builds a prediction graph and xports the model.
Args:
last_checkpoint: Path to the latest checkpoint file from training.
output_dir: Path to the folder to be used to output the model.
"""
logging.info('Exporting prediction graph to %s', output_dir)
with tf.Session(graph=tf.Graph()) as sess:
# Build and save prediction meta graph and trained variable values.
inputs, outputs = self.build_prediction_graph()
signature_def_map = {
'serving_default': signature_def_utils.predict_signature_def(inputs, outputs)
}
init_op = tf.global_variables_initializer()
sess.run(init_op)
self.restore_from_checkpoint(sess, self.inception_checkpoint_file,
last_checkpoint)
init_op_serving = control_flow_ops.group(
variables.local_variables_initializer(),
tf.tables_initializer())
builder = saved_model_builder.SavedModelBuilder(output_dir)
builder.add_meta_graph_and_variables(
sess, [tag_constants.SERVING],
signature_def_map=signature_def_map,
legacy_init_op=init_op_serving)
builder.save(False)
def test_skipgram_randomize(self):
test_dataset = tf.contrib.data.Dataset.from_tensor_slices([
'passj',
'word',
'db'
])
config = pe.EmbeddingConfig(
alphabet='abcdefghijklmnopqrstuvwxyz',
password_batch=5,
batch_size=10,
embedding_window_size=3)
emb_trainer = pe.EmbeddingTrainer(config)
examples, labels = emb_trainer.skipgram(test_dataset, randomize=True)
with self.test_session() as session:
session.run([tf.tables_initializer()])
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
try:
session.run([examples, labels])
except tf.errors.OutOfRangeError:
pass
finally:
coord.request_stop()
coord.join(threads)
def testLabels2(self):
self._input_config["label_feature"] = "label_str"
self._input_config["label_map"] = {"PC": 1, "AFP": 0, "NTP": 0}
dataset = dataset_ops.build_dataset(
file_pattern=self._file_pattern,
input_config=self._input_config,
batch_size=4)
# We need an initializable iterator when using labels because of the
# stateful label id hash table.
iterator = dataset.make_initializable_iterator()
inputs = iterator.get_next()
init_op = tf.tables_initializer()
# Expect features and labels.
self.assertItemsEqual(["time_series_features", "aux_features", "labels"],
inputs.keys())
labels = inputs["labels"]
with self.test_session() as sess:
sess.run([init_op, iterator.initializer])
# Fetch 3 batches.
np.testing.assert_array_equal([1, 0, 0, 1], sess.run(labels))
np.testing.assert_array_equal([0, 0, 1, 0], sess.run(labels))
np.testing.assert_array_equal([0, 1], sess.run(labels))
# No more batches.
with self.assertRaises(tf.errors.OutOfRangeError):
sess.run(labels)
def testDuplicateAssetCopy(self):
export_path = os.path.join(self.get_temp_dir(), "assets-module")
def module_with_duplicate_asset():
vocabulary_file = self.create_vocab_file("tokens2.txt", ["1", "2", "3"])
indices1 = tf.placeholder(dtype=tf.int64, name="indices1")
indices2 = tf.placeholder(dtype=tf.int64, name="indices2")
hub.add_signature(
inputs={
"indices_1": indices1,
"indices_2": indices2,
},
outputs={
"x": do_table_lookup(indices1, vocabulary_file),
"y": do_table_lookup(indices2, vocabulary_file),
})
with tf.Graph().as_default():
spec = hub.create_module_spec(module_with_duplicate_asset)
module_a = hub.Module(spec)
module_a({"indices_1": tf.constant([1, 2], dtype=tf.int64),
"indices_2": tf.constant([1, 2], dtype=tf.int64)}, as_dict=True)
with tf.Session() as sess:
sess.run(tf.tables_initializer())
module_a.export(export_path, sess)
def export_model(model_info, class_count, saved_model_dir):
# The SavedModel should hold the eval graph.
sess, _, _, _, _ = build_eval_session(model_info, class_count)
graph = sess.graph
with graph.as_default():
input_tensor = model_info['resized_input_tensor_name']
in_image = sess.graph.get_tensor_by_name(input_tensor)
inputs = {'image': tf.saved_model.utils.build_tensor_info(in_image)}
out_classes = sess.graph.get_tensor_by_name('final_result:0')
outputs = {
'prediction': tf.saved_model.utils.build_tensor_info(out_classes)
}
signature = tf.saved_model.signature_def_utils.build_signature_def(
inputs=inputs,
outputs=outputs,
method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME)
legacy_init_op = tf.group(tf.tables_initializer(), name='legacy_init_op')
# Save out the SavedModel.
builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir)
print("signature=", signature)
print("key:", tf.saved_model.signature_constants.
DEFAULT_SERVING_SIGNATURE_DEF_KEY)
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
tf.saved_model.signature_constants.
DEFAULT_SERVING_SIGNATURE_DEF_KEY:
signature
},
legacy_init_op=legacy_init_op)
builder.save()
def execute_cpu(self, graph_fn, inputs):
"""Constructs the graph, executes it on CPU and returns the result.
Args:
graph_fn: a callable that constructs the tensorflow graph to test. The
arguments of this function should correspond to `inputs`.
inputs: a list of numpy arrays to feed input to the computation graph.
Returns:
A list of numpy arrays or a scalar returned from executing the tensorflow
graph.
"""
with self.test_session(graph=tf.Graph()) as sess:
placeholders = [tf.placeholder_with_default(v, v.shape) for v in inputs]
results = graph_fn(*placeholders)
sess.run([tf.global_variables_initializer(), tf.tables_initializer(),
tf.local_variables_initializer()])
materialized_results = sess.run(results, feed_dict=dict(zip(placeholders,
inputs)))
if (hasattr(materialized_results, '__len__') and
len(materialized_results) == 1 and
(isinstance(materialized_results, list) or
isinstance(materialized_results, tuple))):
materialized_results = materialized_results[0]
return materialized_results
def test_with_counts(self):
vocab_list = ["Hello", ".", "笑"]
vocab_counts = [100, 200, 300]
vocab_file = test_utils.create_temporary_vocab_file(vocab_list,
vocab_counts)
vocab_to_id_table, id_to_vocab_table, word_to_count_table, vocab_size = \
vocab.create_vocabulary_lookup_table(vocab_file.name)
self.assertEqual(vocab_size, 6)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
sess.run(tf.tables_initializer())
ids = vocab_to_id_table.lookup(
tf.convert_to_tensor(["Hello", ".", "笑", "??", "xxx"]))
ids = sess.run(ids)
np.testing.assert_array_equal(ids, [0, 1, 2, 3, 3])
words = id_to_vocab_table.lookup(
tf.convert_to_tensor(
[0, 1, 2, 3], dtype=tf.int64))
words = sess.run(words)
np.testing.assert_array_equal(
np.char.decode(words.astype("S"), "utf-8"),
["Hello", ".", "笑", "UNK"])
counts = word_to_count_table.lookup(
tf.convert_to_tensor(["Hello", ".", "笑", "??", "xxx"]))
counts = sess.run(counts)
np.testing.assert_array_equal(counts, [100, 200, 300, -1, -1])
def _run_eval(self):
"""Run model evaluation and generate summaries."""
coord = tf.train.Coordinator(clean_stop_exception_types=(
tf.errors.CancelledError, tf.errors.OutOfRangeError))
with tf.Session(graph=self._graph) as session:
# Restores previously saved variables from latest checkpoint
self._saver.restore(session, self._latest_checkpoint)
session.run([
tf.tables_initializer(),
tf.local_variables_initializer()])
tf.train.start_queue_runners(coord=coord, sess=session)
train_step = session.run(self._gs)
tf.logging.info('Starting Evaluation For Step: {}'.format(train_step))
with coord.stop_on_exception():
eval_step = 0
while not coord.should_stop() and (self._eval_steps is None or
eval_step < self._eval_steps):
summaries, final_values, _ = session.run(
[self._summary_op, self._final_ops_dict, self._eval_ops])
if eval_step % 100 == 0:
tf.logging.info('On Evaluation Step: {}'.format(eval_step))
eval_step += 1
# Write the summaries
self._file_writer.add_summary(summaries, global_step=train_step)
self._file_writer.flush()
tf.logging.info(final_values)
def testUnknownLabel(self):
self._input_config["label_feature"] = "label_str"
# label_map does not include "NTP".
self._input_config["label_map"] = {"PC": 1, "AFP": 0}
dataset = dataset_ops.build_dataset(
file_pattern=self._file_pattern,
input_config=self._input_config,
batch_size=4)
# We need an initializable iterator when using labels because of the
# stateful label id hash table.
iterator = dataset.make_initializable_iterator()
inputs = iterator.get_next()
init_op = tf.tables_initializer()
# Expect features and labels.
self.assertItemsEqual(["time_series_features", "aux_features", "labels"],
inputs.keys())
labels = inputs["labels"]
with self.test_session() as sess:
sess.run([init_op, iterator.initializer])
# Unknown label "NTP".
with self.assertRaises(tf.errors.InvalidArgumentError):
sess.run(labels)
def test_module_export_vocab_on_custom_fs(self):
root_dir = "file://%s" % self.get_temp_dir()
export_dir = "%s_%s" % (root_dir, "export")
tf.gfile.MakeDirs(export_dir)
# Create a module with a vocab file located on a custom filesystem.
vocab_dir = os.path.join(root_dir, "vocab_location")
tf.gfile.MakeDirs(vocab_dir)
vocab_filename = os.path.join(vocab_dir, "tokens.txt")
tf_utils.atomic_write_string_to_file(vocab_filename, "one", False)
def create_assets_module_fn():
def assets_module_fn():
indices = tf.placeholder(dtype=tf.int64, name="indices")
table = tf.contrib.lookup.index_to_string_table_from_file(
vocabulary_file=vocab_filename, default_value="UNKNOWN")
outputs = table.lookup(indices)
hub.add_signature(inputs=indices, outputs=outputs)
return assets_module_fn
with tf.Graph().as_default():
assets_module_fn = create_assets_module_fn()
spec = hub.create_module_spec(assets_module_fn)
embedding_module = hub.Module(spec)
with tf.Session() as sess:
sess.run(tf.tables_initializer())
embedding_module.export(export_dir, sess)
module_files = tf.gfile.ListDirectory(export_dir)
self.assertListEqual(
["assets", "saved_model.pb", "tfhub_module.pb", "variables"],
sorted(module_files))
module_files = tf.gfile.ListDirectory(os.path.join(export_dir, "assets"))
self.assertListEqual(["tokens.txt"], module_files)
def load_model(model, ckpt, session, name):
start_time = time.time()
model.saver.restore(session, ckpt)
session.run(tf.tables_initializer())
print " loaded %s model parameters from %s, time %.2fs" % \
(name, ckpt, time.time() - start_time)
return model
def testDeprecatedFunction(self, mock_warning):
self.assertEqual(0, mock_warning.call_count)
tf.compat.v1.initializers.tables_initializer()
self.assertEqual(0, mock_warning.call_count)
tf.tables_initializer()
self.assertEqual(1, mock_warning.call_count)
self.assertRegexpMatches(
mock_warning.call_args[0][1],
"deprecation_test.py:")
self.assertRegexpMatches(
mock_warning.call_args[0][2], r"tables_initializer")
self.assertRegexpMatches(
mock_warning.call_args[0][3],
r"compat.v1.tables_initializer")
tf.tables_initializer()
self.assertEqual(1, mock_warning.call_count)
def test_create_summaries_is_runnable(self):
ocr_model = self.create_model()
data = data_provider.InputEndpoints(
images=self.fake_images,
images_orig=self.fake_images,
labels=self.fake_labels,
labels_one_hot=slim.one_hot_encoding(self.fake_labels,
self.num_char_classes))
endpoints = ocr_model.create_base(
images=self.fake_images, labels_one_hot=None)
charset = create_fake_charset(self.num_char_classes)
summaries = ocr_model.create_summaries(
data, endpoints, charset, is_training=False)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
tf.tables_initializer().run()
sess.run(summaries) # just check it is runnable
def initialize_sessions(self) -> None:
log("Initializing variables")
init_op = tf.global_variables_initializer()
init_tables = tf.tables_initializer()
for sess in self.sessions:
sess.run([init_op, init_tables])
log("Initializing tf.train.Saver")
self.saver = tf.train.Saver(max_to_keep=None,
var_list=[g for g in tf.global_variables()
if "reward_" not in g.name])
def _RunInPlaceImpl(self, preprocessing_fn,
metadata,
transform_output_path):
"""Runs a transformation iteration in-place without looking at the data.
Args:
preprocessing_fn: The tf.Transform preprocessing_fn.
metadata: A DatasetMetadata object for the input data.
transform_output_path: An absolute path to write the output to.
Returns:
Status of the execution.
"""
tf.logging.info('Processing an in-place transform')
raw_metadata_dir = os.path.join(transform_output_path,
tft.TFTransformOutput.RAW_METADATA_DIR)
metadata_io.write_metadata(metadata, raw_metadata_dir)
with tf.Graph().as_default() as graph:
with tf.Session(graph=graph) as sess:
input_signature = impl_helper.feature_spec_as_batched_placeholders(
metadata.schema.as_feature_spec())
# In order to avoid a bug where import_graph_def fails when the
# input_map and return_elements of an imported graph are the same
# (b/34288791), we avoid using the placeholder of an input column as an
# output of a graph. We do this by applying tf.identity to all inputs of
# the preprocessing_fn. Note this applies at the level of raw tensors.
# TODO(b/34288791): Remove this workaround and use a shallow copy of
# inputs instead. A shallow copy is needed in case
# self._preprocessing_fn mutates its input.
copied_inputs = impl_helper.copy_tensors(input_signature)
output_signature = preprocessing_fn(copied_inputs)
sess.run(tf.global_variables_initializer())
sess.run(tf.tables_initializer())
transform_fn_path = os.path.join(transform_output_path,
tft.TFTransformOutput.TRANSFORM_FN_DIR)
saved_transform_io.write_saved_transform_from_session(
sess, input_signature, output_signature, transform_fn_path)
transformed_metadata = dataset_metadata.DatasetMetadata(
schema=tft.schema_inference.infer_feature_schema(
output_signature, graph, sess))
transformed_metadata_dir = os.path.join(
transform_output_path, tft.TFTransformOutput.TRANSFORMED_METADATA_DIR)
metadata_io.write_metadata(transformed_metadata, transformed_metadata_dir)
return _Status.OK()
def testDecodeImageLabels(self):
image_tensor = np.random.randint(256, size=(4, 5, 3)).astype(np.uint8)
encoded_jpeg = self._EncodeImage(image_tensor)
example = tf.train.Example(
features=tf.train.Features(
feature={
'image/encoded': dataset_util.bytes_feature(encoded_jpeg),
'image/format': dataset_util.bytes_feature('jpeg'),
'image/class/label': dataset_util.int64_list_feature([1, 2]),
})).SerializeToString()
example_decoder = tf_example_decoder.TfExampleDecoder()
tensor_dict = example_decoder.decode(tf.convert_to_tensor(example))
with self.test_session() as sess:
tensor_dict = sess.run(tensor_dict)
self.assertTrue(
fields.InputDataFields.groundtruth_image_classes in tensor_dict)
self.assertAllEqual(
tensor_dict[fields.InputDataFields.groundtruth_image_classes],
np.array([1, 2]))
example = tf.train.Example(
features=tf.train.Features(
feature={
'image/encoded':
dataset_util.bytes_feature(encoded_jpeg),
'image/format':
dataset_util.bytes_feature('jpeg'),
'image/class/text':
dataset_util.bytes_list_feature(['dog', 'cat']),
})).SerializeToString()
label_map_string = """
item {
id:3
name:'cat'
}
item {
id:1
name:'dog'
}
"""
label_map_path = os.path.join(self.get_temp_dir(), 'label_map.pbtxt')
with tf.gfile.Open(label_map_path, 'wb') as f:
f.write(label_map_string)
example_decoder = tf_example_decoder.TfExampleDecoder(
label_map_proto_file=label_map_path)
tensor_dict = example_decoder.decode(tf.convert_to_tensor(example))
with self.test_session() as sess:
sess.run(tf.tables_initializer())
tensor_dict = sess.run(tensor_dict)
self.assertTrue(
fields.InputDataFields.groundtruth_image_classes in tensor_dict)
self.assertAllEqual(
tensor_dict[fields.InputDataFields.groundtruth_image_classes],
np.array([1, 3]))
def build_and_run_exports(latest, job_dir, serving_input_fn, hidden_units):
"""Given the latest checkpoint file export the saved model.
Args:
latest (string): Latest checkpoint file
job_dir (string): Location of checkpoints and model files
name (string): Name of the checkpoint to be exported. Used in building the
export path.
hidden_units (list): Number of hidden units
learning_rate (float): Learning rate for the SGD
"""
prediction_graph = tf.Graph()
exporter = tf.saved_model.builder.SavedModelBuilder(
os.path.join(job_dir, 'export'))
with prediction_graph.as_default():
features, inputs_dict = serving_input_fn()
prediction_dict = model.model_fn(
model.PREDICT,
features.copy(),
None, # labels
hidden_units=hidden_units,
learning_rate=None # learning_rate unused in prediction mode
)
saver = tf.train.Saver()
inputs_info = {
name: tf.saved_model.utils.build_tensor_info(tensor)
for name, tensor in inputs_dict.iteritems()
}
output_info = {
name: tf.saved_model.utils.build_tensor_info(tensor)
for name, tensor in prediction_dict.iteritems()
}
signature_def = tf.saved_model.signature_def_utils.build_signature_def(
inputs=inputs_info,
outputs=output_info,
method_name=sig_constants.PREDICT_METHOD_NAME
)
with tf.Session(graph=prediction_graph) as session:
session.run([tf.local_variables_initializer(), tf.tables_initializer()])
saver.restore(session, latest)
exporter.add_meta_graph_and_variables(
session,
tags=[tf.saved_model.tag_constants.SERVING],
signature_def_map={
sig_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature_def
},
legacy_init_op=main_op()
)
exporter.save()
def tfhub_to_savedmodel(model_name, export_path,
uri_pattern='https://tfhub.dev/google/imagenet/{}/classification/2'):
"""Download a model from TensorFlow Hub, add inputs and outputs
suitable for serving inference requests, and export the resulting
graph as a SavedModel. This function should work for most
image classification model on TensorFlow Hub.
Args:
model_name (str): The model name (e.g. mobilenet_v2_140_224)
export_path (str): The exported model will be saved at <export_path>/<model_name>
uri_pattern (str): Optional. The model name is combined with this
pattern to form a TensorFlow Hub uri. The default value works for MobileNetV2,
but a different pattern may be needed for other models.
Returns:
str: The path to the exported SavedModel (including model_name and version).
"""
# the model will output the topk predicted classes and probabilities
topk = 3
model_path = '{}/{}/00000001'.format(export_path, model_name)
tfhub_uri = uri_pattern.format(model_name)
with tf.Session(graph=tf.Graph()) as sess:
module = hub.Module(tfhub_uri)
input_params = module.get_input_info_dict()
dtype = input_params['images'].dtype
shape = input_params['images'].get_shape()
# define the model inputs
inputs = {'images': tf.placeholder(dtype, shape, 'images')}
# define the model outputs
# we want the class ids and probabilities for the top 3 classes
logits = module(inputs['images'])
softmax = tf.nn.softmax(logits, name=None)
probs, classes = tf.nn.top_k(softmax, k=topk, sorted=True, name=None)
outputs = {
'classes': classes,
'probabilities': probs
}
# export the model
sess.run([tf.global_variables_initializer(), tf.tables_initializer()])
tf.saved_model.simple_save(
sess,
model_path,
inputs=inputs,
outputs=outputs)
return model_path
def create_or_load_model(model, model_dir, session, name):
latest_ckpt = tf.train.latest_checkpoint(model_dir)
if latest_ckpt:
model = load_model(model, latest_ckpt, session, name)
else:
start_time = time.time()
session.run(tf.global_variables_initializer())
session.run(tf.tables_initializer())
print " created %s model with fresh parameters, time %.2fs" % \
(name, time.time() - start_time)
global_step = model.global_step.eval(session=session)
return model, global_step
def _createTestEvalModel(self, m_creator, hparams, sess):
eval_mode = tf.contrib.learn.ModeKeys.EVAL
eval_iterator, src_vocab_table, tgt_vocab_table = (
common_test_utils.create_test_iterator(hparams, eval_mode))
eval_m = m_creator(
hparams,
eval_mode,
eval_iterator,
src_vocab_table,
tgt_vocab_table,
scope='dynamic_seq2seq')
sess.run(tf.tables_initializer())
sess.run(eval_iterator.initializer)
return eval_m
def create_or_load_model(model, model_dir, session, name):
"""Create translation model and initialize or load parameters in session."""
latest_ckpt = tf.train.latest_checkpoint(model_dir)
if latest_ckpt:
model = load_model(model, latest_ckpt, session, name)
else:
start_time = time.time()
session.run(tf.global_variables_initializer())
session.run(tf.tables_initializer())
utils.print_out(" created %s model with fresh parameters, time %.2fs" %
(name, time.time() - start_time))
global_step = model.global_step.eval(session=session)
return model, global_step
def compute_data_mean_and_std(data, axis, num_samples):
"""Computes data mean and std."""
with tf.Session() as sess:
sess.run([
tf.global_variables_initializer(),
tf.local_variables_initializer(),
tf.tables_initializer()
])
with tf.contrib.slim.queues.QueueRunners(sess):
data_value = np.concatenate(
[sess.run(data) for _ in range(num_samples)], axis=0)
mean = np.mean(data_value, axis=tuple(axis), keepdims=True)
std = np.std(data_value, axis=tuple(axis), keepdims=True)
return mean, std
def testAssets(self):
export_path = os.path.join(self.get_temp_dir(), "assets-module")
vocabulary_file = self.create_vocab_file("tokens.txt",
["emerson", "lake", "palmer"])
with tf.Graph().as_default():
assets_module_fn = create_assets_module_fn(vocabulary_file)
spec = hub.create_module_spec(assets_module_fn)
embedding_module = hub.Module(spec)
output = embedding_module(tf.constant([1, 2], dtype=tf.int64))
with tf.Session() as sess:
sess.run(tf.tables_initializer())
self.assertAllEqual(list(sess.run(output)), [b"lake", b"palmer"])
embedding_module.export(export_path, sess)
asset_file = os.path.join(*[export_path, "assets", "tokens.txt"])
# Check that asset file got written to the expected place:
self.assertTrue(tf.gfile.Exists(asset_file))
# Assets should be hermetic, so we can delete the original vocab file:
tf.gfile.Remove(vocabulary_file)
with tf.Graph().as_default():
spec = load_module_spec(export_path)
embedding_module = hub.Module(spec)
output = embedding_module(tf.constant([1, 2], dtype=tf.int64))
with tf.Session() as sess:
sess.run(tf.tables_initializer())
# Check functionality:
self.assertAllEqual(list(sess.run(output)), [b"lake", b"palmer"])
# Check that the ASSET_FILEPATHS collection was restored properly:
asset_filepaths = [
sess.run(tensor)
for tensor in tf.get_collection(tf.GraphKeys.ASSET_FILEPATHS)
]
# ASSET_FILEPATHS are added for the state graph and for the apply graph:
self.assertAllEqual(asset_filepaths,
[tf.compat.as_bytes(asset_file)] * 2)
def main():
# data batch
datasets = {
mode: tx.data.MultiAlignedData(hparams)
for mode, hparams in config_data.datas.items()
}
data_iterator = tx.data.FeedableDataIterator(datasets)
data_batch = data_iterator.get_next()
global_step = tf.train.get_or_create_global_step()
train_ops, bs_outputs, \
= build_model(data_batch, datasets['train'], global_step)
summary_ops = {
name: tf.summary.merge(tf.get_collection(
tf.GraphKeys.SUMMARIES, scope=get_scope_name_of_train_op(name)),
name=get_scope_name_of_summary_op(name))
for name in train_ops.keys()
}
saver = tf.train.Saver(max_to_keep=None)
global best_ever_val_bleu
best_ever_val_bleu = 0.
def _save_to(directory, step):
print('saving to {} ...'.format(directory))
saved_path = saver.save(sess, directory, global_step=step)
print('saved to {}'.format(saved_path))
def _restore_from_path(ckpt_path):
print('restoring from {} ...'.format(ckpt_path))
try:
saver.restore(sess, ckpt_path)
except tf.errors.NotFoundError:
print('Some variables are missing. Try optimistically restoring.')
(get_optimistic_saver(ckpt_path)).restore(sess, ckpt_path)
print('done.')
def _restore_from(directory):
if os.path.exists(directory):
ckpt_path = tf.train.latest_checkpoint(directory)
_restore_from_path(ckpt_path)
else:
print('cannot find checkpoint directory {}'.format(directory))
def _train_epoch(sess, summary_writer, mode, train_op, summary_op):
print('in _train_epoch')
data_iterator.restart_dataset(sess, mode)
feed_dict = {
tx.global_mode(): tf.estimator.ModeKeys.TRAIN,
data_iterator.handle: data_iterator.get_handle(sess, mode),
}
while True:
try:
loss, summary = sess.run((train_op, summary_op), feed_dict)
step = tf.train.global_step(sess, global_step)
print('step {:d}: loss = {:.6f}'.format(step, loss))
summary_writer.add_summary(summary, step)
# if step % config_train.steps_per_eval == 0:
# _eval_epoch(sess, summary_writer, 'val')
except tf.errors.OutOfRangeError:
break
print('end _train_epoch')
def _eval_epoch(sess, summary_writer, mode):
global best_ever_val_bleu
print('in _eval_epoch with mode {}'.format(mode))
data_iterator.restart_dataset(sess, mode)
feed_dict = {
tx.global_mode(): tf.estimator.ModeKeys.EVAL,
data_iterator.handle: data_iterator.get_handle(sess, mode)
}
step = tf.train.global_step(sess, global_step)
ref_hypo_pairs = []
fetches = [
[data_batch['y_aux_text'], data_batch['y_ref_text']],
[data_batch['x_value_text'], data_batch['x_ref_value_text']],
bs_outputs.predicted_ids,
]
if not os.path.exists(dir_model):
os.makedirs(dir_model)
hypo_file_name = os.path.join(dir_model,
"hypos.step{}.{}.txt".format(step, mode))
hypo_file = open(hypo_file_name, "w")
cnt = 0
while True:
try:
target_texts, entry_texts, output_ids = sess.run(
fetches, feed_dict)
target_texts = [
tx.utils.strip_special_tokens(texts[:, 1:].tolist(),
is_token_list=True)
for texts in target_texts
]
entry_texts = [
tx.utils.strip_special_tokens(texts[:, 1:].tolist(),
is_token_list=True)
for texts in entry_texts
]
output_ids = output_ids[:, :, 0]
output_texts = tx.utils.map_ids_to_strs(
ids=output_ids.tolist(),
vocab=datasets[mode].vocab('y_aux'),
join=False)
target_texts = list(zip(*target_texts))
entry_texts = list(zip(*entry_texts))
for ref, hypo in zip(target_texts, output_texts):
if cnt < 10:
print('cnt = {}'.format(cnt))
for i, s in enumerate(ref):
print('ref{}: {}'.format(i, ' '.join(s)))
print('hypo: {}'.format(' '.join(hypo)))
print(' '.join(hypo), file=hypo_file)
cnt += 1
print('processed {} samples'.format(cnt))
ref_hypo_pairs.extend(
zip(target_texts, entry_texts, output_texts))
except tf.errors.OutOfRangeError:
break
hypo_file.close()
if FLAGS.eval_ie:
gold_file_name = os.path.join(
config_data.dst_dir,
"gold.{}.txt".format(config_data.mode_to_filemode[mode]))
inter_file_name = "{}.h5".format(hypo_file_name[:-len(".txt")])
prec, rec = get_precrec(gold_file_name,
hypo_file_name,
inter_file_name,
gpuid=FLAGS.eval_ie_gpuid)
refs, entrys, hypos = zip(*ref_hypo_pairs)
bleus = []
get_bleu_name = '{}_BLEU'.format
for i in range(1, 2):
refs_ = list(map(lambda ref: ref[i:i + 1], refs))
ents_ = list(map(lambda ent: ent[i:i + 1], entrys))
entrys = list(zip(*entrys))
bleu = corpus_bleu(refs_, hypos)
print('==========={}: {:.2f}'.format(get_bleu_name(i), bleu))
bleus.append(bleu)
summary = tf.Summary()
for i, bleu in enumerate(bleus):
summary.value.add(tag='{}/{}'.format(mode, get_bleu_name(i)),
simple_value=bleu)
if FLAGS.eval_ie:
for name, value in {'precision': prec, 'recall': rec}.items():
summary.value.add(tag='{}/{}'.format(mode, name),
simple_value=value)
summary_writer.add_summary(summary, step)
summary_writer.flush()
bleu = bleus[0]
if mode == 'val':
if bleu > best_ever_val_bleu:
best_ever_val_bleu = bleu
print('updated best val bleu: {}'.format(bleu))
_save_to(ckpt_best, step)
print('end _eval_epoch')
return bleu
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
sess.run(tf.tables_initializer())
if restore_from:
_restore_from_path(restore_from)
else:
_restore_from(dir_model)
summary_writer = tf.summary.FileWriter(dir_summary,
sess.graph,
flush_secs=30)
epoch = 0
while epoch < config_train.max_epochs:
name = 'joint'
train_op = train_ops[name]
summary_op = summary_ops[name]
val_bleu = _eval_epoch(sess, summary_writer, 'val')
test_bleu = _eval_epoch(sess, summary_writer, 'test')
step = tf.train.global_step(sess, global_step)
print('epoch: {} ({}), step: {}, '
'val BLEU: {:.2f}, test BLEU: {:.2f}'.format(
epoch, name, step, val_bleu, test_bleu))
_train_epoch(sess, summary_writer, 'train', train_op, summary_op)
epoch += 1
step = tf.train.global_step(sess, global_step)
_save_to(ckpt_model, step)
test_bleu = _eval_epoch(sess, summary_writer, 'test')
print('epoch: {}, test BLEU: {}'.format(epoch, test_bleu))
def main(_):
pipeline_proto = _load_pipeline_proto(FLAGS.pipeline_proto)
tf.logging.info("Pipeline configure: %s", '=' * 128)
tf.logging.info(pipeline_proto)
g = tf.Graph()
with g.as_default():
# Infer saliency.
model = builder.build(pipeline_proto.model, is_training=False)
predictions = model.build_prediction(
examples={}, prediction_task=GAPPredictionTasks.word_saliency)
saver = tf.train.Saver()
invalid_variable_names = tf.report_uninitialized_variables()
with tf.Session(graph=g) as sess:
sess.run(tf.tables_initializer())
# Load the latest checkpoint.
checkpoint_path = tf.train.latest_checkpoint(pipeline_proto.model_dir)
assert checkpoint_path is not None
saver.restore(sess, checkpoint_path)
assert len(sess.run(invalid_variable_names)) == 0
predictions = sess.run(predictions)
# Process kNN retrieval.
name_to_class_id = {}
with open(FLAGS.name_to_class_id_file, 'r') as fid:
for line in fid.readlines():
name, class_id = line.strip('\n').split('\t')
name_to_class_id[name] = class_id
(vocabulary, word_saliency,
word_embedding) = (predictions[GAPPredictions.vocabulary],
predictions[GAPPredictions.word_saliency],
predictions[GAPPredictions.word_embedding])
queries = list(name_to_class_id)
synonyms_list = _knn_retrieval(queries, vocabulary, word_embedding,
word_saliency)
# Print to the terminal.
expanded_name_to_class_id = []
for query, synonyms in zip(queries, synonyms_list):
elems = []
for synonym in synonyms:
if synonym['saliency'] < FLAGS.saliency_threshold:
continue
if synonym['similarity'] < FLAGS.similarity_threshold:
continue
elems.append(synonym['word'])
expanded_name_to_class_id.append((synonym['word'],
name_to_class_id[query]))
print('%s\t%s' % (query, ','.join(elems)))
# Write to output file.
with open(FLAGS.expanded_name_to_class_id_file, 'w') as fid:
for word, class_id in expanded_name_to_class_id:
fid.write('%s\t%s\n' % (word, class_id))
tf.logging.info('Done')
def export():
# Create index->synset mapping
synsets = []
with open(SYNSET_FILE) as f:
synsets = f.read().splitlines()
# Create synset->metadata mapping
texts = {}
with open(METADATA_FILE) as f:
for line in f.read().splitlines():
parts = line.split('\t')
assert len(parts) == 2
texts[parts[0]] = parts[1]
with tf.Graph().as_default():
# Build inference model.
# Please refer to Tensorflow inception model for details.
# Input transformation.
serialized_tf_example = tf.placeholder(tf.string, name='tf_example')
feature_configs = {
'image/encoded': tf.FixedLenFeature(shape=[], dtype=tf.string),
}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
jpegs = tf_example['image/encoded']
images = tf.map_fn(preprocess_image, jpegs, dtype=tf.float32)
# Run inference.
logits, _ = inception_model.inference(images, NUM_CLASSES + 1)
# Transform output to topK result.
values, indices = tf.nn.top_k(logits, NUM_TOP_CLASSES)
# Create a constant string Tensor where the i'th element is
# the human readable class description for the i'th index.
# Note that the 0th index is an unused background class
# (see inception model definition code).
class_descriptions = ['unused background']
for s in synsets:
class_descriptions.append(texts[s])
class_tensor = tf.constant(class_descriptions)
table = tf.contrib.lookup.index_to_string_table_from_tensor(
class_tensor)
classes = table.lookup(tf.to_int64(indices))
# Restore variables from training checkpoint.
variable_averages = tf.train.ExponentialMovingAverage(
inception_model.MOVING_AVERAGE_DECAY)
variables_to_restore = variable_averages.variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
with tf.Session() as sess:
# Restore variables from training checkpoints.
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
# Assuming model_checkpoint_path looks something like:
# /my-favorite-path/imagenet_train/model.ckpt-0,
# extract global_step from it.
global_step = ckpt.model_checkpoint_path.split('/')[-1].split(
'-')[-1]
print('Successfully loaded model from %s at step=%s.' %
(ckpt.model_checkpoint_path, global_step))
else:
print('No checkpoint file found at %s' % FLAGS.checkpoint_dir)
return
# Export inference model.
init_op = tf.group(tf.tables_initializer(), name='init_op')
classification_signature = exporter.classification_signature(
input_tensor=serialized_tf_example,
classes_tensor=classes,
scores_tensor=values)
named_graph_signature = {
'inputs':
exporter.generic_signature({'images': jpegs}),
'outputs':
exporter.generic_signature({
'classes': classes,
'scores': values
})
}
model_exporter = exporter.Exporter(saver)
model_exporter.init(
init_op=init_op,
default_graph_signature=classification_signature,
named_graph_signatures=named_graph_signature)
model_exporter.export(FLAGS.export_dir, tf.constant(global_step),
sess)
print('Successfully exported model to %s' % FLAGS.export_dir)
def train_text_feature(epochs,train_text,train_label,test_text,test_label,
batch_size=100,
hub_module = "https://tfhub.dev/google/nnlm-en-dim128/1",
optimizer = tf.train.AdamOptimizer):
'''
Use a pretrained text embedding model followed by a 2 hidden layers NN to predict price movement.
'''
tf.reset_default_graph()
train_flag = tf.placeholder(dtype = tf.bool)
with tf.device('/cpu:0'):
train_dataset = tf.data.Dataset.from_tensor_slices((train_text,train_label)).shuffle(1000).batch(batch_size).prefetch(10)
test_dataset = tf.data.Dataset.from_tensor_slices((test_text,test_label)).shuffle(100).batch(batch_size).prefetch(1)
iter_train = train_dataset.make_initializable_iterator()
sentences_train, y_train = iter_train.get_next()
iter_test = test_dataset.make_initializable_iterator()
sentences_test, y_test = iter_test.get_next()
# load pretrained model
pretrained_module = hub.Module(hub_module)
# text embedding using pretrained model
input = tf.cond(train_flag,lambda:pretrained_module(sentences_train),lambda:pretrained_module(sentences_test))
# followed by several dense layers
# note the dataset is small...so we don't want NN to be too large
dense1 = tf.layers.Dense(64,activation=tf.nn.sigmoid)
dense2 = tf.layers.Dense(16,activation=tf.nn.tanh)
dense3 = tf.layers.Dense(1)
dense1_output = dense1(input)
dense2_output = dense2(dense1_output)
outputs = dense3(dense2_output)
loss = tf.cond(train_flag,lambda:tf.nn.sigmoid_cross_entropy_with_logits(labels=y_train,logits=outputs),
lambda:tf.nn.sigmoid_cross_entropy_with_logits(labels=y_test,logits=outputs))
loss = tf.reduce_mean(loss)
opt = optimizer(learning_rate=0.03)
training_op = opt.minimize(loss)
saver = tf.train.Saver()
with tf.Session() as sess:
for ep in range(1,epochs+2):
sess.run(iter_test.initializer)
if ep==1:
sess.run([tf.global_variables_initializer(),tf.tables_initializer()])
if ep%10==1:
sess.run(iter_train.initializer)
sess.run(training_op,feed_dict={train_flag:True})
with tf.variable_scope('eval2', reuse=True):
if ep%100 == 1:
sess.run(iter_train.initializer)
sess.run(iter_test.initializer)
print('Training Epoch : {}'.format(ep))
train_loss = sess.run(loss,feed_dict={train_flag:True})
sess.run(iter_test.initializer)
sess.run(iter_train.initializer)
test_loss = sess.run(loss,feed_dict={train_flag:False})
print('Training Loss at step {} : {}'.format(ep,train_loss))
print('Test Loss at step {} : {}'.format(ep,test_loss))
save_path = saver.save(sess, "/training_text/model.ckpt")
print("Model saved in path: %s" % save_path)
def __init__(
self,
batch_size=2,
feature_num=2,
train_epoch_num=2,
train_steps=1000,
tag2value=None,
tag2valueOneline=None,
custom_tags=[],
wide_side_node=100,
deep_side_nodes=[700, 100],
# deep_side_nodes=[1024,512,256],
eval_freq=1000, #每隔1000个step，evaluate一次
moving_average_decay=0.99, # 滑动平均的衰减系数
learning_rate_base=0.00001, # 基学习率
learning_rate_decay=0.99, # 学习率的衰减率
total_example_num=2000000,
train_filename="",
eval_filename="",
test_filename="",
features_to_exclude=[],
features_to_keep=[],
args=None):
assert tag2value is not None, "tag2value should be a dict, but found None"
assert tag2valueOneline is not None, "tag2valueOneline should be a dict, but found None"
self.batch_size = batch_size
self.feature_num = feature_num
self.train_epoch_num = train_epoch_num
self.train_steps = train_steps
self.args = args
self.logger = logging.getLogger("brc")
self.tag2valueOneline = tag2valueOneline
self.tag2value = tag2value
self.custom_tags = custom_tags
self.wide_side_node = wide_side_node
self.deep_side_nodes = deep_side_nodes
self.eval_freq = eval_freq
self.moving_average_decay = moving_average_decay
self.learning_rate_base = learning_rate_base
self.learning_rate_decay = learning_rate_decay
self.total_example_num = total_example_num
self.train_filename = train_filename
self.eval_filename = eval_filename
self.test_filename = test_filename
self.features_to_exclude = features_to_exclude
self.features_to_keep = features_to_keep
self.tag2value = {}
self.tag2valueOneline = {}
if self.features_to_keep:
for key in self.features_to_keep:
if key in tag2value:
self.tag2value[key] = tag2value[key]
if key in tag2valueOneline:
self.tag2valueOneline[key] = tag2valueOneline[key]
start_t = time.time()
self.sess = tf.Session()
self._setup_placeholder()
self._setup_mappings()
self._realize_mappings()
self._build_graph()
self._build_loss()
self._create_train_op()
self.saver = tf.train.Saver()
self.sess.run(
[tf.global_variables_initializer(),
tf.tables_initializer()])
def testGenerateSimple(self):
# Reader config
with open(TEST_DATA + "/config.json") as json_file:
self.reader_config = json.load(json_file)
# Initialize the reader
receptive_field_size = WaveNetModel.calculate_receptive_field(2, LAYERS, False, 8)
self.reader = CsvReader(
[TEST_DATA + "/test.dat", TEST_DATA + "/test.emo", TEST_DATA + "/test.pho"],
batch_size=1,
receptive_field=receptive_field_size,
sample_size=SAMPLE_SIZE,
config=self.reader_config
)
# WaveNet model
self.net = WaveNetModel(batch_size=1,
dilations=LAYERS,
filter_width=2,
residual_channels=8,
dilation_channels=8,
skip_channels=8,
quantization_channels=2,
use_biases=True,
scalar_input=False,
initial_filter_width=8,
histograms=False,
global_channels=GC_CHANNELS,
local_channels=LC_CHANNELS)
loss = self.net.loss(input_batch=self.reader.data_batch,
global_condition=self.reader.gc_batch,
local_condition=self.reader.lc_batch,
l2_regularization_strength=L2)
optimizer = optimizer_factory['adam'](learning_rate=0.003, momentum=0.9)
trainable = tf.trainable_variables()
train_op = optimizer.minimize(loss, var_list=trainable)
samples = tf.placeholder(tf.float32, shape=(receptive_field_size, self.reader.data_dim), name="samples")
gc = tf.placeholder(tf.int32, shape=(receptive_field_size), name="gc")
lc = tf.placeholder(tf.int32, shape=(receptive_field_size), name="lc")
gc = tf.one_hot(gc, GC_CHANNELS)
lc = tf.one_hot(lc, LC_CHANNELS)
predict = self.net.predict_proba(samples, gc, lc)
'''does nothing'''
with self.test_session() as session:
session.run([
tf.local_variables_initializer(),
tf.global_variables_initializer(),
tf.tables_initializer(),
])
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=session, coord=coord)
for ITER in range(1):
for i in range(1000):
_, loss_val = session.run([train_op, loss])
print("step %d loss %.4f" % (i, loss_val), end='\r')
sys.stdout.flush()
print()
data_samples = np.random.random((receptive_field_size, self.reader.data_dim))
gc_samples = np.zeros((receptive_field_size))
lc_samples = np.zeros((receptive_field_size))
output = []
for EMO in range(3):
for PHO in range(3):
for _ in range(100):
prediction = session.run(predict, feed_dict={'samples:0': data_samples, 'gc:0': gc_samples, 'lc:0': lc_samples})
data_samples = data_samples[1:, :]
data_samples = np.append(data_samples, prediction, axis=0)
gc_samples = gc_samples[1:]
gc_samples = np.append(gc_samples, [EMO], axis=0)
lc_samples = lc_samples[1:]
lc_samples = np.append(lc_samples, [PHO], axis=0)
output.append(prediction[0])
output = np.array(output)
print("ITER %d" % ITER)
plt.imsave("./test/SINE_test_%d.png" % ITER, np.kron(output[:, :], np.ones([1, 500])), vmin=0.0, vmax=1.0)
def _main(_):
'''
if config.distributed:
import horovod.tensorflow as hvd
hvd.init()
for i in range(14):
config.train_data["datasets"][i]["num_shards"] = hvd.size()
config.train_data["datasets"][i]["shard_id"] = hvd.rank()
config.train_data["batch_size"] //= hvd.size()
'''
# Data
train_data = MultiAlignedNumpyData(config.train_data)
val_data = MultiAlignedNumpyData(config.val_data)
test_data = MultiAlignedNumpyData(config.test_data)
vocab = train_data.vocab(0)
ctx_maxSeqLen = config.max_sequence_length
# Each training batch is used twice: once for updating the generator and
# once for updating the discriminator. Feedable data iterator is used for
# such case.
iterator = tx.data.FeedableDataIterator(
{'train_pre': train_data, 'val': val_data, 'test': test_data})
batch = iterator.get_next()
# Model
#global_step = tf.placeholder(tf.int32) #hvd
lr = tf.placeholder(dtype=tf.float32, shape=[], name='lr') #hvd
if config.model_name == 'EvolveGTAE':
#model = EvolveGTAE(batch, vocab, ctx_maxSeqLen, hvd, global_step, config.model)
model = EvolveGTAE(batch, vocab, ctx_maxSeqLen, lr, config.model)
else:
logger.error('config.model_name: {} is incorrect'.format(config.model_name))
raise ValueError('config.model_name: {} is incorrect'.format(config.model_name))
def _train_epoch(sess, lr_, writer, epoch, flag, verbose=True):
avg_meters_pre = tx.utils.AverageRecorder(size=10)
step = 0
while True:
try:
step += 1
feed_dict = {
iterator.handle: iterator.get_handle(sess, 'train_pre'),
lr:lr_
}
vals_pre = sess.run(model.fetches_train_pre, feed_dict=feed_dict)
merged_summary = vals_pre.pop('merged')
iteration = (epoch-1)*(config.train_num/config.batch_size)+step
if iteration % 20 == 0:
writer.add_summary(merged_summary, iteration)
avg_meters_pre.add(vals_pre)
if verbose and (step == 1 or step % config.display == 0):
logger.info('step: {}, {}'.format(step, avg_meters_pre.to_str(4)))
sys.stdout.flush()
except tf.errors.OutOfRangeError:
logger.info('epoch: {}, {}'.format(epoch, avg_meters_pre.to_str(4)))
sys.stdout.flush()
break
def _eval_epoch(sess, lr_, writer, epoch, val_or_test='val'):##1
avg_meters = tx.utils.AverageRecorder()
step = 0
while True:
try:
step += 1
feed_dict = {
iterator.handle: iterator.get_handle(sess, val_or_test),
lr:lr_,
tx.context.global_mode(): tf.estimator.ModeKeys.EVAL
}
vals = sess.run(model.fetches_eval, feed_dict=feed_dict)
iteration = (epoch-1)*(config.dev_num/config.batch_size)+step
if val_or_test is 'val' and iteration % 10 == 0:
merged_summary = vals['merged']
writer.add_summary(merged_summary, iteration)
vals.pop('merged')
batch_size = vals.pop('batch_size')
# Computes BLEU
samples = tx.utils.dict_pop(vals, list(model.samples.keys()))
x1x2 = tx.utils.map_ids_to_strs(samples['x1x2'], vocab)
x1xx2 = tx.utils.map_ids_to_strs(samples['x1xx2'], vocab)
hyps_y1 = tx.utils.map_ids_to_strs(samples['transferred_yy1_pred'], vocab)
refs_y1 = tx.utils.map_ids_to_strs(samples['transferred_yy1_gt'], vocab) ## text == ['a sentence', 'parsed from ids']
origin_y1 = tx.utils.map_ids_to_strs(samples['origin_y1'], vocab)
refs_y1 = np.expand_dims(refs_y1, axis=1) #[32,1]
bleu = tx.evals.corpus_bleu_moses(refs_y1, hyps_y1) #[32]
vals['bleu_y1'] = bleu
hyps_y2 = tx.utils.map_ids_to_strs(samples['transferred_yy2_pred'], vocab)
refs_y2 = tx.utils.map_ids_to_strs(samples['transferred_yy2_gt'], vocab)
origin_y2 = tx.utils.map_ids_to_strs(samples['origin_y2'], vocab)
refs_y2 = np.expand_dims(refs_y2, axis=1)
bleu = tx.evals.corpus_bleu_moses(refs_y2, hyps_y2)
vals['bleu_y2'] = bleu
hyps_y3 = tx.utils.map_ids_to_strs(samples['transferred_yy3_pred'], vocab)
refs_y3 = tx.utils.map_ids_to_strs(samples['transferred_yy3_gt'], vocab)
origin_y3 = tx.utils.map_ids_to_strs(samples['origin_y3'], vocab)
refs_y3 = np.expand_dims(refs_y3, axis=1)
bleu = tx.evals.corpus_bleu_moses(refs_y3, hyps_y3)
vals['bleu_y3'] = bleu
avg_meters.add(vals, weight=batch_size)
# Writes samples
if val_or_test is 'test':
tx.utils.write_paired_text(
x1x2, x1xx2,
os.path.join(sample_path, 'val_x.%d'%epoch),
append=True, mode='v')
tx.utils.write_paired_text(
refs_y1.squeeze(), hyps_y1,
os.path.join(sample_path, 'val_y1.%d'%epoch),
append=True, mode='v')
tx.utils.write_paired_text(
refs_y2.squeeze(), hyps_y2,
os.path.join(sample_path, 'val_y2.%d'%epoch),
append=True, mode='v')
tx.utils.write_paired_text(
refs_y3.squeeze(), hyps_y3,
os.path.join(sample_path, 'val_y3.%d'%epoch),
append=True, mode='v')
tx.utils.write_paired_text(
refs_y1.squeeze(), origin_y1,
os.path.join(sample_path, 'val_yy1gt_y1.%d'%epoch),
append=True, mode='v')
tx.utils.write_paired_text(
refs_y2.squeeze(), origin_y2,
os.path.join(sample_path, 'val_yy2gt_y2.%d'%epoch),
append=True, mode='v')
tx.utils.write_paired_text(
refs_y3.squeeze(), origin_y3,
os.path.join(sample_path, 'val_yy3gt_y3.%d'%epoch),
append=True, mode='v')
except tf.errors.OutOfRangeError:
logger.info('{}: {}'.format(
val_or_test, avg_meters.to_str(precision=4)))
break
return avg_meters.avg()
'''
if config.distributed:
bcast = hvd.broadcast_global_variables(0)
tf.ConfigProto().gpu_options.visible_device_list = str(hvd.local_rank())
'''
# Runs the logics
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
sess.run(tf.tables_initializer())
# visulization
graph_writer=tf.summary.FileWriter(vis_graph_path, sess.graph)
# visulization
train_writer = tf.summary.FileWriter(vis_train_path, tf.Graph())
# visulization
val_writer = tf.summary.FileWriter(vis_val_path, tf.Graph())
saver = tf.train.Saver(max_to_keep=None)
if config.restore:
logger.info('Restore from: {}'.format(config.restore))
saver.restore(sess, config.restore)
iterator.initialize_dataset(sess)
lr_ = config.initial_lr
for epoch in range(0, config.max_nepochs + 1): ###modify
flag = True
'''
if epoch<=3:
lr_=config.initial_lr*(epoch+1)
if epoch>=10 and epoch %2==0:
lr_*=0.25
'''
logger.info('learning rate: {}'.format(lr_))
# Train
iterator.restart_dataset(sess, ['train_pre'])
_train_epoch(sess, lr_, train_writer, epoch, flag)
# Val
iterator.restart_dataset(sess, 'val')
_eval_epoch(sess, lr_,val_writer, epoch, 'val') ##1
if epoch%3==0:
saver.save(
sess, os.path.join(checkpoint_path, 'ckpt'), epoch) ###modify
# Test
iterator.restart_dataset(sess, 'test')
_eval_epoch(sess, lr_,val_writer, epoch, 'test')
graph_writer.close()
train_writer.close()
val_writer.close()
def main():
width = 120
height = 80
num_labels = 2
batch_size = 20
num_train_examples = 1000
num_test_examples = 20
train_steps = 100
# get the default tensorflow graph
g = tf.get_default_graph()
# create an iterator (not bound to any dataset)
it = tf.data.Iterator.from_structure(output_types={
"image": tf.float64,
"label": tf.int64
},
output_shapes={
"image":
tf.TensorShape(
[None, height, width, 1]),
"label":
tf.TensorShape([None])
})
# build classifier
input = it.get_next()
logits = build_classifier(input["image"], num_labels)
# loss
label = input["label"]
loss = tf.losses.sparse_softmax_cross_entropy(label, logits)
# gradient descent
optimizer = tf.train.GradientDescentOptimizer(0.01)
train = optimizer.minimize(loss)
# initialization
var_init = tf.global_variables_initializer()
table_init = tf.tables_initializer()
# create training and test datasets
_, _, train_dataset = sample_dataset(num_train_examples, width, height)
_, _, test_dataset = sample_dataset(num_test_examples, width, height)
with tf.summary.FileWriter('log') as writer:
writer.add_graph(g)
# run
s = tf.Session()
s.run((var_init, table_init))
# initialize the iterator to the training set
train_init = it.make_initializer(
train_dataset.shuffle(
buffer_size=num_train_examples).repeat().batch(batch_size))
s.run(train_init)
for step in range(train_steps):
cur_loss, _ = s.run((loss, train))
print("step {0} of {1}: loss={2:.5f}".format(step + 1, train_steps,
cur_loss))
if (step + 1) % 100 == 0:
# evaluate
print("evaluating")
i = 0
num_correct = 0
num_incorrect = 0
s.run(it.make_initializer(test_dataset.batch(1)))
punchlist = ""
while True:
try:
out, correct_label = s.run((logits, input["label"]))
except tf.errors.OutOfRangeError:
print("reached end of sequence")
break
estimated_label = np.argmax(out)
if correct_label == estimated_label:
num_correct += 1
punchlist += "_"
flag = "CORRECT"
else:
num_incorrect += 1
flag = "INCORRECT"
punchlist += "x"
#print("item {0}: estimated {1}, labelled {2} {3}".format(
# i+1, estimated_label, int(correct_label), flag))
i += 1
total = num_correct + num_incorrect
print("test accuracy: {0}/{1}: {2:.2%} |{3}|".format(
num_correct, total,
float(num_correct) / total, punchlist))
s.run(train_init)
def restore_and_save(input_checkpoint, export_path_base):
checkpoint_file = tf.train.latest_checkpoint(input_checkpoint)
graph = tf.Graph()
with graph.as_default():
session_conf = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
sess = tf.Session(config=session_conf)
with sess.as_default():
# 载入保存好的meta graph，恢复图中变量，通过SavedModelBuilder保存可部署的模型
saver = tf.train.import_meta_graph(
"{}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
print("***", graph.get_name_scope())
export_path_base = export_path_base
export_path = os.path.join(tf.compat.as_bytes(export_path_base),
tf.compat.as_bytes(str(count)))
print('Exporting trained model to', export_path)
builder = tf.saved_model.builder.SavedModelBuilder(export_path)
# 建立签名映射，需要包括计算图中的placeholder（ChatInputs, SegInputs, Dropout）和我们需要的结果（project/logits,crf_loss/transitions）
"""
build_tensor_info：建立一个基于提供的参数构造的TensorInfo protocol buffer，
输入：tensorflow graph中的tensor；
输出：基于提供的参数（tensor）构建的包含TensorInfo的protocol buffer
get_operation_by_name：通过name获取checkpoint中保存的变量，能够进行这一步的前提是在模型保存的时候给对应的变量赋予name
"""
char_inputs = tf.saved_model.utils.build_tensor_info(
graph.get_operation_by_name("ChatInputs").outputs[0])
seg_inputs = tf.saved_model.utils.build_tensor_info(
graph.get_operation_by_name("SegInputs").outputs[0])
dropout = tf.saved_model.utils.build_tensor_info(
graph.get_operation_by_name("Dropout").outputs[0])
logits = tf.saved_model.utils.build_tensor_info(
graph.get_operation_by_name("project/Reshape").outputs[0])
transition_params = tf.saved_model.utils.build_tensor_info(
graph.get_operation_by_name("crf_loss/Mean").outputs[0])
# graph.get_operation_by_name("crf_loss/transitions").outputs[0]) # 原代码,修改无变化
"""
signature_constants：SavedModel保存和恢复操作的签名常量。
在序列标注的任务中，这里的method_name是"tensorflow/serving/predict"
"""
# 定义模型的输入输出，建立调用接口与tensor签名之间的映射
labeling_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={
"charinputs": char_inputs,
"dropout": dropout,
"seginputs": seg_inputs,
},
outputs={
"logits": logits,
"transitions": transition_params
},
method_name="tensorflow/serving/predict"))
"""
tf.group : 创建一个将多个操作分组的操作，返回一个可以执行所有输入的操作
"""
legacy_init_op = tf.group(tf.tables_initializer(),
name='legacy_init_op')
"""
add_meta_graph_and_variables：建立一个Saver来保存session中的变量，
输出对应的原图的定义，这个函数假设保存的变量已经被初始化；
对于一个SavedModelBuilder，这个API必须被调用一次来保存meta graph；
对于后面添加的图结构，可以使用函数 add_meta_graph()来进行添加
"""
# 建立模型名称与模型签名之间的映射
builder.add_meta_graph_and_variables(
sess,
[tf.saved_model.tag_constants.SERVING],
# 保存模型的方法名，与客户端的request.model_spec.signature_name对应
signature_def_map={
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
labeling_signature
},
legacy_init_op=legacy_init_op)
builder.save()
print("Build Done")
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.8,
allow_growth=False)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
K.set_session(sess)
K._LEARNING_PHASE = tf.constant(0)
K.set_learning_phase(0)
VGG = VGG16(weights='imagenet')
prediction_signature = tf.saved_model.signature_def_utils.predict_signature_def(
{"image": VGG.input}, {"prediction": VGG.output})
builder = saved_model_builder.SavedModelBuilder(saved_model_dir)
legacy_init_op = tf.group(tf.tables_initializer(), name='legacy_init_op')
# Initialize global variables and the model
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess.run(init_op)
# Add the meta_graph and the variables to the builder
builder.add_meta_graph_and_variables(sess, [tag_constants.SERVING],
signature_def_map={
'prediction': prediction_signature,
},
legacy_init_op=legacy_init_op)
builder.save()
print('Successfully exported model to ', saved_model_dir)
def _main(_):
# Data
train_data = tx.data.MultiAlignedData(config.train_data)
val_data = tx.data.MultiAlignedData(config.val_data)
test_data = tx.data.MultiAlignedData(config.test_data)
vocab = train_data.vocab(0)
# Each training batch is used twice: once for updating the generator and
# once for updating the discriminator. Feedable data iterator is used for
# such case.
iterator = tx.data.FeedableDataIterator({
'train_g': train_data,
'train_d': train_data,
'val': val_data,
'test': test_data
})
batch = iterator.get_next()
# Model
gamma = tf.placeholder(dtype=tf.float32, shape=[], name='gamma')
lambda_g = tf.placeholder(dtype=tf.float32, shape=[], name='lambda_g')
model = CtrlGenModel(batch, vocab, gamma, lambda_g, config.model)
def _train_epoch(sess, gamma_, lambda_g_, epoch, verbose=True):
avg_meters_d = tx.utils.AverageRecorder(size=10)
avg_meters_g = tx.utils.AverageRecorder(size=10)
step = 0
while True:
try:
step += 1
feed_dict = {
iterator.handle: iterator.get_handle(sess, 'train_d'),
gamma: gamma_,
lambda_g: lambda_g_
}
vals_d = sess.run(model.fetches_train_d, feed_dict=feed_dict)
avg_meters_d.add(vals_d)
feed_dict = {
iterator.handle: iterator.get_handle(sess, 'train_g'),
gamma: gamma_,
lambda_g: lambda_g_
}
vals_g = sess.run(model.fetches_train_g, feed_dict=feed_dict)
avg_meters_g.add(vals_g)
if verbose and (step == 1 or step % config.display == 0):
print('step: {}, {}'.format(step, avg_meters_d.to_str(4)))
print('step: {}, {}'.format(step, avg_meters_g.to_str(4)))
if verbose and step % config.display_eval == 0:
iterator.restart_dataset(sess, 'val')
_eval_epoch(sess, gamma_, lambda_g_, epoch)
except tf.errors.OutOfRangeError:
print('epoch: {}, {}'.format(epoch, avg_meters_d.to_str(4)))
print('epoch: {}, {}'.format(epoch, avg_meters_g.to_str(4)))
break
def _eval_epoch(sess, gamma_, lambda_g_, epoch, val_or_test='val'):
avg_meters = tx.utils.AverageRecorder()
while True:
try:
feed_dict = {
iterator.handle: iterator.get_handle(sess, val_or_test),
gamma: gamma_,
lambda_g: lambda_g_,
tx.context.global_mode(): tf.estimator.ModeKeys.EVAL
}
vals = sess.run(model.fetches_eval, feed_dict=feed_dict)
batch_size = vals.pop('batch_size')
# Computes BLEU
samples = tx.utils.dict_pop(vals, list(model.samples.keys()))
hyps = tx.utils.map_ids_to_strs(samples['transferred'], vocab)
refs = tx.utils.map_ids_to_strs(samples['original'], vocab)
refs = np.expand_dims(refs, axis=1)
bleu = tx.evals.corpus_bleu_moses(refs, hyps)
tf.summary.scalar('Bleu', bleu)
vals['bleu'] = bleu
avg_meters.add(vals, weight=batch_size)
# Writes samples
tx.utils.write_paired_text(refs.squeeze(),
hyps,
os.path.join(
config.sample_path,
'val.%d' % epoch),
append=True,
mode='v')
merged = tf.summary.merge_all()
writer = tf.summary.FileWriter('./summary', sess.graph)
result = sess.run(merged)
writer.add_summary(result)
except tf.errors.OutOfRangeError:
print('{}: {}'.format(val_or_test,
avg_meters.to_str(precision=4)))
break
return avg_meters.avg()
tf.gfile.MakeDirs(config.sample_path)
tf.gfile.MakeDirs(config.checkpoint_path)
# Runs the logics
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
sess.run(tf.tables_initializer())
saver = tf.train.Saver(max_to_keep=None)
if config.restore:
print('Restore from: {}'.format(config.restore))
saver.restore(sess, config.restore)
iterator.initialize_dataset(sess)
gamma_ = 1.
lambda_g_ = 0.
for epoch in range(1, config.max_nepochs + 1):
if epoch > config.pretrain_nepochs:
# Anneals the gumbel-softmax temperature
gamma_ = max(0.001, gamma_ * config.gamma_decay)
lambda_g_ = config.lambda_g
print('gamma: {}, lambda_g: {}'.format(gamma_, lambda_g_))
# Train
iterator.restart_dataset(sess, ['train_g', 'train_d'])
_train_epoch(sess, gamma_, lambda_g_, epoch)
# Val
iterator.restart_dataset(sess, 'val')
_eval_epoch(sess, gamma_, lambda_g_, epoch, 'val')
saver.save(sess, os.path.join(config.checkpoint_path, 'ckpt'),
epoch)
# Test
iterator.restart_dataset(sess, 'test')
_eval_epoch(sess, gamma_, lambda_g_, epoch, 'test')
def initialize_vars(tf_session):
"""Initialize tf session variables."""
tf_session.run(tf.local_variables_initializer())
tf_session.run(tf.global_variables_initializer())
tf_session.run(tf.tables_initializer())
K.set_session(tf_session)
def _run_checkpoint_once(tensor_dict,
evaluators=None,
batch_processor=None,
checkpoint_dirs=None,
variables_to_restore=None,
restore_fn=None,
num_batches=1,
master='',
save_graph=False,
save_graph_dir='',
losses_dict=None,
eval_export_path=None,
process_metrics_fn=None):
"""Evaluates metrics defined in evaluators and returns summaries.
This function loads the latest checkpoint in checkpoint_dirs and evaluates
all metrics defined in evaluators. The metrics are processed in batch by the
batch_processor.
Args:
tensor_dict: a dictionary holding tensors representing a batch of detections
and corresponding groundtruth annotations.
evaluators: a list of object of type DetectionEvaluator to be used for
evaluation. Note that the metric names produced by different evaluators
must be unique.
batch_processor: a function taking four arguments:
1. tensor_dict: the same tensor_dict that is passed in as the first
argument to this function.
2. sess: a tensorflow session
3. batch_index: an integer representing the index of the batch amongst
all batches
By default, batch_processor is None, which defaults to running:
return sess.run(tensor_dict)
To skip an image, it suffices to return an empty dictionary in place of
result_dict.
checkpoint_dirs: list of directories to load into an EnsembleModel. If it
has only one directory, EnsembleModel will not be used --
a DetectionModel
will be instantiated directly. Not used if restore_fn is set.
variables_to_restore: None, or a dictionary mapping variable names found in
a checkpoint to model variables. The dictionary would normally be
generated by creating a tf.train.ExponentialMovingAverage object and
calling its variables_to_restore() method. Not used if restore_fn is set.
restore_fn: None, or a function that takes a tf.Session object and correctly
restores all necessary variables from the correct checkpoint file. If
None, attempts to restore from the first directory in checkpoint_dirs.
num_batches: the number of batches to use for evaluation.
master: the location of the Tensorflow session.
save_graph: whether or not the Tensorflow graph is stored as a pbtxt file.
save_graph_dir: where to store the Tensorflow graph on disk. If save_graph
is True this must be non-empty.
losses_dict: optional dictionary of scalar detection losses.
eval_export_path: Path for saving a json file that contains the detection
results in json format.
process_metrics_fn: a callback called with evaluation results after each
evaluation is done. It could be used e.g. to back up checkpoints with
best evaluation scores, or to call an external system to update evaluation
results in order to drive best hyper-parameter search. Parameters are:
int checkpoint_number, Dict[str, ObjectDetectionEvalMetrics] metrics,
str checkpoint_file path.
Returns:
global_step: the count of global steps.
all_evaluator_metrics: A dictionary containing metric names and values.
Raises:
ValueError: if restore_fn is None and checkpoint_dirs doesn't have at least
one element.
ValueError: if save_graph is True and save_graph_dir is not defined.
"""
if save_graph and not save_graph_dir:
raise ValueError('`save_graph_dir` must be defined.')
sess = tf.Session(master, graph=tf.get_default_graph())
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
sess.run(tf.tables_initializer())
checkpoint_file = None
if restore_fn:
restore_fn(sess)
else:
if not checkpoint_dirs:
raise ValueError('`checkpoint_dirs` must have at least one entry.')
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dirs[0])
saver = tf.train.Saver(variables_to_restore)
saver.restore(sess, checkpoint_file)
if save_graph:
tf.train.write_graph(sess.graph_def, save_graph_dir, 'eval.pbtxt')
counters = {'skipped': 0, 'success': 0}
aggregate_result_losses_dict = collections.defaultdict(list)
with tf.contrib.slim.queues.QueueRunners(sess):
try:
for batch in range(int(num_batches)):
if (batch + 1) % 100 == 0:
tf.logging.info('Running eval ops batch %d/%d', batch + 1,
num_batches)
if not batch_processor:
try:
if not losses_dict:
losses_dict = {}
result_dict, result_losses_dict = sess.run(
[tensor_dict, losses_dict])
counters['success'] += 1
except tf.errors.InvalidArgumentError:
tf.logging.info('Skipping image')
counters['skipped'] += 1
result_dict = {}
else:
result_dict, result_losses_dict = batch_processor(
tensor_dict,
sess,
batch,
counters,
losses_dict=losses_dict)
if not result_dict:
continue
for key, value in iter(result_losses_dict.items()):
aggregate_result_losses_dict[key].append(value)
for evaluator in evaluators:
# TODO(b/65130867): Use image_id tensor once we fix the input data
# decoders to return correct image_id.
# TODO(akuznetsa): result_dict contains batches of images, while
# add_single_ground_truth_image_info expects a single image. Fix
if (isinstance(result_dict, dict)
and fields.InputDataFields.key in result_dict
and result_dict[fields.InputDataFields.key]):
image_id = result_dict[fields.InputDataFields.key]
else:
image_id = batch
evaluator.add_single_ground_truth_image_info(
image_id=image_id, groundtruth_dict=result_dict)
evaluator.add_single_detected_image_info(
image_id=image_id, detections_dict=result_dict)
tf.logging.info('Running eval batches done.')
except tf.errors.OutOfRangeError:
tf.logging.info('Done evaluating -- epoch limit reached')
finally:
# When done, ask the threads to stop.
tf.logging.info('# success: %d', counters['success'])
tf.logging.info('# skipped: %d', counters['skipped'])
all_evaluator_metrics = {}
print('#' * 20, 'eval_export_path:', eval_export_path)
if eval_export_path and eval_export_path is not None:
for evaluator in evaluators:
print('#' * 20, 'evaluator:', evaluator)
if (isinstance(evaluator,
coco_evaluation.CocoDetectionEvaluator)
or isinstance(evaluator,
coco_evaluation.CocoMaskEvaluator)):
tf.logging.info('Started dumping to json file.')
evaluator.dump_detections_to_json_file(
json_output_path=eval_export_path)
tf.logging.info('Finished dumping to json file.')
for evaluator in evaluators:
metrics = evaluator.evaluate()
evaluator.clear()
if any(key in all_evaluator_metrics for key in metrics):
raise ValueError(
'Metric names between evaluators must not collide.')
all_evaluator_metrics.update(metrics)
global_step = tf.train.global_step(sess,
tf.train.get_global_step())
for key, value in iter(aggregate_result_losses_dict.items()):
all_evaluator_metrics['Losses/' + key] = np.mean(value)
if process_metrics_fn and checkpoint_file:
m = re.search(r'model.ckpt-(\d+)$', checkpoint_file)
if not m:
tf.logging.error(
'Failed to parse checkpoint number from: %s',
checkpoint_file)
else:
checkpoint_number = int(m.group(1))
process_metrics_fn(checkpoint_number,
all_evaluator_metrics, checkpoint_file)
sess.close()
return (global_step, all_evaluator_metrics)
def fsae_model_fn(mode, inputs, params, reuse=False):
"""Model ae function defining the graph operations
Args:
mode: (string) 'train', 'eval', etc.
inputs: (dict) contains the inputs of the graph (features, labels,...)
params: (dict) contains hyperparameters of the model (i.e. learning_rate,...)
reuse: (bool) whether to reuse Variables (weights, bias,...)
Returns:
model_spec: (dict) contains the graph operations or nodes needed for training / evaluation
"""
if mode == 'train':
is_training = True
else:
is_training = False
# -----------------------------------------------------------
# MODEL: define the layers of the model
with tf.variable_scope('fsae_model', reuse=reuse):
# Compute the output distribution of the model and the predictions
loss_likelihood, sampled, reconstructed_mean, sigma_placeholder = build_model(inputs, is_training, params)
with tf.variable_scope('feature_selection', reuse=reuse):
m = params.n_latent//2
help_p = tf.get_variable(name='temporary_p', shape=(params.n_latent, 1), initializer=tf.zeros_initializer)
y_t = tf.get_variable(name='y_t', shape=params.n_latent, initializer=tf.zeros_initializer)
P = tf.get_variable(name='projection_matrix', shape=(params.n_latent, m), initializer=tf.initializers.identity)
gamma = tf.get_variable(name='gamma', shape=(), initializer=tf.zeros_initializer)
z = inputs["img"]/tf.norm(inputs["img"], ord=1)
z_flat = tf.contrib.layers.flatten(z)
r = tf.reshape(tf.reduce_sum(z_flat*z_flat, 1), [-1, 1])
sim_original_img = r - 2*tf.matmul(z_flat, z_flat, transpose_b=True) + tf.transpose(r)
sum_anchor = tf.constant(15)
t = tf.constant(1.0)
inter_cluster_val, inter_cluster_idx = tf.nn.top_k(sim_original_img, k=sum_anchor)
min_inter_cluster_val = tf.reduce_min(inter_cluster_val, axis=1)
mask_less = tf.less(sim_original_img, min_inter_cluster_val)
sim_inter_cluster = sim_original_img*(tf.cast(mask_less, sim_original_img.dtype))
S_w = tf.exp(-sim_inter_cluster/t)
L_w = tf.diag_part(S_w) - S_w
betweeen_cluster_val, between_cluster_idx = tf.nn.top_k(-sim_original_img, k=sum_anchor)
max_betweeen_cluster_val = tf.reduce_max(-betweeen_cluster_val, axis=1)
mask_greater = tf.greater(sim_original_img, max_betweeen_cluster_val)
sim_between_cluster = sim_original_img*(tf.cast(mask_greater, sim_original_img.dtype))
S_b = tf.exp(-sim_between_cluster/t)
L_b = tf.diag_part(S_b) - S_b
# Optimization of trace-ratio problem
for i in range(params.n_latent):
help_p.assign(tf.zeros(help_p.get_shape()))
help_p[i].assign(tf.constant(1.0))
feature = tf.matmul(sampled, help_p)
y_t[i].assign(tf.trace(tf.matmul(tf.matmul(feature, L_w, transpose_a=True), feature)))/(tf.trace(tf.matmul(tf.matmul(feature, L_b, transpose_a=True), feature)))
_, leading_features = tf.nn.top_k(-y_t, k=m)
for i in range(m):
P.assign(tf.zeros(P.get_shape()))
P[leading_features[i], i].assign(tf.constant(1.0))
new_sample = tf.matmul(sampled, P)
gamma.assign(tf.trace(tf.matmul(tf.matmul(new_sample, L_w, transpose_a=True), new_sample)))/(tf.trace(tf.matmul(tf.matmul(new_sample, L_b, transpose_a=True), new_sample)))
# Define the Loss
lambd = tf.placeholder(tf.float32, [], name="lambda")
trace_loss_feature = tf.trace(tf.matmul(tf.matmul(new_sample, (L_w-gamma*L_b), transpose_a=True), new_sample))
loss = tf.reduce_mean(0.5*loss_likelihood + 0.5*lambd*trace_loss_feature)
# Define training step that minimizes the loss with the Adam optimizer
learning_rate_ph = tf.placeholder(tf.float32, [], name="learning_rate")
if mode == 'train':
optimizer = tf.train.AdamOptimizer(learning_rate_ph)
global_step = tf.train.get_or_create_global_step()
vars_ae_train_op = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "fsae_model")
vars_trace_ratio_train_op = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "feature_selection")
with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
train_op = optimizer.minimize(loss, global_step=global_step, var_list=vars_ae_train_op)
train_op_trace_ratio = optimizer.minimize(loss, global_step=global_step, var_list=vars_trace_ratio_train_op)
# -----------------------------------------------------------
# METRICS AND SUMMARIES
# Metrics for evaluation using tf.metrics (average over whole dataset)
with tf.variable_scope("fsae_metrics"):
metrics = {
'loss': tf.metrics.mean(loss),
'neg_log_likelihood': tf.metrics.mean(loss_likelihood)
}
# Group the update ops for the tf.metrics
update_metrics_op = tf.group(*[op for _, op in metrics.values()])
# Get the op to reset the local variables used in tf.metrics
metric_variables = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES, scope="fsae_metrics")
metrics_init_op = tf.variables_initializer(metric_variables)
# Summaries for training
tf.summary.scalar('loss', loss)
tf.summary.scalar('neg_log_likelihood', loss_likelihood)
# Summary for reconstruction and original image with max_outpus images
tf.summary.image('Original Image', inputs['img'], max_outputs=6, collections=None, family=None)
latent_img = tf.reshape(sampled, [-1, 1, params.n_latent, 1])
tf.summary.image('Latent Space', latent_img, max_outputs=6, collections=None, family=None)
latent_img_feature_selected = tf.reshape(new_sample, [-1, 1, m, 1])
tf.summary.image('Latent Space Feature Selected', latent_img_feature_selected, max_outputs=6, collections=None, family=None)
tf.summary.image('Reconstructions', tf.sigmoid(reconstructed_mean), max_outputs=6, collections=None, family=None)
# -----------------------------------------------------------
# -----------------------------------------------------------
# MODEL SPECIFICATION
# Create the model specification and return it
# It contains nodes or operations in the graph that will be used for training and evaluation
model_spec = inputs
variable_init_op = tf.group(*[tf.global_variables_initializer(), tf.tables_initializer()])
model_spec['variable_init_op'] = variable_init_op
model_spec['loss'] = loss
model_spec['metrics_init_op'] = metrics_init_op
model_spec['metrics'] = metrics
model_spec['update_metrics'] = update_metrics_op
model_spec['summary_op'] = tf.summary.merge_all()
model_spec['reconstructions'] = tf.sigmoid(reconstructed_mean)
model_spec['sigma_placeholder'] = sigma_placeholder
model_spec['learning_rate_placeholder'] = learning_rate_ph
model_spec['lambda_r_placeholder'] = lambd
model_spec['lambda_c_placeholder'] = tf.placeholder(tf.float32, [], name="lambda_c")
model_spec['lambda_d_placeholder'] = tf.placeholder(tf.float32, [], name="lambda_d")
model_spec['lambda_b_placeholder'] = tf.placeholder(tf.float32, [], name="lambda_b")
model_spec['lambda_w_placeholder'] = tf.placeholder(tf.float32, [], name="lambda_w")
model_spec['sigma_placeholder'] = tf.placeholder(tf.float32, [], name="sigma_ratio")
model_spec['gamma_placeholder'] = tf.placeholder(tf.float32, [], name="gamma")
if mode == 'train':
model_spec['train_op'] = train_op
model_spec['train_op_trace_ratio'] = train_op_trace_ratio
elif mode == 'cluster':
model_spec['sample'] = new_sample
return model_spec
def main(_):
# Pick up any one-off hyper-parameters.
hparams = lexnet_model.LexNETModel.default_hparams()
hparams.corpus = FLAGS.corpus
hparams.input = FLAGS.input
hparams.path_embeddings_file = 'path_embeddings/%s/%s' % (FLAGS.dataset,
FLAGS.corpus)
input_dir = hparams.input if hparams.input != 'path' else 'path_classifier'
# Set the number of classes
classes_filename = os.path.join(FLAGS.dataset_dir, FLAGS.dataset,
'classes.txt')
with open(classes_filename) as f_in:
classes = f_in.read().splitlines()
hparams.num_classes = len(classes)
print('Model will predict into %d classes' % hparams.num_classes)
# Get the datasets
train_set, val_set, test_set = (os.path.join(FLAGS.dataset_dir,
FLAGS.dataset, FLAGS.corpus,
filename + '.tfrecs.gz')
for filename in ['train', 'val', 'test'])
print('Running with hyper-parameters: {}'.format(hparams))
# Load the instances
print('Loading instances...')
opts = tf.python_io.TFRecordOptions(
compression_type=tf.python_io.TFRecordCompressionType.GZIP)
train_instances = list(tf.python_io.tf_record_iterator(train_set, opts))
val_instances = list(tf.python_io.tf_record_iterator(val_set, opts))
test_instances = list(tf.python_io.tf_record_iterator(test_set, opts))
# Load the word embeddings
print('Loading word embeddings...')
relata_embeddings, path_embeddings, nc_embeddings, path_to_index = (None,
None,
None,
None)
if hparams.input in ['dist', 'dist-nc', 'integrated', 'integrated-nc']:
relata_embeddings = lexnet_common.load_word_embeddings(
FLAGS.embeddings_base_path, hparams.relata_embeddings_file)
if hparams.input in ['path', 'integrated', 'integrated-nc']:
path_embeddings, path_to_index = path_model.load_path_embeddings(
os.path.join(FLAGS.embeddings_base_path,
hparams.path_embeddings_file), hparams.path_dim)
if hparams.input in ['dist-nc', 'integrated-nc']:
nc_embeddings = lexnet_common.load_word_embeddings(
FLAGS.embeddings_base_path, hparams.nc_embeddings_file)
# Define the graph and the model
with tf.Graph().as_default():
model = lexnet_model.LexNETModel(hparams, relata_embeddings,
path_embeddings, nc_embeddings,
path_to_index)
# Initialize a session and start training
session = tf.Session()
session.run(tf.global_variables_initializer())
# Initalize the path mapping
if hparams.input in ['path', 'integrated', 'integrated-nc']:
session.run(tf.tables_initializer())
session.run(model.initialize_path_op,
{model.path_initial_value_t: path_embeddings})
# Initialize the NC embeddings
if hparams.input in ['dist-nc', 'integrated-nc']:
session.run(model.initialize_nc_op,
{model.nc_initial_value_t: nc_embeddings})
# Load the labels
print('Loading labels...')
train_labels = model.load_labels(session, train_instances)
val_labels = model.load_labels(session, val_instances)
test_labels = model.load_labels(session, test_instances)
save_path = '{logdir}/results/{dataset}/{input}/{corpus}'.format(
logdir=FLAGS.logdir,
dataset=FLAGS.dataset,
corpus=model.hparams.corpus,
input=input_dir)
if not os.path.exists(save_path):
os.makedirs(save_path)
# Train the model
print('Training the model...')
model.fit(session, train_instances, epoch_completed, val_instances,
val_labels, save_path)
# Print the best performance on the validation set
print('Best performance on the validation set: F1=%.3f' %
epoch_completed.best_f1)
# Evaluate on the train and validation sets
lexnet_common.full_evaluation(model, session, train_instances,
train_labels, 'Train', classes)
lexnet_common.full_evaluation(model, session, val_instances,
val_labels, 'Validation', classes)
test_predictions = lexnet_common.full_evaluation(
model, session, test_instances, test_labels, 'Test', classes)
# Write the test predictions to a file
predictions_file = os.path.join(save_path, 'test_predictions.tsv')
print('Saving test predictions to %s' % save_path)
test_pairs = model.load_pairs(session, test_instances)
lexnet_common.write_predictions(test_pairs, test_labels,
test_predictions, classes,
predictions_file)
def initialize_vars(sess):
sess.run(tf.local_variables_initializer())
sess.run(tf.global_variables_initializer())
sess.run(tf.tables_initializer())
K.set_session(sess)
def run_embed(embed, headlines):
with tf.Session() as session:
session.run(
[tf.global_variables_initializer(),
tf.tables_initializer()])
return session.run(embed(headlines))
def __init__(self, *args, **kwargs):
super(TrainerTpu, self).__init__(*args, **kwargs)
# Multiple TPU trainer tasks not tested/implemented.
assert self._cluster.num_replicas == 1
data_parallelism = self._cluster.num_splits_per_client
assert data_parallelism
num_devices_per_split = self._cluster.num_devices_per_split
tf.logging.info('data_parallelism: %d, num_devices_per_split: %d',
data_parallelism, num_devices_per_split)
def ComputationShape(split_size):
"""Decides the computation shape based on the split_size."""
computation_shape = None
if split_size == 1:
computation_shape = [1, 1, 1]
elif split_size == 2:
computation_shape = [1, 1, 2]
elif split_size == 4:
computation_shape = [1, 2, 2]
elif split_size == 8:
computation_shape = [2, 2, 2]
elif split_size == 16:
computation_shape = [4, 2, 2]
else:
assert False, ('Model parallelism with %d devices is currently not'
' supported.' % split_size)
assert computation_shape is not None
return computation_shape
self._steps_per_loop = min(self.params.train.tpu_steps_per_loop,
self.params.train.max_steps)
tf.logging.info(
'Creating TrainerTpu using data parallelism %s '
'and %s steps_per_loop', data_parallelism, self._steps_per_loop)
@py_utils.RetryOnTransientTfError()
def _WaitTillInit():
"""Wait until the model is ready."""
try:
with self._GetSession() as sess:
topology = sess.run(
tf.contrib.tpu.initialize_system(embedding_config=None, job=None))
device_assignment = tf.contrib.tpu.device_assignment(
topology,
computation_shape=ComputationShape(num_devices_per_split),
num_replicas=data_parallelism)
py_utils.SetTpuDeviceAssignment(device_assignment)
tf.logging.info('device_assignment.core_assignment: %s',
str(device_assignment.core_assignment))
tf.logging.info('device_assignment.topology.device_coordinates: %s',
str(device_assignment.topology.device_coordinates))
except py_utils.transient_tf_errors as e:
tf.logging.info('TPU initialization failed: %s', e)
raise
_WaitTillInit()
with self._graph.as_default(), tf.container(self._container_id):
with self._cluster, tf.device(self._cluster.job_spec.name):
self._eval_metrics = metrics.TpuEvalMetrics()
def TpuTrainStep(*args):
self._model = self.params.cls(self.params)
self._model.ConstructFPropBPropGraph()
per_step_eval_metrics = self._eval_metrics.SetMetrics(
self._model.GetTask().eval_metrics, args)
summed_metrics = []
assert len(per_step_eval_metrics) == len(args)
for x, y in zip(per_step_eval_metrics, args):
summed_metrics.append(x + y)
return summed_metrics + [self._model.GetTask().train_op]
def TpuTrain():
loop_result = tf.contrib.tpu.repeat(
self._steps_per_loop,
TpuTrainStep,
inputs=self._eval_metrics.initial_values,
name='train_loop')
# Final metrics are the avg across self._steps_per_loop steps.
return self._eval_metrics.FinalizeMetrics(loop_result)
batch_parallel_res = tf.contrib.tpu.batch_parallel(
TpuTrain,
num_shards=data_parallelism,
device_assignment=py_utils.GetTpuDeviceAssignment())
# Get metric result from a single replica; they are all same here.
self._tpu_train_ops = [t[0] for t in batch_parallel_res]
self.initialize_tables = tf.tables_initializer()
self.enqueue_ops = tf.get_collection(py_utils.ENQUEUE_OPS)
assert not tf.get_collection(py_utils.CLOSE_QUEUE_OPS)
tf.logging.info('Trainer number of enqueue ops: %d',
len(self.enqueue_ops))
self._summary_writer = self._CreateSummaryWriter(self._train_dir)
# Saves the graph def.
tf.train.write_graph(self._graph.as_graph_def(), self._train_dir,
'train.pbtxt')
# hooks=[DecodeOnce({}, callback_func=_save_prediction_to_dict)])
_predictions = graph_utils.get_dict_from_collection("predictions")
fetches = {}
fetches["predicted_tokens"] = _predictions["predicted_tokens"]
fetches["predicted_ids"] = _predictions["predicted_ids"]
fetches["features.source_tokens"] = _predictions["features.source_tokens"]
fetches["features.source_candidate_tokens"] = _predictions[
"features.source_candidate_tokens"]
#fetches["beam_parent_ids"] = _predictions["beam_search_output.beam_parent_ids"]
tf.train.SessionRunArgs(fetches)
sess = tf.Session()
sess.run([
tf.global_variables_initializer(),
tf.local_variables_initializer(),
tf.tables_initializer()
])
saver.restore(sess, checkpoint_path)
print("start to export model:")
saved_model_path = "fin_biseq2seq_model"
if os.path.exists(saved_model_path):
print("removing old saved_model:" + saved_model_path)
shutil.rmtree(saved_model_path)
builder = tf.saved_model.builder.SavedModelBuilder(saved_model_path)
# Relevant change to the linked example is here!
builder.add_meta_graph_and_variables(sess, ["fin_biseq2seq"],
legacy_init_op=tf.tables_initializer())
builder.save()
print("finish")
def setup_char_to_id(self, sess):
_, _, chr_to_id = data_pipe.create_chr_dicts('./example_data/', chr(0),
chr(1))
sess.run(tf.tables_initializer())
return chr_to_id
def mainRun():
print("___Start!___" + datetime.now().strftime('%H:%M:%S') )
final = "_" ;
des = "MOD1"
#md.DESC = "MOD1";
# ALL_DS = md.LOGDAT + md.DESC + md.DSC
execc = get_models( des ) #md.DESC)
# -------------------------------------------------------------
# DATA READ
# -------------------------------------------------------------
# md.mainRead2(ALL_DS, 1, 2 ) # , all = True, shuffle = True )
# url_test = md.LOGDAT + "EXP1/" ; # url_test = "url"
# force = False; excel = True # dataFile = "frall2_json.txt"; labelFile = "datal.csv"
# md.get_tests(url_test, force, excel )
# -------------------------------------------------------------
# READ MODEL
# -------------------------------------------------------------
# md.get_columns(force) # for ex in execc:
ex = execc[2]
# md.spn = ex["spn"]; md.dType = ex["dt"]; mr.epochs = ex["e"]; mr.lr = ex["lr"]; mr.h = ex["h"]
# md.normalize()
# mr.ninp = 1814
# mr.ninp, mr.nout, mr.top_k = md.getnn(mr.ninp)
# md.MODEL_DIR = md.LOGDIR + md.DESC + '/' + mr.get_hpar(mr.epochs, final=final) +"/"
# mr.model_path = md.MODEL_DIR + "model.ckpt"
# mr.build_network3()
# print(mr.model_path)
# ex["pe"] = mr.evaluate( )
# ex["pt"] = mr.tests(url_test, p_col=False )
# -------------------------------------------------------------
# EXPORT MODEL
# -------------------------------------------------------------
# sess = tf.InteractiveSession()
with tf.Session() as sess:
# Restore the model from last checkpoints
# ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir)
# saver.restore(sess, ckpt.model_checkpoint_path)
sess.run(tf.global_variables_initializer())
# mr.restore_model(sess)
export_path = "./export/" #+ mr.get_hpar(mr.epochs, final=final)
print('Exporting trained model to', export_path)
builder = tf.saved_model.builder.SavedModelBuilder(export_path)
# Build the signature_def_map.
x = 1
classification_inputs = tf.saved_model.utils.build_tensor_info( x ) #mr.x)
# classification_outputs_classes = tf.saved_model.utils.build_tensor_info(prediction_classes)
# classification_outputs_scores = tf.saved_model.utils.build_tensor_info(mr.softmaxT)
predict_tensor_scores_info = tf.saved_model.utils.build_tensor_info( x ) #mr.prediction)
prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={ tf.saved_model.signature_constants.CLASSIFY_INPUTS: classification_inputs },
outputs={tf.saved_model.signature_constants.CLASSIFY_OUTPUT_CLASSES: predict_tensor_scores_info},
# outputs={'scores': predict_tensor_scores_info},
# outputs={
# tf.saved_model.signature_constants.CLASSIFY_OUTPUT_CLASSES: classification_outputs_classes,
# tf.saved_model.signature_constants.CLASSIFY_OUTPUT_SCORES: classification_outputs_scores
method_name= tf.saved_model.signature_constants.PREDICT_METHOD_NAME))
# - save...
legacy_init_op = tf.group(tf.tables_initializer(), name='legacy_init_op')
builder.add_meta_graph_and_variables(
sess,
[tf.saved_model.tag_constants.SERVING],
signature_def_map={ 'predict_val': prediction_signature },
legacy_init_op=legacy_init_op)
builder.save()
print('Done exporting!')
def model_fn(mode, inputs, params, reuse=False):
"""Model function defining the graph operations.
Args:
mode: (string) 'train', 'eval', etc.
inputs: (dict) contains the inputs of the graph (features, labels...)
this can be `tf.placeholder` or outputs of `tf.data`
params: (Params) contains hyperparameters of the model (ex: `params.learning_rate`)
reuse: (bool) whether to reuse the weights
Returns:
model_spec: (dict) contains the graph operations or nodes needed for training / evaluation
"""
is_training = (mode == 'train')
labels = inputs['labels']
sentence_lengths = inputs['sentence_lengths']
# -----------------------------------------------------------
# MODEL: define the layers of the model
with tf.variable_scope('model', reuse=reuse):
# Compute the output distribution of the model and the predictions
logits = build_model(mode, inputs, params, is_training)
predictions = tf.cast(tf.argmax(logits, -1), tf.int32)
#labels = tf.reshape(labels, [-1,1])
#predictions = [1 for i in logits > 0 else 0]
# Define loss and accuracy (we need to apply a mask to account for padding)
losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
labels=labels)
loss = tf.reduce_mean(losses)
accuracy = tf.reduce_mean(
tf.cast(tf.equal(labels, predictions), tf.float32))
# Define training step that minimizes the loss with the Adam optimizer
if is_training:
optimizer = tf.train.AdamOptimizer(params.learning_rate)
global_step = tf.train.get_or_create_global_step()
train_op = optimizer.minimize(loss, global_step=global_step)
# -----------------------------------------------------------
# METRICS AND SUMMARIES
# Metrics for evaluation using tf.metrics (average over whole dataset)
with tf.variable_scope("metrics"):
metrics = {
'accuracy': tf.metrics.accuracy(labels=labels,
predictions=predictions),
'loss': tf.metrics.mean(loss)
}
# Group the update ops for the tf.metrics
update_metrics_op = tf.group(*[op for _, op in metrics.values()])
# Get the op to reset the local variables used in tf.metrics
metric_variables = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES,
scope="metrics")
metrics_init_op = tf.variables_initializer(metric_variables)
# Summaries for training
tf.summary.scalar('loss', loss)
tf.summary.scalar('accuracy', accuracy)
# -----------------------------------------------------------
# MODEL SPECIFICATION
# Create the model specification and return it
# It contains nodes or operations in the graph that will be used for training and evaluation
model_spec = inputs
variable_init_op = tf.group(
*[tf.global_variables_initializer(),
tf.tables_initializer()])
model_spec['variable_init_op'] = variable_init_op
model_spec["predictions"] = predictions
model_spec['loss'] = loss
model_spec['accuracy'] = accuracy
model_spec['metrics_init_op'] = metrics_init_op
model_spec['metrics'] = metrics
model_spec['update_metrics'] = update_metrics_op
model_spec['summary_op'] = tf.summary.merge_all()
if is_training:
model_spec['train_op'] = train_op
return model_spec
def getFeatures(arrayOfTexts):
with tf.Session() as session:
session.run([tf.global_variables_initializer(), tf.tables_initializer()])
features = session.run(embed(arrayOfTexts))
return features
def classify():
# my_cats = ['rec.autos', 'soc.religion.christian', 'rec.sport.baseball', 'sci.electronics', 'sci.med']
# my_cats = ['rec.autos', 'soc.religion.christian']
# X_train, y_train = generate_data_set(my_cats, subset='train')
X_train = np.loadtxt('local_universal_encoder_data\\X_train.csv',
delimiter=',')
y_train = np.loadtxt('local_universal_encoder_data\\y_train.csv',
delimiter=',')
centroids = [['car', 'engine', 'drive', 'speed'],
['religion', 'jesus', 'god', 'believe', 'heaven', 'sin'],
[
'baseball', 'player', 'run', 'sport', 'hit', 'bat',
'rotation'
],
['electronics', 'conductive', 'power', 'resistor', 'circuit'],
['medical', 'methodology', 'science', 'molecule', 'virus']]
centroids_sents = [' '.join(centroid) for centroid in centroids]
# centroids = [['car', 'engine', 'drive', 'speed'],
# ['religion', 'jesus', 'god', 'believe', 'heaven', 'sin']]
#@param ["https://tfhub.dev/google/universal-sentence-encoder/2", "https://tfhub.dev/google/universal-sentence-encoder-large/3"]
module_url = "https://tfhub.dev/google/universal-sentence-encoder-large/3"
embed = hub.Module(module_url)
tf.logging.set_verbosity(tf.logging.ERROR)
with tf.Session() as session:
session.run(
[tf.global_variables_initializer(),
tf.tables_initializer()])
message_embeddings = session.run(embed(centroids_sents))
centroids_doc_vec_list = np.vstack(message_embeddings)
# centroids_doc_vec_list = [model.infer_vector(doc) for doc in centroids]
# X_train = np.loadtxt('custom_doc2vec_data\\X_train.csv', delimiter=',')
# y_train = np.loadtxt('custom_doc2vec_data\\y_train.csv', delimiter=',')
# dist = met.pairwise_distances(X= X_train,Y=list_centroids_vectors[0].reshape(1, -1),metric='cosine')
dist = met.pairwise_distances(X=X_train,
Y=np.vstack(centroids_doc_vec_list),
metric='cosine')
print(dist.shape)
indexes = np.argmin(dist, axis=1)
diff_list = (indexes - y_train).tolist()
diff = [1 if d == 0 else 0 for d in diff_list]
print('taxonomy-based semi supervised classification accuracy : {}'.format(
sum(diff) / len(diff)))
print(
f'f1 score of semi-supervised classification is {f1_score(y_train, indexes, average=None)}'
)
plt.plot(indexes, '.')
plt.title('semi-supervised classification using BERT embeddings')
plt.figure()
plt.plot(y_train)
plt.show()
def testGetIterator(self):
vocab_table = lookup_ops.index_table_from_tensor(
tf.constant(["a", "b", "c", "eos", "sos"]))
src_dataset = tf.contrib.data.Dataset.from_tensor_slices(
tf.constant(["c c a", "c a", "d", "f e a g"]))
tgt_dataset = tf.contrib.data.Dataset.from_tensor_slices(
tf.constant(["a b", "b c", "", "c c"]))
hparams = tf.contrib.training.HParams(
random_seed=3,
num_buckets=5,
source_reverse=False,
eos="eos",
sos="sos")
batch_size = 2
src_max_len = 3
iterator = iterator_utils.get_iterator(
src_dataset=src_dataset,
tgt_dataset=tgt_dataset,
vocab_table=vocab_table,
batch_size=batch_size,
sos=hparams.sos,
eos=hparams.eos,
src_reverse=hparams.source_reverse,
random_seed=hparams.random_seed,
num_buckets=hparams.num_buckets,
src_max_len=src_max_len)
table_initializer = tf.tables_initializer()
source = iterator.source
target_input = iterator.target_input
target_output = iterator.target_output
src_seq_len = iterator.source_sequence_length
tgt_seq_len = iterator.target_sequence_length
self.assertEqual([None, None], source.shape.as_list())
self.assertEqual([None, None], target_input.shape.as_list())
self.assertEqual([None, None], target_output.shape.as_list())
self.assertEqual([None], src_seq_len.shape.as_list())
self.assertEqual([None], tgt_seq_len.shape.as_list())
with self.test_session() as sess:
sess.run(table_initializer)
sess.run(iterator.initializer)
(source_v, src_len_v, target_input_v, target_output_v, tgt_len_v) = (
sess.run((source, src_seq_len, target_input, target_output,
tgt_seq_len)))
self.assertAllEqual(
[[-1, -1, 0], # "f" == unknown, "e" == unknown, a
[2, 0, 3]], # c a eos -- eos is padding
source_v)
self.assertAllEqual([3, 2], src_len_v)
self.assertAllEqual(
[[4, 2, 2], # sos c c
[4, 1, 2]], # sos b c
target_input_v)
self.assertAllEqual(
[[2, 2, 3], # c c eos
[1, 2, 3]], # b c eos
target_output_v)
self.assertAllEqual([3, 3], tgt_len_v)
(source_v, src_len_v, target_input_v, target_output_v, tgt_len_v) = (
sess.run((source, src_seq_len, target_input, target_output,
tgt_seq_len)))
self.assertAllEqual(
[[2, 2, 0]], # c c a
source_v)
self.assertAllEqual([3], src_len_v)
self.assertAllEqual(
[[4, 0, 1]], # sos a b
target_input_v)
self.assertAllEqual(
[[0, 1, 3]], # a b eos
target_output_v)
self.assertAllEqual([3], tgt_len_v)
with self.assertRaisesOpError("End of sequence"):
sess.run(source)
def initialize_session_variables(sess):
sess.run(
tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer(),
tf.tables_initializer()))
def main(args):
tf.logging.set_verbosity(tf.logging.INFO)
# Load configs
model_cls_list = [models.get_model(model) for model in args.models]
params_list = [default_parameters() for _ in range(len(model_cls_list))]
params_list = [
merge_parameters(params, model_cls.get_parameters())
for params, model_cls in zip(params_list, model_cls_list)
]
params_list = [
import_params(args.checkpoints[i], args.models[i], params_list[i])
for i in range(len(args.checkpoints))
]
params_list = [
override_parameters(params_list[i], args)
for i in range(len(model_cls_list))
]
# Build Graph
with tf.Graph().as_default():
model_var_lists = []
# Load checkpoints
for i, checkpoint in enumerate(args.checkpoints):
tf.logging.info("Loading %s" % checkpoint)
var_list = tf.train.list_variables(checkpoint)
values = {}
reader = tf.train.load_checkpoint(checkpoint)
for (name, shape) in var_list:
if not name.startswith(model_cls_list[i].get_name()):
continue
if name.find("losses_avg") >= 0:
continue
tensor = reader.get_tensor(name)
values[name] = tensor
model_var_lists.append(values)
# Build models
model_list = []
for i in range(len(args.checkpoints)):
name = model_cls_list[i].get_name()
model = model_cls_list[i](params_list[i], name + "_%d" % i)
model_list.append(model)
params = params_list[0]
# Read input file
sorted_keys, sorted_inputs = dataset.sort_input_file(args.input)
# Build input queue
features = dataset.get_inference_input(sorted_inputs, params)
# Create placeholders
placeholders = []
for i in range(len(params.device_list)):
placeholders.append({
"source": tf.placeholder(tf.int32, [None, None],
"source_%d" % i),
"source_length": tf.placeholder(tf.int32, [None],
"source_length_%d" % i)
})
# A list of outputs
if params.generate_samples:
inference_fn = sampling.create_sampling_graph
else:
inference_fn = inference.create_inference_graph
predictions = parallel.data_parallelism(
params.device_list, lambda f: inference_fn(model_list, f, params),
placeholders)
# Create assign ops
assign_ops = []
feed_dict = {}
all_var_list = tf.trainable_variables()
for i in range(len(args.checkpoints)):
un_init_var_list = []
name = model_cls_list[i].get_name()
for v in all_var_list:
if v.name.startswith(name + "_%d" % i):
un_init_var_list.append(v)
ops = set_variables(un_init_var_list, model_var_lists[i],
name + "_%d" % i, feed_dict)
assign_ops.extend(ops)
assign_op = tf.group(*assign_ops)
init_op = tf.tables_initializer()
results = []
tf.get_default_graph().finalize()
# Create session
with tf.Session(config=session_config(params)) as sess:
# Restore variables
sess.run(assign_op, feed_dict=feed_dict)
sess.run(init_op)
while True:
try:
feats = sess.run(features)
op, feed_dict = shard_features(feats, placeholders,
predictions)
results.append(sess.run(op, feed_dict=feed_dict))
message = "Finished batch %d" % len(results)
tf.logging.log(tf.logging.INFO, message)
except tf.errors.OutOfRangeError:
break
# Convert to plain text
vocab = params.vocabulary["target"]
outputs = []
scores = []
for result in results:
for item in result[0]:
outputs.append(item.tolist())
for item in result[1]:
scores.append(item.tolist())
outputs = list(itertools.chain(*outputs))
scores = list(itertools.chain(*scores))
restored_inputs = []
restored_outputs = []
restored_scores = []
for index in range(len(sorted_inputs)):
restored_inputs.append(sorted_inputs[sorted_keys[index]])
restored_outputs.append(outputs[sorted_keys[index]])
restored_scores.append(scores[sorted_keys[index]])
# Write to file
with open(args.output, "w") as outfile:
count = 0
for outputs, scores in zip(restored_outputs, restored_scores):
for output, score in zip(outputs, scores):
decoded = []
for idx in output:
if idx == params.mapping["target"][params.eos]:
break
decoded.append(vocab[idx])
decoded = " ".join(decoded)
if not args.verbose:
outfile.write("%s\n" % decoded)
break
else:
pattern = "%d ||| %s ||| %s ||| %f\n"
source = restored_inputs[count]
values = (count, source, decoded, score)
outfile.write(pattern % values)
count += 1
def main():
"""Entrypoint.
"""
train_data = tx.data.PairedTextData(hparams=config_data.train)
val_data = tx.data.PairedTextData(hparams=config_data.val)
test_data = tx.data.PairedTextData(hparams=config_data.test)
iterator = tx.data.FeedableDataIterator({
'train': train_data,
'val': val_data,
'test': test_data
})
batch = iterator.get_next()
outputs, sequence_length, infer_outputs = build_model(batch, train_data)
agent = tx.agents.SeqPGAgent(samples=outputs.sample_id,
logits=outputs.logits,
sequence_length=sequence_length,
hparams=config_model.agent)
def _train_and_eval(sess, agent):
iterator.restart_dataset(sess, 'train')
best_val_bleu = -1.
step = 0
while True:
try:
# Samples
extra_fetches = {
'truth': batch['target_text_ids'],
}
feed_dict = {
iterator.handle: iterator.get_handle(sess, 'train')
}
fetches = agent.get_samples(extra_fetches=extra_fetches,
feed_dict=feed_dict)
sample_text = tx.utils.map_ids_to_strs(fetches['samples'],
train_data.target_vocab,
strip_eos=False,
join=False)
truth_text = tx.utils.map_ids_to_strs(fetches['truth'],
train_data.target_vocab,
strip_eos=False,
join=False)
# Computes rewards
reward = []
for ref, hyp in zip(truth_text, sample_text):
r = tx.evals.sentence_bleu([ref], hyp, smooth=True)
reward.append(r)
# Updates
loss = agent.observe(reward=reward)
# Displays & evaluates
step += 1
if step == 1 or step % config_data.display == 0:
print("step={}, loss={:.4f}, reward={:.4f}".format(
step, loss, np.mean(reward)))
if step % config_data.display_eval == 0:
val_bleu = _eval_epoch(sess, 'val')
best_val_bleu = max(best_val_bleu, val_bleu)
print('val step={}, BLEU={:.4f}; best-ever={:.4f}'.format(
step, val_bleu, best_val_bleu))
test_bleu = _eval_epoch(sess, 'test')
print('test step={}, BLEU={:.4f}'.format(step, test_bleu))
print('=' * 50)
except tf.errors.OutOfRangeError:
break
def _eval_epoch(sess, mode):
"""`mode` is one of {'val', 'test'}
"""
iterator.restart_dataset(sess, mode)
refs, hypos = [], []
while True:
try:
fetches = [
batch['target_text'][:, 1:],
infer_outputs.predicted_ids[:, :, 0]
]
feed_dict = {
tx.global_mode(): tf.estimator.ModeKeys.PREDICT,
iterator.handle: iterator.get_handle(sess, mode)
}
target_texts, output_ids = \
sess.run(fetches, feed_dict=feed_dict)
target_texts = tx.utils.strip_special_tokens(target_texts)
output_texts = tx.utils.map_ids_to_strs(
ids=output_ids, vocab=val_data.target_vocab)
for hypo, ref in zip(output_texts, target_texts):
hypos.append(hypo)
refs.append([ref])
except tf.errors.OutOfRangeError:
break
return tx.evals.corpus_bleu_moses(list_of_references=refs,
hypotheses=hypos)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
sess.run(tf.tables_initializer())
agent.sess = sess
_train_and_eval(sess, agent)
y = pad_sequences(maxlen=max_len,
sequences=y,
padding="post",
value=tag2idx["O"])
batch_size = 2
import tensorflow as tf
import tensorflow_hub as hub
from keras import backend as K
sess = tf.Session()
K.set_session(sess)
elmo_model = hub.Module("https://tfhub.dev/google/elmo/2", trainable=True)
sess.run(tf.global_variables_initializer())
sess.run(tf.tables_initializer())
def ElmoEmbedding(x):
return elmo_model(inputs={
"tokens": tf.squeeze(tf.cast(x, tf.string)),
"sequence_len": tf.constant(batch_size * [max_len])
},
signature="tokens",
as_dict=True)["elmo"]
from keras.models import Model, Input
from keras.layers.merge import add
from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional, Lambda
def train(self, result_dir, target=""):
"""
Train a seq2seq model.
Args:
result_dir: Directory where we are going to store our results
target: Target of the session
"""
# Summary writer
summary_name = "train_log"
summary_writer = tf.summary.FileWriter(
os.path.join(result_dir, summary_name), self.graph)
log_device_placement = self.hparams.log_device_placement
# Number of epochs of the train
num_epochs = self.hparams.num_epochs
config_proto = tf.ConfigProto(
log_device_placement=log_device_placement,
allow_soft_placement=True)
config_proto.gpu_options.allow_growth = True
with tf.Session(target=target, config=config_proto,
graph=self.graph) as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.tables_initializer())
global_step = self.model.global_step.eval(session=sess)
# Initialize all of the iterators
sess.run(self.train_batch.initializer)
# Initialize the statistic variables
ckpt_loss, ckpt_predict_count = 0.0, 0.0
train_perp, last_record_perp = 2000.0, 2.0
train_epoch = 0
print("# Training loop started @ {}".format(
time.strftime("%Y-%m-%d %H:%M:%S")))
epoch_start_time = time.time()
while train_epoch < num_epochs:
# Each run of this while loop is a training step, multiple time/steps will trigger
# the train_epoch to be increased.
# Parameter of the algorithm - Based on the perplexity
learning_rate = self._get_learning_rate(train_perp)
try:
step_result = self.model.train_step(
sess, learning_rate=learning_rate)
(_, step_loss, step_predict_count, step_summary,
global_step, step_word_count, batch_size) = step_result
# Write step summary.
summary_writer.add_summary(step_summary, global_step)
# update statistics
ckpt_loss += (step_loss * batch_size)
ckpt_predict_count += step_predict_count
except tf.errors.OutOfRangeError:
# Finished going through the training dataset. Go to next epoch.
train_epoch += 1
# Average of the loss, calculated to check it
mean_loss = ckpt_loss / ckpt_predict_count
# Perplexity of the epoch -> How well a probability distribution or probability model predicts a sample
train_perp = math.exp(
float(mean_loss)) if mean_loss < 300 else math.inf
epoch_dur = time.time() - epoch_start_time
print(
"# Finished epoch {:2d} @ step {:5d} @ {}. In the epoch, learning rate = {:.6f}, "
"mean loss = {:.4f}, perplexity = {:8.4f}, and {:.2f} seconds elapsed."
.format(train_epoch, global_step,
time.strftime("%Y-%m-%d %H:%M:%S"),
learning_rate, mean_loss, train_perp,
round(epoch_dur, 2)))
epoch_start_time = time.time(
) # The start time of the next epoch
summary = tf.Summary(value=[
tf.Summary.Value(tag="train_perp",
simple_value=train_perp)
])
summary_writer.add_summary(summary, global_step)
# Save checkpoint
if train_perp < 1.6 and train_perp < last_record_perp:
self.model.saver.save(sess,
os.path.join(
result_dir, "basic"),
global_step=global_step)
last_record_perp = train_perp
ckpt_loss, ckpt_predict_count = 0.0, 0.0
sess.run(self.model.batch_input.initializer)
continue
# Done training
self.model.saver.save(sess,
os.path.join(result_dir, "basic"),
global_step=global_step)
summary_writer.close()
