def _test(self, kwargs, expected_values=None, expected_err=None):
with self.test_session() as sess:
if expected_err:
with self.assertRaisesWithPredicateMatch(
expected_err[0], expected_err[1]):
out = tf.parse_example(**kwargs)
sess.run(flatten_values_tensors_or_sparse(out.values()))
else:
# Returns dict w/ Tensors and SparseTensors.
out = tf.parse_example(**kwargs)
result = flatten_values_tensors_or_sparse(out.values())
# Check values.
tf_result = sess.run(result)
_compare_output_to_expected(self, out, expected_values, tf_result)
# Check shapes; if serialized is a Tensor we need its size to
# properly check.
serialized = kwargs["serialized"]
batch_size = (serialized.eval().size if isinstance(serialized, tf.Tensor)
else np.asarray(serialized).size)
for k, f in kwargs["features"].items():
if isinstance(f, tf.FixedLenFeature) and f.shape is not None:
self.assertEqual(
tuple(out[k].get_shape().as_list()), (batch_size,) + f.shape)
elif isinstance(f, tf.VarLenFeature):
self.assertEqual(
tuple(out[k].indices.get_shape().as_list()), (None, 2))
self.assertEqual(tuple(out[k].values.get_shape().as_list()), (None,))
self.assertEqual(tuple(out[k].shape.get_shape().as_list()), (2,))
def _generate_saved_model_for_half_plus_two(export_dir, as_text=False):
"""Generates SavedModel for half plus two.
Args:
export_dir: The directory to which the SavedModel should be written.
as_text: Writes the SavedModel protocol buffer in text format to disk.
"""
builder = saved_model_builder.SavedModelBuilder(export_dir)
with tf.Session(graph=tf.Graph()) as sess:
# Set up the model parameters as variables to exercise variable loading
# functionality upon restore.
a = tf.Variable(0.5, name="a")
b = tf.Variable(2.0, name="b")
# Create a placeholder for serialized tensorflow.Example messages to be fed.
serialized_tf_example = tf.placeholder(tf.string, name="tf_example")
# Parse the tensorflow.Example looking for a feature named "x" with a single
# floating point value.
feature_configs = {"x": tf.FixedLenFeature([1], dtype=tf.float32),}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
# Use tf.identity() to assign name
x = tf.identity(tf_example["x"], name="x")
y = tf.add(tf.mul(a, x), b, name="y")
# Create an assets file that can be saved and restored as part of the
# SavedModel.
original_assets_directory = "/tmp/original/export/assets"
original_assets_filename = "foo.txt"
original_assets_filepath = _write_assets(original_assets_directory,
original_assets_filename)
# Set up the assets collection.
assets_filepath = tf.constant(original_assets_filepath)
tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, assets_filepath)
# Set up the signature for regression with input and output tensor
# specification.
input_tensor = meta_graph_pb2.TensorInfo()
input_tensor.name = serialized_tf_example.name
signature_inputs = {signature_constants.REGRESS_INPUTS: input_tensor}
output_tensor = meta_graph_pb2.TensorInfo()
output_tensor.name = tf.identity(y).name
signature_outputs = {signature_constants.REGRESS_OUTPUTS: output_tensor}
signature_def = utils.build_signature_def(
signature_inputs, signature_outputs,
signature_constants.REGRESS_METHOD_NAME)
# Initialize all variables and then save the SavedModel.
sess.run(tf.initialize_all_variables())
builder.add_meta_graph_and_variables(
sess, [constants.TAG_SERVING],
signature_def_map={
signature_constants.REGRESS_METHOD_NAME:
signature_def
},
assets_collection=tf.get_collection(tf.GraphKeys.ASSET_FILEPATHS))
builder.save(as_text)
def _test(self, kwargs, expected_values=None, expected_err_re=None):
with self.test_session() as sess:
# Pull out some keys to check shape inference
serialized = kwargs["serialized"]
dense_keys = kwargs["dense_keys"] if "dense_keys" in kwargs else []
sparse_keys = kwargs["sparse_keys"] if "sparse_keys" in kwargs else []
dense_shapes = kwargs["dense_shapes"] if "dense_shapes" in kwargs else []
# Returns dict w/ Tensors and SparseTensors
out = tf.parse_example(**kwargs)
# Check shapes; if serialized is a Tensor we need its size to
# properly check.
batch_size = (
serialized.eval().size if isinstance(serialized, tf.Tensor)
else np.asarray(serialized).size)
if dense_shapes:
self.assertEqual(len(dense_keys), len(dense_shapes))
for (k, s) in zip(dense_keys, dense_shapes):
self.assertEqual(
tuple(out[k].get_shape().as_list()), (batch_size,) + s)
for k in sparse_keys:
self.assertEqual(
tuple(out[k].indices.get_shape().as_list()), (None, 2))
self.assertEqual(tuple(out[k].values.get_shape().as_list()), (None,))
self.assertEqual(tuple(out[k].shape.get_shape().as_list()), (2,))
# Check values
result = flatten_values_tensors_or_sparse(out.values()) # flatten values
if expected_err_re is None:
tf_result = sess.run(result)
_compare_output_to_expected(self, out, expected_values, tf_result)
else:
with self.assertRaisesOpError(expected_err_re):
sess.run(result)
def _deserialize_train(examples_serialized):
features = tf.parse_example(examples_serialized, _FEATURE_MAP)
train_features = {
movielens.USER_COLUMN: features[movielens.USER_COLUMN],
movielens.ITEM_COLUMN: features[movielens.ITEM_COLUMN],
}
return train_features, features[movielens.RATING_COLUMN]
def parse_example_batch(serialized):
"""Parses a batch of tf.Example protos.
Args:
serialized: A 1-D string Tensor; a batch of serialized tf.Example protos.
Returns:
encode: A SentenceBatch of encode sentences.
decode_pre: A SentenceBatch of "previous" sentences to decode.
decode_post: A SentenceBatch of "post" sentences to decode.
"""
features = tf.parse_example(
serialized,
features={
"encode": tf.VarLenFeature(dtype=tf.int64),
"decode_pre": tf.VarLenFeature(dtype=tf.int64),
"decode_post": tf.VarLenFeature(dtype=tf.int64),
})
def _sparse_to_batch(sparse):
ids = tf.sparse_tensor_to_dense(sparse) # Padding with zeroes.
mask = tf.sparse_to_dense(sparse.indices, sparse.dense_shape,
tf.ones_like(sparse.values, dtype=tf.int32))
return SentenceBatch(ids=ids, mask=mask)
output_names = ("encode", "decode_pre", "decode_post")
return tuple(_sparse_to_batch(features[x]) for x in output_names)
def batch_parse_tf_example(batch_size, example_batch):
'''
Args:
example_batch: a batch of tf.Example
Returns:
A dict of batched tensors
'''
features = {
'x': tf.FixedLenFeature([], tf.string),
'pi': tf.FixedLenFeature([], tf.string),
'outcome': tf.FixedLenFeature([], tf.float32),
}
parsed = tf.parse_example(example_batch, features)
x = tf.decode_raw(parsed['x'], tf.uint8)
x = tf.cast(x, tf.float32)
x = tf.reshape(x, [batch_size, go.N, go.N,
features_lib.NEW_FEATURES_PLANES])
pi = tf.decode_raw(parsed['pi'], tf.float32)
pi = tf.reshape(pi, [batch_size, go.N * go.N + 1])
outcome = parsed['outcome']
outcome.set_shape([batch_size])
return {
'pos_tensor': x,
'pi_tensor': pi,
'value_tensor': outcome,
}
def build_prediction_graph(self):
"""Builds prediction graph and registers appropriate endpoints."""
examples = tf.placeholder(tf.string, shape=(None,))
features = {
'image': tf.FixedLenFeature(
shape=[IMAGE_PIXELS], dtype=tf.float32),
'key': tf.FixedLenFeature(
shape=[], dtype=tf.string),
}
parsed = tf.parse_example(examples, features)
images = parsed['image']
keys = parsed['key']
# Build a Graph that computes predictions from the inference model.
logits = inference(images, self.hidden1, self.hidden2)
softmax = tf.nn.softmax(logits)
prediction = tf.argmax(softmax, 1)
# Mark the inputs and the outputs
# Marking the input tensor with an alias with suffix _bytes. This is to
# indicate that this tensor value is raw bytes and will be base64 encoded
# over HTTP.
# Note that any output tensor marked with an alias with suffix _bytes, shall
# be base64 encoded in the HTTP response. To get the binary value, it
# should be base64 decoded.
tf.add_to_collection('inputs',
json.dumps({'examples_bytes': examples.name}))
tf.add_to_collection('outputs',
json.dumps({
'key': keys.name,
'prediction': prediction.name,
'scores': softmax.name
}))
def parse_examples(examples):
feature_map = {
'labels': tf.FixedLenFeature(
shape=[], dtype=tf.int64, default_value=[-1]),
'images': tf.FixedLenFeature(
shape=[IMAGE_PIXELS], dtype=tf.float32),
}
return tf.parse_example(examples, features=feature_map)
def parse_examples(example_protos):
features = {
"target": tf.FixedLenFeature(shape=[1], dtype=tf.float32, default_value=0),
"age_indices": tf.VarLenFeature(dtype=tf.int64),
"age_values": tf.VarLenFeature(dtype=tf.float32),
"gender_indices": tf.VarLenFeature(dtype=tf.int64),
"gender_values": tf.VarLenFeature(dtype=tf.float32),
}
return tf.parse_example([e.SerializeToString() for e in example_protos], features)
def testBasic(self):
golden_config = example_parser_configuration_pb2.ExampleParserConfiguration()
text_format.Parse(BASIC_PROTO, golden_config)
with tf.Session() as sess:
examples = tf.placeholder(tf.string, shape=[1])
feature_to_type = {"x": tf.FixedLenFeature([1], tf.float32, 33.0), "y": tf.VarLenFeature(tf.string)}
_ = tf.parse_example(examples, feature_to_type)
parse_example_op = sess.graph.get_operation_by_name("ParseExample/ParseExample")
config = extract_example_parser_configuration(parse_example_op, sess)
self.assertProtoEquals(golden_config, config)
def build_prediction_graph(self):
"""Builds prediction graph and registers appropriate endpoints."""
examples = tf.placeholder(tf.string, shape=(None,))
features = {
'image': tf.FixedLenFeature(
shape=[IMAGE_PIXELS], dtype=tf.float32),
'key': tf.FixedLenFeature(
shape=[], dtype=tf.string),
}
parsed = tf.parse_example(examples, features)
images = parsed['image']
keys = parsed['key']
# Build a Graph that computes predictions from the inference model.
logits = inference(images, self.hidden1, self.hidden2)
softmax = tf.nn.softmax(logits)
prediction = tf.argmax(softmax, 1)
# Mark the inputs and the outputs
# Marking the input tensor with an alias with suffix _bytes. This is to
# indicate that this tensor value is raw bytes and will be base64 encoded
# over HTTP.
# Note that any output tensor marked with an alias with suffix _bytes, shall
# be base64 encoded in the HTTP response. To get the binary value, it
# should be base64 decoded.
input_signatures = {}
predict_input_tensor = meta_graph_pb2.TensorInfo()
predict_input_tensor.name = examples.name
predict_input_tensor.dtype = examples.dtype.as_datatype_enum
input_signatures['example_bytes'] = predict_input_tensor
tf.add_to_collection('inputs',
json.dumps({
'examples_bytes': examples.name
}))
tf.add_to_collection('outputs',
json.dumps({
'key': keys.name,
'prediction': prediction.name,
'scores': softmax.name
}))
output_signatures = {}
outputs_dict = {'key': keys.name,
'prediction': prediction.name,
'scores': softmax.name}
for key, val in outputs_dict.iteritems():
predict_output_tensor = meta_graph_pb2.TensorInfo()
predict_output_tensor.name = val
for placeholder in [keys, prediction, softmax]:
if placeholder.name == val:
predict_output_tensor.dtype = placeholder.dtype.as_datatype_enum
output_signatures[key] = predict_output_tensor
return input_signatures, output_signatures
def example_serving_input_fn():
"""Build the serving inputs."""
example_bytestring = tf.placeholder(
shape=[None],
dtype=tf.string,
)
features = tf.parse_example(
example_bytestring,
tf.feature_column.make_parse_example_spec(featurizer.INPUT_COLUMNS))
return tf.estimator.export.ServingInputReceiver(
features, {'example_proto': example_bytestring})
def decode(self,batched_serialized_tensors,batch_size):
"""Decodes the input from batch of serialized tensors
Formats and reshapes image
Args:
batched_serialized_tensors: tensor output from Batcher containing read in
serialized tensors
Returns:
batched_decoded_tensors: dict of batches of decoded TFRecords of batch_size
"""
#faster to decode tensors as a batch
batched_decoded_tensors = tf.parse_example(batched_serialized_tensors[fields.InputDataFields.serialized],
self._keys_to_features)
#Decode and cast tensors if needed
for label in self._multi_task_labels:
tensor = batched_decoded_tensors[label.name]
#only strings need t obe decoded
if label.dtype == "string":
if label.decodetype:
tensor = tf.decode_raw(tensor, TYPE_MAP[label.decodetype])
else:
raise ValueError("string type must have a type to be decoded to.")
if label.casttype:
tensor = tf.cast(tensor, TYPE_MAP[label.casttype])
if label.shape:
tensor = tf.reshape(tensor, [batch_size,*label.shape])
tensor.set_shape([batch_size, *label.shape])
batched_decoded_tensors[label.name] = tensor
#input is handlded separately
image_float = tf.cast(
tf.decode_raw(batched_decoded_tensors['input'],
tf.uint8),
tf.float32)
image_float = tf.reshape(image_float,[batch_size,
self._image_height,
self._image_width,
self._channels])
image_float.set_shape([batch_size,
self._image_height,
self._image_width,
self._channels])
batched_decoded_tensors['input'] = image_float
return batched_decoded_tensors
def load_all_labels(records):
"""Reads TensorFlow examples from a RecordReader and returns only the labels.
Args:
records: a record list with TensorFlow examples.
Returns:
The labels
"""
curr_features = tf.parse_example(records, {
'rel_id': tf.FixedLenFeature([1], dtype=tf.int64),
})
labels = tf.squeeze(curr_features['rel_id'], [-1])
return labels
def load_all_pairs(records):
"""Reads TensorFlow examples from a RecordReader and returns the word pairs.
Args:
records: a record list with TensorFlow examples.
Returns:
The word pairs
"""
curr_features = tf.parse_example(records, {
'pair': tf.FixedLenFeature([1], dtype=tf.string)
})
word_pairs = curr_features['pair']
return word_pairs
def serving_input_fn():
with tf.name_scope("inputs"):
serialized = tf.placeholder(
dtype=tf.string,
shape=tf.tensor_shape.unknown_shape(ndims=1),
name=EXAMPLES_KEY)
parsing_spec = {TERMS_KEY: tf.VarLenFeature(dtype=tf.string)}
features = tf.parse_example(serialized, parsing_spec)
sequence_length = sparse_sequence_length(features[TERMS_KEY])
features[SEQUENCE_LENGTH_KEY] = sequence_length
return tf.contrib.learn.InputFnOps(
features=features,
labels=None,
default_inputs={EXAMPLES_KEY: serialized})
def LoadBinaryCode(input_config, batch_size):
"""Load a batch of binary codes from a tf.Example dataset.
Args:
input_config: An InputConfig proto containing the input configuration.
batch_size: Output batch size of examples.
Returns:
A batched tensor of binary codes.
"""
data = input_config.data
# TODO: Possibly use multiple files (instead of just one).
file_list = [data]
filename_queue = tf.train.string_input_producer(file_list,
capacity=4)
reader = tf.TFRecordReader()
_, values = reader.read(filename_queue)
serialized_example = tf.reshape(values, shape=[1])
serialized_features = {
'code_shape': tf.FixedLenFeature([3],
dtype=tf.int64),
'code': tf.VarLenFeature(tf.float32),
}
example = tf.parse_example(serialized_example, serialized_features)
# 3D shape: height x width x binary_code_depth
z = example['code_shape']
code_shape = tf.reshape(tf.cast(z, tf.int32), [3])
# Un-flatten the binary codes.
code = tf.reshape(tf.sparse_tensor_to_dense(example['code']), code_shape)
queue_size = 10
queue = tf.PaddingFIFOQueue(
queue_size + 3 * batch_size,
dtypes=[code.dtype],
shapes=[[None, None, None]])
enqueue_op = queue.enqueue([code])
dequeue_code = queue.dequeue_many(batch_size)
queue_runner = tf.train.queue_runner.QueueRunner(queue, [enqueue_op])
tf.add_to_collection(tf.GraphKeys.QUEUE_RUNNERS, queue_runner)
return dequeue_code
def _generate_saved_model_for_half_plus_two(export_dir, as_text=False):
"""Generates SavedModel for half plus two.
Args:
export_dir: The directory to which the SavedModel should be written.
as_text: Writes the SavedModel protocol buffer in text format to disk.
"""
builder = saved_model_builder.SavedModelBuilder(export_dir)
with tf.Session(graph=tf.Graph()) as sess:
# Set up the model parameters as variables to exercise variable loading
# functionality upon restore.
a = tf.Variable(0.5, name="a")
b = tf.Variable(2.0, name="b")
# Create a placeholder for serialized tensorflow.Example messages to be fed.
serialized_tf_example = tf.placeholder(tf.string, name="tf_example")
# Parse the tensorflow.Example looking for a feature named "x" with a single
# floating point value.
feature_configs = {"x": tf.FixedLenFeature([1], dtype=tf.float32),}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
# Use tf.identity() to assign name
x = tf.identity(tf_example["x"], name="x")
y = tf.add(tf.mul(a, x), b, name="y")
# Set up the signature for regression with input and output tensor
# specification.
input_tensor = meta_graph_pb2.TensorInfo()
input_tensor.name = serialized_tf_example.name
signature_inputs = {"input": input_tensor}
output_tensor = meta_graph_pb2.TensorInfo()
output_tensor.name = tf.identity(y).name
signature_outputs = {"output": output_tensor}
signature_def = utils.build_signature_def(signature_inputs,
signature_outputs, "regression")
# Initialize all variables and then save the SavedModel.
sess.run(tf.initialize_all_variables())
builder.add_meta_graph_and_variables(
sess, [constants.TAG_SERVING],
signature_def_map={"regression": signature_def})
builder.save(as_text)
def example_serving_input_fn(default_batch_size=None):
"""Build the serving inputs.
Args:
default_batch_size (int): Batch size for the tf.placeholder shape
"""
feature_spec = {}
for feat in CONTINUOUS_COLS:
feature_spec[feat] = tf.FixedLenFeature(shape=[], dtype=tf.int64)
for feat, _ in CATEGORICAL_COLS:
feature_spec[feat] = tf.FixedLenFeature(shape=[], dtype=tf.string)
example_bytestring = tf.placeholder(
shape=[default_batch_size],
dtype=tf.string,
)
features = tf.parse_example(example_bytestring, feature_spec)
return features, {'example': example_bytestring}
def _eval_input_receiver_fn(transform_output, schema):
"""Build everything needed for the tf-model-analysis to run the model.
Args:
transform_output: directory in which the tf-transform model was written
during the preprocessing step.
schema: the schema of the input data.
Returns:
EvalInputReceiver function, which contains:
- Tensorflow graph which parses raw untransformed features, applies the
tf-transform preprocessing operators.
- Set of raw, untransformed features.
- Label against which predictions will be compared.
"""
# Notice that the inputs are raw features, not transformed features here.
raw_feature_spec = _get_raw_feature_spec(schema)
serialized_tf_example = tf.placeholder(
dtype=tf.string, shape=[None], name='input_example_tensor')
# Add a parse_example operator to the tensorflow graph, which will parse
# raw, untransformed, tf examples.
features = tf.parse_example(serialized_tf_example, raw_feature_spec)
# Now that we have our raw examples, process them through the tf-transform
# function computed during the preprocessing step.
_, transformed_features = (
saved_transform_io.partially_apply_saved_transform(
os.path.join(transform_output, transform_fn_io.TRANSFORM_FN_DIR),
features))
# The key name MUST be 'examples'.
receiver_tensors = {'examples': serialized_tf_example}
# NOTE: Model is driven by transformed features (since training works on the
# materialized output of TFT, but slicing will happen on raw features.
features.update(transformed_features)
return tfma.export.EvalInputReceiver(
features=features,
receiver_tensors=receiver_tensors,
labels=transformed_features[_transformed_name(_LABEL_KEY)])
def example_evaluating_input_receiver_fn():
"""Creating an EvalInputReceiver object for TFRecords data.
Returns:
EvalInputReceiver
"""
tf_example = tf.placeholder(shape=[None], dtype=tf.string)
features = tf.parse_example(
tf_example,
features=get_feature_spec(is_serving=False))
for key in features:
features[key] = tf.expand_dims(features[key], -1)
return tfma.export.EvalInputReceiver(
features=process_features(features),
receiver_tensors={'examples': tf_example},
labels=features[metadata.TARGET_NAME])
def _serialize_deserialize(self, num_cores=1, num_rows=20):
np.random.seed(1)
df = pd.DataFrame({
# Serialization order is only deterministic for num_cores=1. raw_row is
# used in validation after the deserialization.
_RAW_ROW: np.array(range(num_rows), dtype=np.int64),
_DUMMY_COL: np.random.randint(0, 35, size=(num_rows,)),
_DUMMY_VEC_COL: [
np.array([np.random.random() for _ in range(_DUMMY_VEC_LEN)])
for i in range(num_rows) # pylint: disable=unused-variable
]
})
with fixed_core_count(num_cores):
buffer_path = file_io.write_to_temp_buffer(
df, self.get_temp_dir(), [_RAW_ROW, _DUMMY_COL, _DUMMY_VEC_COL])
with self.test_session(graph=tf.Graph()) as sess:
dataset = tf.data.TFRecordDataset(buffer_path)
dataset = dataset.batch(1).map(
lambda x: tf.parse_example(x, _FEATURE_MAP))
data_iter = dataset.make_one_shot_iterator()
seen_rows = set()
for i in range(num_rows+5):
row = data_iter.get_next()
try:
row_id, val_0, val_1 = sess.run(
[row[_RAW_ROW], row[_DUMMY_COL], row[_DUMMY_VEC_COL]])
row_id, val_0, val_1 = row_id[0][0], val_0[0][0], val_1[0]
assert row_id not in seen_rows
seen_rows.add(row_id)
self.assertEqual(val_0, df[_DUMMY_COL][row_id])
self.assertAllClose(val_1, df[_DUMMY_VEC_COL][row_id])
self.assertLess(i, num_rows, msg="Too many rows.")
except tf.errors.OutOfRangeError:
self.assertGreaterEqual(i, num_rows, msg="Too few rows.")
file_io._GARBAGE_COLLECTOR.purge()
assert not tf.gfile.Exists(buffer_path)
def prepare_serialized_examples(self, serialized_examples):
# set the mapping from the fields to data types in the proto
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {"id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)}
for feature_index in range(num_features):
feature_map[self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
labels.set_shape([None, self.num_classes])
concatenated_features = tf.concat([
features[feature_name] for feature_name in self.feature_names], 1)
return features["id"], concatenated_features, labels, tf.ones([tf.shape(serialized_examples)[0]])
def read_batch(file_name, max_epoch, batch_size, thread_num, min_after_dequeue):
with tf.name_scope("input"):
filename_queue = tf.train.string_input_producer(
tf.train.match_filenames_once(file_name),
num_epochs=max_epoch)
serialized_example = read_and_decode(filename_queue)
capacity = thread_num * batch_size + min_after_dequeue
batch_serialized_example = tf.train.shuffle_batch(
[serialized_example],
batch_size=batch_size,
num_threads=thread_num,
capacity=capacity,
min_after_dequeue=min_after_dequeue)
features = tf.parse_example(
batch_serialized_example,
features={
"label": tf.FixedLenFeature([], tf.float32),
"ids": tf.FixedLenFeature([39], tf.int64),
"values": tf.FixedLenFeature([39], tf.float32),
})
return features["label"], features["ids"], features["values"]
def example_serving_input_receiver_fn():
"""Creating an ServingInputReceiver object for TFRecords data.
Returns:
ServingInputReceiver
"""
# Note that the inputs are raw features, not transformed features.
receiver_tensors = tf.placeholder(shape=[None], dtype=tf.string)
features = tf.parse_example(
receiver_tensors,
features=get_feature_spec(is_serving=True)
)
for key in features:
features[key] = tf.expand_dims(features[key], -1)
return tf.estimator.export.ServingInputReceiver(
features=process_features(features),
receiver_tensors={'example_proto': receiver_tensors}
)
def build_graph(self, data_paths, batch_size, graph_mod):
"""Builds generic graph for training or eval."""
tensors = GraphReferences()
is_training = graph_mod == GraphMod.TRAIN
if data_paths:
tensors.keys, tensors.examples = util.read_examples(
data_paths,
batch_size,
shuffle=is_training,
num_epochs=None if is_training else 2)
else:
tensors.examples = tf.placeholder(tf.string, name='input', shape=(None,))
if graph_mod == GraphMod.PREDICT:
inception_input, inception_embeddings = self.build_inception_graph()
# Build the Inception graph. We later add final training layers
# to this graph. This is currently used only for prediction.
# For training, we use pre-processed data, so it is not needed.
embeddings = inception_embeddings
tensors.input_jpeg = inception_input
else:
# For training and evaluation we assume data is preprocessed, so the
# inputs are tf-examples.
# Generate placeholders for examples.
with tf.name_scope('inputs'):
feature_map = {
'image_uri':
tf.FixedLenFeature(
shape=[], dtype=tf.string, default_value=['']),
# Some images may have no labels. For those, we assume a default
# label. So the number of labels is label_count+1 for the default
# label.
'label':
tf.FixedLenFeature(
shape=[1], dtype=tf.int64,
default_value=[self.label_count]),
'embedding':
tf.FixedLenFeature(
shape=[BOTTLENECK_TENSOR_SIZE], dtype=tf.float32)
}
parsed = tf.parse_example(tensors.examples, features=feature_map)
labels = tf.squeeze(parsed['label'])
uris = tf.squeeze(parsed['image_uri'])
embeddings = parsed['embedding']
# We assume a default label, so the total number of labels is equal to
# label_count+1.
all_labels_count = self.label_count + 1
with tf.name_scope('final_ops'):
softmax, logits = self.add_final_training_ops(
embeddings,
all_labels_count,
BOTTLENECK_TENSOR_SIZE,
dropout_keep_prob=self.dropout if is_training else None)
# Prediction is the index of the label with the highest score. We are
# interested only in the top score.
prediction = tf.argmax(softmax, 1)
tensors.predictions = [prediction, softmax, embeddings]
if graph_mod == GraphMod.PREDICT:
return tensors
with tf.name_scope('evaluate'):
loss_value = loss(logits, labels)
# Add to the Graph the Ops that calculate and apply gradients.
if is_training:
tensors.train, tensors.global_step = training(loss_value)
else:
tensors.global_step = tf.Variable(0, name='global_step', trainable=False)
# Add means across all batches.
loss_updates, loss_op = util.loss(loss_value)
accuracy_updates, accuracy_op = util.accuracy(logits, labels)
if not is_training:
tf.summary.scalar('accuracy', accuracy_op)
tf.summary.scalar('loss', loss_op)
tensors.metric_updates = loss_updates + accuracy_updates
tensors.metric_values = [loss_op, accuracy_op]
return tensors
def _generate_saved_model_for_half_plus_two(export_dir,
as_text=False,
use_main_op=False,
device_type="cpu"):
"""Generates SavedModel for half plus two.
Args:
export_dir: The directory to which the SavedModel should be written.
as_text: Writes the SavedModel protocol buffer in text format to disk.
use_main_op: Whether to supply a main op during SavedModel build time.
device_name: Device to force ops to run on.
"""
builder = tf.saved_model.builder.SavedModelBuilder(export_dir)
device_name = "/cpu:0"
if device_type == "gpu":
device_name = "/gpu:0"
with tf.Session(
graph=tf.Graph(),
config=tf.ConfigProto(log_device_placement=True)) as sess:
with tf.device(device_name):
# Set up the model parameters as variables to exercise variable loading
# functionality upon restore.
a = tf.Variable(0.5, name="a")
b = tf.Variable(2.0, name="b")
c = tf.Variable(3.0, name="c")
# Create a placeholder for serialized tensorflow.Example messages to be
# fed.
serialized_tf_example = tf.placeholder(tf.string, name="tf_example")
# Parse the tensorflow.Example looking for a feature named "x" with a
# single floating point value.
feature_configs = {
"x": tf.FixedLenFeature([1], dtype=tf.float32),
"x2": tf.FixedLenFeature([1], dtype=tf.float32, default_value=[0.0])
}
# parse_example only works on CPU
with tf.device("/cpu:0"):
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
# Use tf.identity() to assign name
x = tf.identity(tf_example["x"], name="x")
y = tf.add(tf.multiply(a, x), b)
y = tf.identity(y, name="y")
y2 = tf.add(tf.multiply(a, x), c)
y2 = tf.identity(y2, name="y2")
x2 = tf.identity(tf_example["x2"], name="x2")
y3 = tf.add(tf.multiply(a, x2), c)
y3 = tf.identity(y3, name="y3")
# Create an assets file that can be saved and restored as part of the
# SavedModel.
original_assets_directory = "/tmp/original/export/assets"
original_assets_filename = "foo.txt"
original_assets_filepath = _write_assets(original_assets_directory,
original_assets_filename)
# Set up the assets collection.
assets_filepath = tf.constant(original_assets_filepath)
tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, assets_filepath)
filename_tensor = tf.Variable(
original_assets_filename,
name="filename_tensor",
trainable=False,
collections=[])
assign_filename_op = filename_tensor.assign(original_assets_filename)
# Set up the signature for Predict with input and output tensor
# specification.
predict_input_tensor = tf.saved_model.utils.build_tensor_info(x)
predict_signature_inputs = {"x": predict_input_tensor}
predict_output_tensor = tf.saved_model.utils.build_tensor_info(y)
predict_signature_outputs = {"y": predict_output_tensor}
predict_signature_def = (
tf.saved_model.signature_def_utils.build_signature_def(
predict_signature_inputs, predict_signature_outputs,
tf.saved_model.signature_constants.PREDICT_METHOD_NAME))
signature_def_map = {
"regress_x_to_y":
_build_regression_signature(serialized_tf_example, y),
"regress_x_to_y2":
_build_regression_signature(serialized_tf_example, y2),
"regress_x2_to_y3":
_build_regression_signature(x2, y3),
"classify_x_to_y":
_build_classification_signature(serialized_tf_example, y),
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
predict_signature_def
}
# Initialize all variables and then save the SavedModel.
sess.run(tf.global_variables_initializer())
if use_main_op:
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map=signature_def_map,
assets_collection=tf.get_collection(tf.GraphKeys.ASSET_FILEPATHS),
main_op=tf.group(tf.saved_model.main_op.main_op(),
assign_filename_op))
else:
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map=signature_def_map,
assets_collection=tf.get_collection(tf.GraphKeys.ASSET_FILEPATHS),
main_op=tf.group(assign_filename_op))
builder.save(as_text)
def export(checkpoint_path, export_dir, export_version, export_for_serving,
do_preprocess, cfg):
graph = tf.Graph()
input_node_name = "images"
output_node_name = None
jpegs = None
with graph.as_default():
global_step = slim.get_or_create_global_step()
input_height = cfg.IMAGE_PROCESSING.INPUT_SIZE
input_width = cfg.IMAGE_PROCESSING.INPUT_SIZE
input_depth = 3
# We want to store the preprocessing operation in the graph
if do_preprocess:
def preprocess_image(image_buffer):
"""Preprocess JPEG encoded bytes to 3D float Tensor."""
# Decode the string as an RGB JPEG.
image = tf.image.decode_jpeg(image_buffer, channels=3)
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
# Resize the image to the original height and width.
image = tf.expand_dims(image, 0)
image = tf.image.resize_bilinear(image,
[input_height, input_width],
align_corners=False)
image = tf.squeeze(image, [0])
# Finally, rescale to [-1,1] instead of [0, 1)
image = tf.subtract(image, 0.5)
image = tf.multiply(image, 2.0)
return image
input_placeholder = tf.placeholder(tf.string, name=input_node_name)
feature_configs = {
'image/encoded': tf.FixedLenFeature(shape=[], dtype=tf.string),
}
tf_example = tf.parse_example(input_placeholder, feature_configs)
jpegs = tf_example['image/encoded']
encoded_jpeg_node_name = jpegs.name[:-2]
images = tf.map_fn(preprocess_image, jpegs, dtype=tf.float32)
# We assume the client has preprocessed the data for us
else:
input_placeholder = tf.placeholder(
tf.float32, [None, input_height * input_width * input_depth],
name=input_node_name)
images = tf.reshape(input_placeholder,
[-1, input_height, input_width, input_depth])
arg_scope = nets_factory.arg_scopes_map[cfg.MODEL_NAME]()
with slim.arg_scope(arg_scope):
logits, end_points = nets_factory.networks_map[cfg.MODEL_NAME](
inputs=images, num_classes=cfg.NUM_CLASSES, is_training=False)
# GVH: I would like to use tf.identity here, but the function tensorflow.python.framework.graph_util.remove_training_nodes
# called in (optimize_for_inference_lib.optimize_for_inference) removes the identity function.
# Sticking with an add 0 operation for now.
output_node = tf.add(end_points['Predictions'], 0., name='Predictions')
output_node_name = output_node.op.name
if 'MOVING_AVERAGE_DECAY' in cfg and cfg.MOVING_AVERAGE_DECAY > 0:
variable_averages = tf.train.ExponentialMovingAverage(
cfg.MOVING_AVERAGE_DECAY, global_step)
variables_to_restore = variable_averages.variables_to_restore(
slim.get_model_variables())
else:
variables_to_restore = slim.get_variables_to_restore()
saver = tf.train.Saver(variables_to_restore, reshape=True)
if os.path.isdir(checkpoint_path):
checkpoint_dir = checkpoint_path
checkpoint_path = tf.train.latest_checkpoint(checkpoint_dir)
if checkpoint_path is None:
raise ValueError("Unable to find a model checkpoint in the " \
"directory %s" % (checkpoint_dir,))
tf.logging.info('Exporting model: %s' % checkpoint_path)
sess_config = tf.ConfigProto(
log_device_placement=cfg.SESSION_CONFIG.LOG_DEVICE_PLACEMENT,
allow_soft_placement=True,
gpu_options=tf.GPUOptions(
per_process_gpu_memory_fraction=cfg.SESSION_CONFIG.
PER_PROCESS_GPU_MEMORY_FRACTION))
sess = tf.Session(graph=graph, config=sess_config)
if export_for_serving:
classification_input_node = input_placeholder
if do_preprocess:
prediction_input_node = jpegs
else:
prediction_input_node = classification_input_node
class_scores, predicted_classes = tf.nn.top_k(
end_points['Predictions'], k=cfg.NUM_CLASSES)
with tf.Session(graph=graph) as sess:
tf.global_variables_initializer().run()
saver.restore(sess, checkpoint_path)
save_path = os.path.join(export_dir, "%d" % (export_version, ))
builder = saved_model_builder.SavedModelBuilder(save_path)
# Build the signature_def_map.
classify_inputs_tensor_info = utils.build_tensor_info(
classification_input_node)
classes_output_tensor_info = utils.build_tensor_info(
predicted_classes)
scores_output_tensor_info = utils.build_tensor_info(
class_scores)
classification_signature = signature_def_utils.build_signature_def(
inputs={
signature_constants.CLASSIFY_INPUTS:
classify_inputs_tensor_info
},
outputs={
signature_constants.CLASSIFY_OUTPUT_CLASSES:
classes_output_tensor_info,
signature_constants.CLASSIFY_OUTPUT_SCORES:
scores_output_tensor_info
},
method_name=signature_constants.CLASSIFY_METHOD_NAME)
predict_inputs_tensor_info = utils.build_tensor_info(
prediction_input_node)
prediction_signature = signature_def_utils.build_signature_def(
inputs={'images': predict_inputs_tensor_info},
outputs={
'classes': classes_output_tensor_info,
'scores': scores_output_tensor_info
},
method_name=signature_constants.PREDICT_METHOD_NAME)
legacy_init_op = tf.group(tf.tables_initializer(),
name='legacy_init_op')
builder.add_meta_graph_and_variables(
sess, [tag_constants.SERVING],
signature_def_map={
'predict_images':
prediction_signature,
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
classification_signature,
},
legacy_init_op=legacy_init_op)
builder.save()
print("Saved optimized model for TensorFlow Serving.")
else:
with sess.as_default():
tf.global_variables_initializer().run()
saver.restore(sess, checkpoint_path)
input_graph_def = graph.as_graph_def()
input_node_names = [input_node_name]
output_node_names = [output_node_name]
constant_graph_def = graph_util.convert_variables_to_constants(
sess=sess,
input_graph_def=input_graph_def,
output_node_names=output_node_names,
variable_names_whitelist=None,
variable_names_blacklist=None)
if do_preprocess:
optimized_graph_def = constant_graph_def
else:
optimized_graph_def = optimize_for_inference_lib.optimize_for_inference(
input_graph_def=constant_graph_def,
input_node_names=input_node_names,
output_node_names=output_node_names,
placeholder_type_enum=dtypes.float32.as_datatype_enum)
if not os.path.exists(export_dir):
os.makedirs(export_dir)
save_path = os.path.join(
export_dir, 'optimized_model-%d.pb' % (export_version, ))
with open(save_path, 'w') as f:
f.write(optimized_graph_def.SerializeToString())
print("Saved optimized model for mobile devices at: %s." %
(save_path, ))
print("Input node name: %s" % (input_node_name, ))
print("Output node name: %s" % (output_node_name, ))
def main(_):
"""
if len(sys.argv) < 2 or sys.argv[-1].startswith('-'):
print('Usage: mnist_export.py [--training_iteration=x] '
'[--model_version=y] export_dir')
sys.exit(-1)
"""
if FLAGS.training_iteration <= 0:
print 'Please specify a positive value for training iteration.'
sys.exit(-1)
if FLAGS.model_version <= 0:
print 'Please specify a positive value for version number.'
sys.exit(-1)
# Train model
print 'Training model...'
mnist = mnist_input_data.read_data_sets(FLAGS.work_dir, one_hot=True)
sess = tf.InteractiveSession()
serialized_tf_example = tf.placeholder(tf.string, name='tf_example')
feature_configs = {
'x': tf.FixedLenFeature(shape=[784], dtype=tf.float32),
}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
x = tf.identity(tf_example['x'],
name='x') # use tf.identity() to assign name
# Similar to
# x=tf.placeholder('float',shape=[None,784],name='x')
y_ = tf.placeholder('float', shape=[None, 10])
w = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
# sess.run(tf.global_variables_initializer())
y = tf.nn.softmax(tf.matmul(x, w) + b, name='y')
# cross_entropy = -tf.reduce_sum(y_ * tf.log(y))
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_,
logits=y)
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(
cross_entropy)
values, indices = tf.nn.top_k(y, 10)
table = tf.contrib.lookup.index_to_string_table_from_tensor(
tf.constant([str(i) for i in xrange(10)]))
prediction_classes = table.lookup(
tf.to_int64(indices)) # Predictive category label
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
# ckpt model save
saver = tf.train.Saver()
# Check whether the checkpoint file has been saved before verifying
if not os.path.exists(FLAGS.model_dir_ckpt):
os.makedirs(FLAGS.model_dir_ckpt)
ckpt = tf.train.get_checkpoint_state(FLAGS.model_dir_ckpt)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
else:
tf.global_variables_initializer().run()
if FLAGS.flag == 1: # train
for step in range(FLAGS.training_iteration):
batch = mnist.train.next_batch(50)
train_step.run(feed_dict={x: batch[0], y_: batch[1]})
if step % 100 == 0:
print(
'training accuracy %g' %
sess.run(accuracy, feed_dict={
x: batch[0],
y_: batch[1]
}))
saver.save(sess,
os.path.join(FLAGS.model_dir_ckpt, 'model.ckpt'),
global_step=step)
print 'Done training!'
if FLAGS.flag == 0: # test
print 'testing accuracy %g' % sess.run(accuracy,
feed_dict={
x: mnist.test.images,
y_: mnist.test.labels
})
print 'Done testing!'
if FLAGS.flag == -1: # Export pb model
# Export model
# WARNING(break-tutorial-inline-code): The following code snippet is
# in-lined in tutorials, please update tutorial documents accordingly
# whenever code changes.
# export_path_base = sys.argv[-1]
export_path_base = '/home/wu/pytest/MNIST/mnist_model'
export_path = os.path.join(
tf.compat.as_bytes(export_path_base),
tf.compat.as_bytes(str(FLAGS.model_version)))
print 'Exporting trained model to', export_path
# Build a pb model
builder = tf.saved_model.builder.SavedModelBuilder(export_path)
# Build the signature_def_map.
classification_inputs = tf.saved_model.utils.build_tensor_info(
serialized_tf_example)
classification_outputs_classes = tf.saved_model.utils.build_tensor_info(
prediction_classes)
classification_outputs_scores = tf.saved_model.utils.build_tensor_info(
values)
classification_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={
tf.saved_model.signature_constants.CLASSIFY_INPUTS:
classification_inputs
},
outputs={
tf.saved_model.signature_constants.CLASSIFY_OUTPUT_CLASSES:
classification_outputs_classes,
tf.saved_model.signature_constants.CLASSIFY_OUTPUT_SCORES:
classification_outputs_scores
},
method_name=tf.saved_model.signature_constants.
CLASSIFY_METHOD_NAME))
tensor_info_x = tf.saved_model.utils.build_tensor_info(x)
tensor_info_y = tf.saved_model.utils.build_tensor_info(y)
prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={'images': tensor_info_x},
outputs={'scores': tensor_info_y},
method_name=tf.saved_model.signature_constants.
PREDICT_METHOD_NAME))
legacy_init_op = tf.group(tf.tables_initializer(),
name='legacy_init_op')
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
'predict_images':
prediction_signature,
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
classification_signature,
},
legacy_init_op=legacy_init_op)
builder.save()
print 'Done exporting!'
query_camera = cameras[:, -1]
context = Context(cameras=context_cameras, frames=context_frames)
query = Query(context=context, query_camera=query_camera)
return TaskData(query=query, target=target)
def _make_read_op(self, reader, filename_queue):
"""Instantiates the ops used to read and parse the data into tensors."""
_, raw_data = reader.read_up_to(filename_queue, num_records=16)
feature_map = {
'frames': tf.FixedLenFeature(
shape=self._dataset_info.sequence_size, dtype=tf.string),
'cameras': tf.FixedLenFeature(
shape=[self._dataset_info.sequence_size * _NUM_RAW_CAMERA_PARAMS],
dtype=tf.float32)
}
example = tf.parse_example(raw_data, feature_map)
indices = self._get_randomized_indices()
frames = self._preprocess_frames(example, indices)
cameras = self._preprocess_cameras(example, indices)
return frames, cameras
def _get_randomized_indices(self):
"""Generates randomized indices into a sequence of a specific length."""
indices = tf.range(0, self._dataset_info.sequence_size)
indices = tf.random_shuffle(indices)
indices = tf.slice(indices, begin=[0], size=[self._example_size])
return indices
def _preprocess_frames(self, example, indices):
"""Instantiates the ops used to preprocess the frames data."""
frames = tf.concat(example['frames'], axis=0)
def build_graph(self, data_paths, batch_size, graph_mod):
"""Builds generic graph for training or eval."""
tensors = GraphReferences()
is_training = graph_mod == GraphMod.TRAIN
if data_paths:
_, tensors.examples = util.read_examples(
data_paths,
batch_size,
shuffle=is_training,
num_epochs=None if is_training else 2)
else:
tensors.examples = tf.placeholder(tf.string,
name='input',
shape=(None, ))
if graph_mod == GraphMod.PREDICT:
inception_input, inception_embeddings = self.build_inception_graph(
)
# Build the Inception graph. We later add final training layers
# to this graph. This is currently used only for prediction.
# For training, we use pre-processed data, so it is not needed.
embeddings = inception_embeddings
tensors.input_jpeg = inception_input
else:
# For training and evaluation we assume data is preprocessed, so the
# inputs are tf-examples.
# Generate placeholders for examples.
with tf.name_scope('inputs'):
feature_map = {
'image_uri':
tf.FixedLenFeature(shape=[],
dtype=tf.string,
default_value=['']),
# Some images may have no labels. For those, we assume a default
# label. So the number of labels is label_count+1 for the default
# label.
'label':
tf.FixedLenFeature(shape=[1],
dtype=tf.int64,
default_value=[self.label_count]),
'embedding':
tf.FixedLenFeature(shape=[BOTTLENECK_TENSOR_SIZE],
dtype=tf.float32)
}
parsed = tf.parse_example(tensors.examples,
features=feature_map)
labels = tf.squeeze(parsed['label'])
uris = tf.squeeze(parsed['image_uri'])
embeddings = parsed['embedding']
# We assume a default label, so the total number of labels is equal to
# label_count+1.
all_labels_count = self.label_count + 1
with tf.name_scope('final_ops'):
softmax, logits = self.add_final_training_ops(
embeddings,
all_labels_count,
BOTTLENECK_TENSOR_SIZE,
dropout_keep_prob=self.dropout if is_training else None)
# Prediction is the index of the label with the highest score. We are
# interested only in the top score.
prediction = tf.argmax(softmax, 1)
tensors.predictions = [prediction, softmax, embeddings]
if graph_mod == GraphMod.PREDICT:
return tensors
with tf.name_scope('evaluate'):
loss_value = loss(logits, labels)
# Add to the Graph the Ops that calculate and apply gradients.
if is_training:
tensors.train, tensors.global_step = training(loss_value)
else:
tensors.global_step = tf.Variable(0,
name='global_step',
trainable=False)
tensors.uris = uris
# Add means across all batches.
loss_updates, loss_op = util.loss(loss_value)
accuracy_updates, accuracy_op = util.accuracy(logits, labels)
if not is_training:
tf.summary.scalar('accuracy', accuracy_op)
tf.summary.scalar('loss', loss_op)
tensors.metric_updates = loss_updates + accuracy_updates
tensors.metric_values = [loss_op, accuracy_op]
return tensors
def input_fn(params=None):
"""Input function using queues for GPU."""
del params
filenames = gfile.Glob(os.path.join(flags.data_dir, pattern))
if not filenames:
raise RuntimeError('No data files found.')
filename_queue = tf.train.string_input_producer(filenames,
shuffle=True)
reader = tf.TFRecordReader()
_, val = reader.read(filename_queue)
serialized_input = tf.reshape(val, shape=[1])
image_seq = None
for i in range(0, flags.sequence_length, flags.skip_num):
image_name = 'image_' + str(i)
if flags.dataset_type == 'robot':
pose_name = 'state_' + str(i)
action_name = 'action_' + str(i)
joint_pos_name = 'joint_positions_' + str(i)
features = {
pose_name:
tf.FixedLenFeature([flags.pose_dim], tf.float32),
image_name:
tf.FixedLenFeature([1], tf.string),
action_name:
tf.FixedLenFeature([flags.pose_dim], tf.float32),
joint_pos_name:
tf.FixedLenFeature([flags.joint_pos_dim], tf.float32)
}
else:
features = {
image_name: tf.FixedLenFeature([1], tf.string),
}
parsed_input = tf.parse_example(serialized_input, features)
# Process image
image_buffer = tf.reshape(parsed_input[image_name], shape=[])
image = tf.image.decode_jpeg(image_buffer, channels=COLOR_CHAN)
image = tf.image.resize_images(
image, (IMG_HEIGHT, IMG_WIDTH),
method=tf.image.ResizeMethod.BICUBIC)
image = tf.cast(tf.expand_dims(image, 0), tf.float32) / 255.0
if flags.dataset_type == 'robot':
pose = tf.reshape(parsed_input[pose_name],
shape=[flags.pose_dim])
pose = tf.expand_dims(pose, 0)
action = tf.reshape(parsed_input[action_name],
shape=[flags.pose_dim])
action = tf.expand_dims(action, 0)
joint_pos = tf.reshape(parsed_input[joint_pos_name],
shape=[flags.joint_pos_dim])
joint_pos = tf.expand_dims(joint_pos, 0)
else:
pose = tf.zeros([1, flags.pose_dim])
action = tf.zeros([1, flags.pose_dim])
joint_pos = tf.zeros([1, flags.joint_pos_dim])
if i == 0:
image_seq = image
action_seq, pose_seq, joint_pos_seq = action, pose, joint_pos
else:
image_seq = tf.concat([image_seq, image], 0)
action_seq = tf.concat([action_seq, action], 0)
pose_seq = tf.concat([pose_seq, pose], 0)
joint_pos_seq = tf.concat([joint_pos_seq, joint_pos], 0)
[images, actions, poses, joint_pos] = tf.train.shuffle_batch(
[image_seq, action_seq, pose_seq, joint_pos_seq],
batch_size,
num_threads=4,
capacity=200 * batch_size,
min_after_dequeue=batch_size * 10,
)
joint_poses = tf.concat([joint_pos, poses], 2)
output_features = {
IMAGE_FEATURE_NAME: images,
JOINT_POSE_FEATURE_NAME: joint_poses,
ACTION_FEATURE_NAME: actions
}
return output_features, None
def main(_):
if len(sys.argv) < 2 or sys.argv[-1].startswith('-'):
print('Usage: mnist_export.py [--training_iteration=x] '
'[--export_version=y] export_dir')
sys.exit(-1)
if FLAGS.training_iteration <= 0:
print('Please specify a positive value for training iteration.')
sys.exit(-1)
if FLAGS.export_version <= 0:
print('Please specify a positive value for version number.')
sys.exit(-1)
# Train model
print('Training model...')
mnist = mnist_input_data.read_data_sets(FLAGS.work_dir, one_hot=True)
sess = tf.InteractiveSession()
serialized_tf_example = tf.placeholder(tf.string, name='tf_example')
feature_configs = {
'x': tf.FixedLenFeature(shape=[784], dtype=tf.float32),
}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
x = tf.identity(tf_example['x'],
name='x') # use tf.identity() to assign name
y_ = tf.placeholder('float', shape=[None, 10])
w = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
sess.run(tf.global_variables_initializer())
y = tf.nn.softmax(tf.matmul(x, w) + b, name='y')
cross_entropy = -tf.reduce_sum(y_ * tf.log(y))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(
cross_entropy)
values, indices = tf.nn.top_k(y, 10)
table = tf.contrib.lookup.index_to_string_table_from_tensor(
tf.constant([str(i) for i in range(10)]))
prediction_classes = table.lookup(tf.to_int64(indices))
for _ in range(FLAGS.training_iteration):
batch = mnist.train.next_batch(50)
train_step.run(feed_dict={x: batch[0], y_: batch[1]})
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
print('training accuracy %g' % sess.run(accuracy,
feed_dict={
x: mnist.test.images,
y_: mnist.test.labels
}))
print('Done training!')
# Export model
# WARNING(break-tutorial-inline-code): The following code snippet is
# in-lined in tutorials, please update tutorial documents accordingly
# whenever code changes.
export_path = sys.argv[-1]
print('Exporting trained model to %s' % export_path)
init_op = tf.group(tf.tables_initializer(), name='init_op')
saver = tf.train.Saver(sharded=True)
model_exporter = exporter.Exporter(saver)
model_exporter.init(
sess.graph.as_graph_def(),
init_op=init_op,
default_graph_signature=exporter.classification_signature(
input_tensor=serialized_tf_example,
classes_tensor=prediction_classes,
scores_tensor=values),
named_graph_signatures={
'inputs': exporter.generic_signature({'images': x}),
'outputs': exporter.generic_signature({'scores': y})
})
model_exporter.export(export_path, tf.constant(FLAGS.export_version), sess)
print('Done exporting!')
def read_files_batches(self,
batch_size,
data_mode,
records,
is_training,
memory_factor,
reader_threads,
shuffle=True):
""" Reads the content in the selected TFRecords and returns them
into batches """
has_image = self.settings.image_column()
if self.settings.image_column() is not None \
and self.settings.image_specs is None:
raise ValueError('Image field provided but no image specs found')
with tf.name_scope('batch_processing'):
# Define queue for the files
if is_training:
file_queue = tf.train.string_input_producer(
records,
shuffle=shuffle,
capacity=memory_factor,
num_epochs=None)
else:
file_queue = tf.train.string_input_producer(records,
shuffle=shuffle,
num_epochs=1)
# Read records
reader = tf.TFRecordReader()
_, example = reader.read(file_queue)
# Decode image data if required
if has_image:
to_batch = [example, self.decode_images(example)]
else:
to_batch = [example]
# Group into batches
batched = self.batch_data(data=to_batch,
batch_size=batch_size,
memory_factor=memory_factor,
reader_threads=reader_threads)
# Decode image data if required
if has_image:
batched_examples, batched_images = batched
else:
batched_examples = batched
# The serialized example is converted back to actual values
# Important to use parse_example after batching for Sparse Data
# Doc: goo.gl/Vm19xf
parsed = tf.parse_example(batched_examples,
self.settings.get_feature_dictionary())
base_features, labels = self.separate_target(parsed, batch_size)
# Decode image data if required
if has_image:
base_features.update({self._image_tag: batched_images})
return base_features, tf.cast(labels, self.settings.target_type())
def parse_wrapper(example, spec_dict):
"""Wrap tf.parse_example to support bfloat16 dtypes.
This allows models which declare bfloat16 as inputs to not require an
additional preprocessing step to cast all inputs from float32 to bfloat16.
Consider this to be analogous to JPEG decoding in the data step.
Args:
example: TFExample
spec_dict: Dictionary of feature name -> tf.FixedLenFeature
Returns:
Parsed feature map
"""
def is_bfloat_feature(value):
return value.dtype == tf.bfloat16
def maybe_map_bfloat(value):
"""Maps bfloat16 to float32."""
if is_bfloat_feature(value):
if isinstance(value, tf.FixedLenFeature):
return tf.FixedLenFeature(
value.shape,
tf.float32,
default_value=value.default_value)
elif isinstance(value, tf.VarLenFeature):
return tf.VarLenFeature(
value.shape,
tf.float32,
default_value=value.default_value)
else:
return tf.FixedLenSequenceFeature(
value.shape,
tf.float32,
default_value=value.default_value)
return value
# Change bfloat features to float32 for parsing.
new_spec_dict = {
k: maybe_map_bfloat(v)
for k, v in six.iteritems(spec_dict)
}
for k, v in six.iteritems(new_spec_dict):
if v.dtype not in [tf.float32, tf.string, tf.int64]:
raise ValueError(
'Feature specification with invalid data type for '
'tf.Example parsing: "%s": %s' % (k, v.dtype))
# Separate new_spec_dict into Context and Sequence features. In the event
# that there are no SequenceFeatures, the context_features dictionary
# (containing FixedLenFeatures) is passed to tf.parse_examples.
context_features, sequence_features = {}, {}
for k, v in six.iteritems(new_spec_dict):
v = maybe_map_bfloat(v)
if isinstance(v, tf.FixedLenSequenceFeature):
sequence_features[k] = v
elif isinstance(v, tf.FixedLenFeature):
context_features[k] = v
elif isinstance(v, tf.VarLenFeature):
context_features[k] = v
else:
raise ValueError(
'Only FixedLenFeature and FixedLenSequenceFeature are currently '
'supported.')
# If there are any sequence features, we use parse_sequence_example.
if sequence_features:
# Filter out '_length' context features; don't parse them from records.
for parse_name in sequence_features:
# Sometimes, the '_length' context feature doesn't exist.
if parse_name + '_length' in context_features:
del context_features[parse_name + '_length']
result, sequence_result, feature_lengths = tf.io.parse_sequence_example(
example,
context_features=context_features,
sequence_features=sequence_features)
result.update(sequence_result)
# Augment the parsed tensors with feature length tensors.
for parse_name, length_tensor in feature_lengths.items():
result[parse_name + '_length'] = length_tensor
else:
result = tf.parse_example(example, context_features)
to_convert = [
k for k, v in six.iteritems(spec_dict) if is_bfloat_feature(v)
]
for c in to_convert:
result[c] = tf.cast(result[c], tf.bfloat16)
return result
def _generate_saved_model_for_half_plus_two(export_dir,
as_text=False,
use_main_op=False,
device_type="cpu"):
"""Generates SavedModel for half plus two.
Args:
export_dir: The directory to which the SavedModel should be written.
as_text: Writes the SavedModel protocol buffer in text format to disk.
use_main_op: Whether to supply a main op during SavedModel build time.
device_name: Device to force ops to run on.
"""
builder = tf.saved_model.builder.SavedModelBuilder(export_dir)
device_name = "/cpu:0"
if device_type == "gpu":
device_name = "/gpu:0"
with tf.Session(graph=tf.Graph(),
config=tf.ConfigProto(log_device_placement=True)) as sess:
with tf.device(device_name):
# Set up the model parameters as variables to exercise variable loading
# functionality upon restore.
a = tf.Variable(0.5, name="a")
b = tf.Variable(2.0, name="b")
c = tf.Variable(3.0, name="c")
# Create a placeholder for serialized tensorflow.Example messages to be
# fed.
serialized_tf_example = tf.placeholder(tf.string,
name="tf_example")
# Parse the tensorflow.Example looking for a feature named "x" with a
# single floating point value.
feature_configs = {
"x":
tf.FixedLenFeature([1], dtype=tf.float32),
"x2":
tf.FixedLenFeature([1], dtype=tf.float32, default_value=[0.0])
}
# parse_example only works on CPU
with tf.device("/cpu:0"):
tf_example = tf.parse_example(serialized_tf_example,
feature_configs)
# Use tf.identity() to assign name
x = tf.identity(tf_example["x"], name="x")
y = tf.add(tf.multiply(a, x), b)
y = tf.identity(y, name="y")
y2 = tf.add(tf.multiply(a, x), c)
y2 = tf.identity(y2, name="y2")
x2 = tf.identity(tf_example["x2"], name="x2")
y3 = tf.add(tf.multiply(a, x2), c)
y2 = tf.identity(y3, name="y3")
# Create an assets file that can be saved and restored as part of the
# SavedModel.
original_assets_directory = "/tmp/original/export/assets"
original_assets_filename = "foo.txt"
original_assets_filepath = _write_assets(original_assets_directory,
original_assets_filename)
# Set up the assets collection.
assets_filepath = tf.constant(original_assets_filepath)
tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, assets_filepath)
filename_tensor = tf.Variable(original_assets_filename,
name="filename_tensor",
trainable=False,
collections=[])
assign_filename_op = filename_tensor.assign(original_assets_filename)
# Set up the signature for Predict with input and output tensor
# specification.
predict_input_tensor = tf.saved_model.utils.build_tensor_info(x)
predict_signature_inputs = {"x": predict_input_tensor}
predict_output_tensor = tf.saved_model.utils.build_tensor_info(y)
predict_signature_outputs = {"y": predict_output_tensor}
predict_signature_def = (
tf.saved_model.signature_def_utils.build_signature_def(
predict_signature_inputs, predict_signature_outputs,
tf.saved_model.signature_constants.PREDICT_METHOD_NAME))
signature_def_map = {
"regress_x_to_y":
_build_regression_signature(serialized_tf_example, y),
"regress_x_to_y2":
_build_regression_signature(serialized_tf_example, y2),
"regress_x2_to_y3":
_build_regression_signature(x2, y3),
"classify_x_to_y":
_build_classification_signature(serialized_tf_example, y),
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
predict_signature_def
}
# Initialize all variables and then save the SavedModel.
sess.run(tf.global_variables_initializer())
if use_main_op:
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map=signature_def_map,
assets_collection=tf.get_collection(
tf.GraphKeys.ASSET_FILEPATHS),
main_op=tf.group(tf.saved_model.main_op.main_op(),
assign_filename_op))
else:
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map=signature_def_map,
assets_collection=tf.get_collection(
tf.GraphKeys.ASSET_FILEPATHS),
main_op=tf.group(assign_filename_op))
builder.save(as_text)
def parse_fn(example):
return tf.parse_example(example,
{'varlen': tf.VarLenFeature(tf.int64)})
def read_and_decode(filename_queue,
model_input_image_size,
tf_dict,
tf_reader_settings,
data_augmentations,
number_of_files,
resize_output=None):
"""Read and decode tensors from tf_records and apply augmentations."""
reader = tf.TFRecordReader()
# Switch between single/multi-file reading
if number_of_files == 1:
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(serialized_example,
features=tf_dict)
else:
_, serialized_examples = reader.read_up_to(filename_queue,
num_records=number_of_files)
features = tf.parse_example(serialized_examples, features=tf_dict)
# Handle decoding of each element
image = decode_data(features=features['image'],
reader_settings=tf_reader_settings['image']['dtype'])
label = decode_data(features=features['label'],
reader_settings=tf_reader_settings['label']['dtype'])
# Reshape each element
image = tf.reshape(image, tf_reader_settings['image']['reshape'])
if tf_reader_settings['label']['reshape'] is not None:
label = tf.reshape(label, tf_reader_settings['label']['reshape'])
if image.dtype == tf.float64:
print 'Forcing float64 image to float32.'
image = tf.cast(image, tf.float32)
if label.dtype == tf.float64:
print 'Forcing float64 label to float32.'
label = tf.cast(label, tf.float32)
# Preprocess images and heatmaps
if len(model_input_image_size) == 3:
# 2D image augmentations
image, label = image_augmentations(
image=image,
label=label,
model_input_image_size=model_input_image_size,
data_augmentations=data_augmentations)
if resize_output is not None:
# Resize labels after augmentations
label = resize_image_label(im=label,
model_input_image_size=resize_output,
f='nearest')
elif len(model_input_image_size) == 4:
# 3D image augmentations.
# TODO: optimize 3D augmentations with c++. This is slow.
split_images = tf.split(image, model_input_image_size[0], axis=0)
split_images = [tf.squeeze(im, axis=0) for im in split_images]
images, labels = [], []
if np.any(['label' in x for x in data_augmentations if x is not None]):
split_labels = tf.split(label, model_input_image_size[0], axis=0)
split_labels = [tf.squeeze(lab, axis=0) for lab in split_labels]
for im, lab in zip(split_images, split_labels):
it_im, it_lab = image_augmentations(
image=im,
label=lab,
model_input_image_size=model_input_image_size[1:],
data_augmentations=data_augmentations)
if resize_output is not None:
# Resize labels after augmentations
it_lab = resize_image_label(
im=it_lab,
model_input_image_size=resize_output,
f='area')
images += [it_im]
labels += [it_lab]
label = tf.stack(labels, axis=0)
image = tf.stack(images, axis=0)
else:
if None not in data_augmentations:
for im in split_images:
it_im = image_augmentations(
image=im,
model_input_image_size=model_input_image_size[1:],
data_augmentations=data_augmentations)
images += [it_im]
image = tf.stack(images, axis=0)
if image.dtype != tf.float32:
image = tf.cast(image, tf.float32)
return image, label
class YT8MAggregatedFeatureReader(BaseReader):
"""Reads TFRecords of pre-aggregated Examples.
The TFRecords must contain Examples with a sparse int64 'labels' feature and
a fixed length float32 feature, obtained from the features in 'feature_name'.
The float features are assumed to be an average of dequantized values.
"""
def __init__(self,
num_classes=3862,
feature_sizes=[1024, 128],
feature_names=["mean_rgb", "mean_audio"]):
"""Construct a YT8MAggregatedFeatureReader.
Args:
num_classes: a positive integer for the number of classes.
feature_sizes: positive integer(s) for the feature dimensions as a list.
feature_names: the feature name(s) in the tensorflow record as a list.
"""
assert len(feature_names) == len(feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(feature_names), len(feature_sizes))
self.num_classes = num_classes
self.feature_sizes = feature_sizes
self.feature_names = feature_names
def prepare_reader(self, filename_queue, batch_size=1024):
"""Creates a single reader thread for pre-aggregated YouTube 8M Examples.
Args:
filename_queue: A tensorflow queue of filename locations.
Returns:
A tuple of video indexes, features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_examples = reader.read_up_to(filename_queue, batch_size)
tf.add_to_collection("serialized_examples", serialized_examples)
return self.prepare_serialized_examples(serialized_examples)
def prepare_serialized_examples(self, serialized_examples):
# set the mapping from the fields to data types in the proto
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {"id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)}
for feature_index in range(num_features):
feature_map[self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
labels.set_shape([None, self.num_classes])
concatenated_features = tf.concat([
features[feature_name] for feature_name in self.feature_names], 1)
return features["id"], concatenated_features, labels, tf.ones([tf.shape(serialized_examples)[0]])
def export():
# Create index->synset mapping
synsets = []
with open(SYNSET_FILE) as f:
synsets = f.read().splitlines()
# Create synset->metadata mapping
texts = {}
with open(METADATA_FILE) as f:
for line in f.read().splitlines():
parts = line.split('\t')
assert len(parts) == 2
texts[parts[0]] = parts[1]
with tf.Graph().as_default():
# Build inference model.
# Please refer to Tensorflow inception model for details.
# Input transformation.
# serialized_tf_example = tf.placeholder(tf.string, name='tf_example')
# feature_configs = {
# 'image/encoded': tf.FixedLenFeature(
# shape=[], dtype=tf.string),
# }
# tf_example = tf.parse_example(serialized_tf_example, feature_configs)
# jpegs = tf_example['image/encoded']
serialized_tf_example = tf.placeholder(tf.string, name='tf_example')
feature_configs = {
'x': tf.FixedLenFeature(shape=[], dtype=tf.float32),
}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
# reshape the input image to its original dimension
tf_example['x'] = tf.reshape(tf_example['x'], (1, 224, 224, 3))
input_tensor = tf.identity(
tf_example['x'], name='x') # use tf.identity() to assign name
# images = tf.map_fn(preprocess_image, jpegs, dtype=tf.float32)
# Run inference.
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
net, end_points = resnet_v1.resnet_v1_50(input_tensor,
1000,
is_training=False)
# logits, _ = inception_model.inference(images, NUM_CLASSES + 1)
# Transform output to topK result.
values, indices = tf.nn.top_k(net, NUM_TOP_CLASSES)
# Create a constant string Tensor where the i'th element is
# the human readable class description for the i'th index.
# Note that the 0th index is an unused background class
# (see inception model definition code).
class_descriptions = ['unused background']
for s in synsets:
class_descriptions.append(texts[s])
class_tensor = tf.constant(class_descriptions)
table = tf.contrib.lookup.index_to_string_table_from_tensor(
class_tensor)
classes = table.lookup(tf.to_int64(indices))
# Restore variables from training checkpoint.
# variable_averages = tf.train.ExponentialMovingAverage(
# inception_model.MOVING_AVERAGE_DECAY)
# variables_to_restore = variable_averages.variables_to_restore()
# saver = tf.train.Saver(variables_to_restore)
saver = tf.train.Saver()
with tf.Session() as sess:
# Restore variables from training checkpoints.
# ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir)
# if ckpt and ckpt.model_checkpoint_path:
# saver.restore(sess, ckpt.model_checkpoint_path)
# Assuming model_checkpoint_path looks something like:
# /my-favorite-path/imagenet_train/model.ckpt-0,
# extract global_step from it.
# global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
# print('Successfully loaded model from %s at step=%s.' %
# (ckpt.model_checkpoint_path, global_step))
# else:
# print('No checkpoint file found at %s' % FLAGS.checkpoint_dir)
# return
# Export inference model.
saver.restore(
sess, os.path.join(pre_trained_model_dir, "resnet_v1_50.ckpt"))
print("Model", model_name, "restored.")
output_path = os.path.join(
tf.compat.as_bytes(FLAGS.output_dir),
tf.compat.as_bytes(str(FLAGS.model_version)))
print('Exporting trained model to', output_path)
builder = tf.saved_model.builder.SavedModelBuilder(output_path)
# Build the signature_def_map.
classify_inputs_tensor_info = tf.saved_model.utils.build_tensor_info(
serialized_tf_example)
classes_output_tensor_info = tf.saved_model.utils.build_tensor_info(
classes)
scores_output_tensor_info = tf.saved_model.utils.build_tensor_info(
values)
classification_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={
tf.saved_model.signature_constants.CLASSIFY_INPUTS:
classify_inputs_tensor_info
},
outputs={
tf.saved_model.signature_constants.CLASSIFY_OUTPUT_CLASSES:
classes_output_tensor_info,
tf.saved_model.signature_constants.CLASSIFY_OUTPUT_SCORES:
scores_output_tensor_info
},
method_name=tf.saved_model.signature_constants.
CLASSIFY_METHOD_NAME))
predict_inputs_tensor_info = tf.saved_model.utils.build_tensor_info(
input_tensor)
prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={'images': predict_inputs_tensor_info},
outputs={
'classes': classes_output_tensor_info,
'scores': scores_output_tensor_info
},
method_name=tf.saved_model.signature_constants.
PREDICT_METHOD_NAME))
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
'predict_images':
prediction_signature,
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
classification_signature,
},
main_op=tf.tables_initializer(),
strip_default_attrs=True)
builder.save()
print('Successfully exported model to %s' % FLAGS.output_dir)
# Read TFRecords files for training
filename_queue = tf.train.string_input_producer(
tf.train.match_filenames_once("data/a8a_train.libsvm.tfrecords"),
num_epochs=epoch_number)
serialized_example = read_and_decode(filename_queue)
batch_serialized_example = tf.train.shuffle_batch(
[serialized_example],
batch_size=batch_size,
num_threads=thread_number,
capacity=capacity,
min_after_dequeue=min_after_dequeue)
features = tf.parse_example(batch_serialized_example,
features={
"label": tf.FixedLenFeature([], tf.float32),
"ids": tf.VarLenFeature(tf.int64),
"values": tf.VarLenFeature(tf.float32),
})
batch_labels = features["label"]
batch_ids = features["ids"]
batch_values = features["values"]
# Read TFRecords file for validatioin
validate_filename_queue = tf.train.string_input_producer(
tf.train.match_filenames_once("data/a8a_test.libsvm.tfrecords"),
num_epochs=epoch_number)
validate_serialized_example = read_and_decode(validate_filename_queue)
validate_batch_serialized_example = tf.train.shuffle_batch(
[validate_serialized_example],
batch_size=validate_batch_size,
num_threads=thread_number,
def _deserialize(examples_serialized): features = tf.parse_example(examples_serialized, _FEATURE_MAP) return features, features[movielens.RATING_COLUMN] / movielens.MAX_RATING
def _generate_saved_model_for_half_plus_two(export_dir, as_text=False):
"""Generates SavedModel for half plus two.
Args:
export_dir: The directory to which the SavedModel should be written.
as_text: Writes the SavedModel protocol buffer in text format to disk.
"""
builder = saved_model_builder.SavedModelBuilder(export_dir)
with tf.Session(graph=tf.Graph()) as sess:
# Set up the model parameters as variables to exercise variable loading
# functionality upon restore.
a = tf.Variable(0.5, name="a")
b = tf.Variable(2.0, name="b")
# Create a placeholder for serialized tensorflow.Example messages to be fed.
serialized_tf_example = tf.placeholder(tf.string, name="tf_example")
# Parse the tensorflow.Example looking for a feature named "x" with a single
# floating point value.
feature_configs = {
"x": tf.FixedLenFeature([1], dtype=tf.float32),
}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
# Use tf.identity() to assign name
x = tf.identity(tf_example["x"], name="x")
y = tf.add(tf.mul(a, x), b, name="y")
# Create an assets file that can be saved and restored as part of the
# SavedModel.
original_assets_directory = "/tmp/original/export/assets"
original_assets_filename = "foo.txt"
original_assets_filepath = _write_assets(original_assets_directory,
original_assets_filename)
# Set up the assets collection.
assets_filepath = tf.constant(original_assets_filepath)
tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, assets_filepath)
filename_tensor = tf.Variable(original_assets_filename,
name="filename_tensor",
trainable=False,
collections=[])
assign_filename_op = filename_tensor.assign(original_assets_filename)
# Set up the signature for regression with input and output tensor
# specification.
input_tensor = meta_graph_pb2.TensorInfo()
input_tensor.name = serialized_tf_example.name
signature_inputs = {signature_constants.REGRESS_INPUTS: input_tensor}
output_tensor = meta_graph_pb2.TensorInfo()
output_tensor.name = tf.identity(y).name
signature_outputs = {
signature_constants.REGRESS_OUTPUTS: output_tensor
}
signature_def = utils.build_signature_def(
signature_inputs, signature_outputs,
signature_constants.REGRESS_METHOD_NAME)
# Initialize all variables and then save the SavedModel.
sess.run(tf.global_variables_initializer())
builder.add_meta_graph_and_variables(
sess, [tag_constants.SERVING],
signature_def_map={
signature_constants.REGRESS_METHOD_NAME: signature_def
},
assets_collection=tf.get_collection(tf.GraphKeys.ASSET_FILEPATHS),
legacy_init_op=tf.group(assign_filename_op))
builder.save(as_text)
encoder = LabelEncoder()
encoder.fit(Y)
encoded_Y = encoder.transform(Y)
# convert integers to dummy variables (i.e. one hot encoded)
one_hot_labels = np_utils.to_categorical(encoded_Y)
model = Sequential()
model.add(Dense(8, input_dim=4, activation="relu"))
model.add(Dense(3, activation="softmax"))
#This is the part that had no documentation or example for keras on how to save the model using keras in the proper format for tensorflow serving
serialized_tf_example = tf.placeholder(tf.string, name='tf_example')
feature_configs = {
'x': tf.FixedLenFeature(shape=[4], dtype=tf.float32),
}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
#end
x = tf.identity(tf_example['x'], name='x') # use tf.identity() to assign name
y = model(x)
model.compile(loss='categorical_crossentropy',
optimizer="adam",
metrics=['accuracy'])
model.fit(X, one_hot_labels, epochs=epoch, batch_size=32)
labels = []
for label in Y:
if label not in labels:
labels.append(label)
def main(_):
if len(sys.argv) < 2 or sys.argv[-1].startswith('-'):
print('Usage: mnist_saved_model.py [--training_iteration=x] '
'[--model_version=y] export_dir')
sys.exit(-1)
if FLAGS.training_iteration <= 0:
print('Please specify a positive value for training iteration.')
sys.exit(-1)
if FLAGS.model_version <= 0:
print('Please specify a positive value for version number.')
sys.exit(-1)
# Train model
print('Training model...')
mnist = input_data.read_data_sets(
r"C:\Users\1TSH4W2\PycharmProjects\untitled1\MNIST_data", one_hot=True)
sess = tf.InteractiveSession()
serialized_tf_example = tf.placeholder(tf.string, name='tf_example')
feature_configs = {
'x': tf.FixedLenFeature(shape=[784], dtype=tf.float32),
}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
x = tf.identity(tf_example['x'],
name='x') # use tf.identity() to assign name
y_ = tf.placeholder('float', shape=[None, 10])
w = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
sess.run(tf.global_variables_initializer())
y = tf.nn.softmax(tf.matmul(x, w) + b, name='y')
cross_entropy = -tf.reduce_sum(y_ * tf.log(y))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(
cross_entropy)
values, indices = tf.nn.top_k(y, 10)
table = tf.contrib.lookup.index_to_string_table_from_tensor(
tf.constant([str(i) for i in range(10)]))
prediction_classes = table.lookup(tf.to_int64(indices))
for _ in range(FLAGS.training_iteration):
batch = mnist.train.next_batch(50)
train_step.run(feed_dict={x: batch[0], y_: batch[1]})
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
print('training accuracy %g' % sess.run(accuracy,
feed_dict={
x: mnist.test.images,
y_: mnist.test.labels
}))
print('Done training!')
# Export model
# WARNING(break-tutorial-inline-code): The following code snippet is
# in-lined in tutorials, please update tutorial documents accordingly
# whenever code changes.
export_path_base = sys.argv[-1]
export_path = os.path.join(tf.compat.as_bytes(export_path_base),
tf.compat.as_bytes(str(FLAGS.model_version)))
print('Exporting trained model to', export_path)
builder = tf.saved_model.builder.SavedModelBuilder(export_path)
# Build the signature_def_map.
classification_inputs = tf.saved_model.utils.build_tensor_info(
serialized_tf_example)
classification_outputs_classes = tf.saved_model.utils.build_tensor_info(
prediction_classes)
classification_outputs_scores = tf.saved_model.utils.build_tensor_info(
values)
classification_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={
tf.saved_model.signature_constants.CLASSIFY_INPUTS:
classification_inputs
},
outputs={
tf.saved_model.signature_constants.CLASSIFY_OUTPUT_CLASSES:
classification_outputs_classes,
tf.saved_model.signature_constants.CLASSIFY_OUTPUT_SCORES:
classification_outputs_scores
},
method_name=tf.saved_model.signature_constants.CLASSIFY_METHOD_NAME
))
tensor_info_x = tf.saved_model.utils.build_tensor_info(x)
tensor_info_y = tf.saved_model.utils.build_tensor_info(y)
prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={'images': tensor_info_x},
outputs={'scores': tensor_info_y},
method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME)
)
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
'predict_images':
prediction_signature,
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
classification_signature,
},
main_op=tf.tables_initializer(),
strip_default_attrs=True)
builder.save()
print('Done exporting!')
def create_sprite_image(self, examples):
"""Returns an encoded sprite image for use in Facets Dive.
Args:
examples: A list of serialized example protos to get images for.
Returns:
An encoded PNG.
"""
def generate_image_from_thubnails(thumbnails, thumbnail_dims):
"""Generates a sprite atlas image from a set of thumbnails."""
num_thumbnails = tf.shape(thumbnails)[0].eval()
images_per_row = int(math.ceil(math.sqrt(num_thumbnails)))
thumb_height = thumbnail_dims[0]
thumb_width = thumbnail_dims[1]
master_height = images_per_row * thumb_height
master_width = images_per_row * thumb_width
num_channels = 3
master = np.zeros([master_height, master_width, num_channels])
for idx, image in enumerate(thumbnails.eval()):
left_idx = idx % images_per_row
top_idx = int(math.floor(idx / images_per_row))
left_start = left_idx * thumb_width
left_end = left_start + thumb_width
top_start = top_idx * thumb_height
top_end = top_start + thumb_height
master[top_start:top_end, left_start:left_end, :] = image
return tf.image.encode_png(master)
with tf.Session():
keys_to_features = {
self.image_feature_name:
tf.FixedLenFeature((), tf.string, default_value=''),
}
parsed = tf.parse_example(examples, keys_to_features)
images = tf.zeros([1, 1, 1, 1], tf.float32)
i = tf.constant(0)
thumbnail_dims = (self.sprite_thumbnail_dim_px,
self.sprite_thumbnail_dim_px)
num_examples = tf.constant(len(examples))
encoded_images = parsed[self.image_feature_name]
# Loop over all examples, decoding the image feature value, resizing
# and appending to a list of all images.
def loop_body(i, encoded_images, images):
encoded_image = encoded_images[i]
image = tf.image.decode_jpeg(encoded_image, channels=3)
resized_image = tf.image.resize_images(image, thumbnail_dims)
expanded_image = tf.expand_dims(resized_image, 0)
images = tf.cond(
tf.equal(i, 0), lambda: expanded_image,
lambda: tf.concat([images, expanded_image], 0))
return i + 1, encoded_images, images
loop_out = tf.while_loop(
lambda i, encoded_images, images: tf.less(i, num_examples),
loop_body, [i, encoded_images, images],
shape_invariants=[
i.get_shape(),
encoded_images.get_shape(),
tf.TensorShape(None)
])
# Create the single sprite atlas image from these thumbnails.
sprite = generate_image_from_thubnails(loop_out[2], thumbnail_dims)
return sprite.eval()
def main(_):
# exit if any parameters not compatible
if FLAGS.version == None:
print 'please input a version number [--version=x]'
sys.exit(-1)
if FLAGS.iterations <= 0:
print 'Please specify a positive value for training iteration.'
sys.exit(-1)
if FLAGS.version <= 0:
print 'Please specify a positive value for version number.'
sys.exit(-1)
# setup the parameters
# number of input values
vals = 3
# max answer, so basically the width of the frame
max_answer = 40
# number of different M's, biggest gradient will fit in frame
gradients = max_answer/(vals)+1
iterations = FLAGS.iterations
learning_rate = 0.3
# I am using a GPU
# this line limits memory usage of the GPU to 0.25 when session is created
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.25)
# defining function to make training data
def training_data():
n = 0
rows = 1
# array which we are using as our x values
# in equation of linear line, y = Mx
# it includes 1 extra value as this will be used as our labal
X = np.arange(vals+1).astype(np.int32)
# empty array to write our training data to
Y = np.array([])
# loop so it cycles through every gradient
for i in range(gradients):
Y = np.append(Y, X*n).reshape(rows,vals+1)
# increase number of rows to reshape it
rows += 1
# increase gradient by 1
n+=1
Y = Y.astype(np.int32)
# return the training data
# and number of lines to learn
return(Y,np.size(Y,0))
# print the training data
print(training_data())
# training_line_data = training data in numbers
# training lines = number of different lines
training_line_data,training_lines = training_data()
# the length is for when we convert the numbers into a binary array
# the array will be all zeros except one, which will be 1
# this will be the particle in this pont in time
# each one is like a frame in a video
length = max_answer
# the full length is the length of all the input frames stacked into one, 1d array
full_length = length*vals
# this function turns the data into the arrays explained above
def set_data():
# there are two arrays, one for training data and one for labels
# input_data is the converted traiing data
input_data = np.zeros([training_lines,vals,length])
# the labels will be one-hot arrays
labels = np.zeros([training_lines,1,gradients])
# this sets the values specified in the training data to one
for i in range(training_lines):
# we need to set each individual input value
for a in range(vals):
# set the value to a 1
input_data[i,a,training_line_data[i,a]] = 1
# set the label value to a 1
labels[i,0,training_line_data[i,vals-(vals-1)]] = 1
# here, we reshape it tto the full length 1d array
input_data = input_data.reshape(training_lines,1,full_length)
# return the data and labels
return(input_data,labels)
# we define the weights, biases and inputs
# set placeholder for x, input training data
serialized_tf_example = tf.placeholder(tf.string, name='tf_example')
feature_configs = {'x': tf.FixedLenFeature(shape=[full_length], dtype=tf.float32),}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
x = tf.identity(tf_example['x'], name='x'))
# weights and biases
W = tf.Variable(tf.zeros([full_length, gradients]))
b = tf.Variable(tf.zeros([gradients]))
# function which gives the probabilities
y = tf.matmul(x, W) + b
# we will feed the labels in here
y_ = tf.placeholder(tf.float32, [None, gradients])
# configure the loss function, using cross entropy
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))
# define the optimizer
# AdagradOptimizer works much better than GradientDescentOptimizer
#train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy)
train_step = tf.train.AdagradOptimizer(learning_rate).minimize(cross_entropy)
# set values to later use for creating the signature
values, indices = tf.nn.top_k(y_conv, gradients)
table = tf.contrib.lookup.index_to_string_table_from_tensor(tf.constant([str(i) for i in xrange(gradients)]))
prediction_classes = table.lookup(tf.to_int64(indices))
# create interactive session using the GPU line for above
sess = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=gpu_options))
# initialize variables
init = tf.global_variables_initializer()
sess.run(init)
# now we run the learning loop
for _ in range(iterations):
# set training data and labels
x_data, y_data = set_data()
# use next line each step
x_data = x_data[_%training_lines]
y_data = y_data[_%training_lines]
# run the training optimizer
sess.run(train_step, feed_dict={x: x_data, y_: y_data})
# print steps and error 20 times in total
if _ % (iterations/20) == 0:
print 'step', _, 'out of', iterations
print 'error =', sess.run(cross_entropy, feed_dict={x: x_data, y_: y_data})
# set directory to save model in
export_path_base = FLAGS.work_dir
# name new directory with version in work dir
export_path = os.path.join(compat.as_bytes(export_path_base),compat.as_bytes(str(FLAGS.version)))
# print where its saving the model
print 'Exporting trained model to', export_path
builder = saved_model_builder.SavedModelBuilder(export_path)
# Build the signature_def_map
classification_inputs = utils.build_tensor_info(serialized_tf_example) # input with placeholder x
classification_outputs_classes = utils.build_tensor_info(prediction_classes) # output of prediction classes
classification_outputs_scores = utils.build_tensor_info(values) # output of prediction probabilities
# build signature for classification
classification_signature = signature_def_utils.build_signature_def(
inputs={
signature_constants.CLASSIFY_INPUTS: classification_inputs},
outputs={
signature_constants.CLASSIFY_OUTPUT_CLASSES:classification_outputs_classes,
signature_constants.CLASSIFY_OUTPUT_SCORES:classification_outputs_scores
},
method_name=signature_constants.CLASSIFY_METHOD_NAME)
def _decode_batch_record(record, name_to_features):
example = tf.parse_example(record, name_to_features)
return example
def _deserialize(examples_serialized): features = tf.parse_example(examples_serialized, _FEATURE_MAP) return features, features[movielens.RATING_COLUMN] / movielens.MAX_RATING
def test_make_feed_dict_error(self, feature_spec, instances, error_msg,
error_type=ValueError):
tensors = tf.parse_example(tf.placeholder(tf.string, [None]), feature_spec)
schema = dataset_schema.from_feature_spec(feature_spec)
with self.assertRaisesRegexp(error_type, error_msg):
impl_helper.make_feed_dict(tensors, schema, instances)
def prepare_serialized_examples(self, serialized_examples):
logging.set_verbosity(tf.logging.DEBUG)
# hardcoded values
len_features_frames = 1024
len_features_audio = 128
name_frames = "mean_rgb"
name_audio = "mean_audio"
# set the mapping from the fields to data types in the proto
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {
"video_id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)
}
logging.debug("self.random_selection es " + str(self.random_selection))
zeros_float = tf.zeros([tf.shape(serialized_examples)[0]])
# Manera cutre de crear un vector de False. Alguna altra manera ha d'haver-hi
is_negative = tf.not_equal(zeros_float, zeros_float)
for feature_index in range(num_features):
feature_map[
self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
features_rgb = features[name_frames]
features_audio = features[name_audio]
labels_audio = tf.sparse_to_indicator(features["labels"],
self.num_classes)
batch_size = tf.shape(features[name_frames])[0]
if self.negative_sampling:
labels = tf.sparse_to_indicator(features["labels"],
self.num_classes)
labels.set_shape([None, self.num_classes])
def return_itself(a, b):
return a, b
# 80% of the samples are negative
number_neg_sample = tf.random_uniform(
[],
minval=0.,
maxval=1.,
dtype=tf.float32,
name="random_number_neg_sample")
constant = tf.constant(self.percentage_negative)
batch_size = tf.shape(features_rgb)[0]
logging.info("-----------------")
logging.info(batch_size)
is_negative = tf.random_uniform([batch_size, 1],
minval=0,
maxval=1)
is_negative = tf.less(is_negative, constant)
features_audio_return, labels_audio = self.sample_negatively(
features, labels, is_negative)
concatenated_features = tf.concat(
[features_rgb, features_audio_return], 1)
else:
# Normal case, leave as it was
# We can use python comparisons because they are checked only when creating the graph
if self.random_selection == 0 | (self.random_selection ==
1 & num_features > 1):
for feature_index in range(num_features):
feature_map[self.feature_names[
feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples,
features=feature_map)
labels = tf.sparse_to_indicator(features["labels"],
self.num_classes)
labels.set_shape([None, self.num_classes])
labels_audio = labels
concatenated_features = tf.concat([
features[feature_name]
for feature_name in self.feature_names
], 1)
# Evaluation with only one of the two features
elif self.random_selection == 1:
feature_map[name_frames] = tf.FixedLenFeature(
[len_features_frames], tf.float32)
feature_map[name_audio] = tf.FixedLenFeature(
[len_features_audio], tf.float32)
features = tf.parse_example(serialized_examples,
features=feature_map)
labels = tf.sparse_to_indicator(features["labels"],
self.num_classes)
labels.set_shape([None, self.num_classes])
# In this point there is only 1 feature_name
# We can use python comparisons because they are checked only when creating the graph
if self.feature_names[0] == name_frames:
concatenated_features = tf.concat([
features[name_frames],
tf.zeros_like(features[name_audio])
], 1)
else:
concatenated_features = tf.concat([
tf.zeros_like(features[name_frames]),
features[name_audio]
], 1)
# Training with thirds
else:
feature_map[name_frames] = tf.FixedLenFeature(
[len_features_frames], tf.float32)
feature_map[name_audio] = tf.FixedLenFeature(
[len_features_audio], tf.float32)
features = tf.parse_example(serialized_examples,
features=feature_map)
labels = tf.sparse_to_indicator(features["labels"],
self.num_classes)
labels.set_shape([None, self.num_classes])
number = tf.random_uniform([],
minval=0.,
maxval=3.,
dtype=tf.float32,
name="random_number")
features_rgb = features[name_frames]
features_audio = features[name_audio]
one = tf.constant(1.)
two = tf.constant(2.)
features_audio = tf.cond(
tf.less(number, one),
lambda: tf.clip_by_value(features_audio, 0, 0),
lambda: features_audio)
features_rgb = tf.cond(
tf.greater(number, two),
lambda: tf.clip_by_value(features_rgb, 0, 0),
lambda: features_rgb)
concatenated_features = tf.concat(
[features_rgb, features_audio], 1, name="concat_features")
return features["video_id"], concatenated_features, labels, tf.ones(
[tf.shape(serialized_examples)[0]]), is_negative, labels_audio
def main(_):
if len(sys.argv) == 1:
print(
'Usage: mnist_export.py [--training_iteration=x] '
'[--data_dir=dataset_dir] [--log_dir=checkpoint_dir]')
if FLAGS.training_iteration <= 0:
print 'Please specify a positive value for training iteration.'
sys.exit(-1)
# Train model
print 'Training model...'
mnist = mnist_input_data.read_data_sets(FLAGS.data_dir, one_hot=True)
# sess = tf.InteractiveSession()
with tf.Session() as sess:
serialized_tf_example = tf.placeholder(tf.string, name='tf_example')
feature_configs = {
'x': tf.FixedLenFeature(shape=[784], dtype=tf.float32),
}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
x = tf.identity(tf_example['x'],
name='x') # use tf.identity() to assign name
with tf.name_scope('input_reshape'):
image_shaped_input = tf.reshape(x, [-1, 28, 28, 1])
tf.summary.image('input', image_shaped_input, 10)
y_ = tf.placeholder('float', shape=[None, 10])
w = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
tf.summary.scalar('w', w)
tf.summary.scalar('b', b)
sess.run(tf.initialize_all_variables())
y = tf.nn.softmax(tf.matmul(x, w) + b, name='y')
cross_entropy = -tf.reduce_sum(y_ * tf.log(y), name='corss_entropy')
tf.summary.scalar('cross_entropy', cross_entropy)
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(
cross_entropy, name='train_step')
values, indices = tf.nn.top_k(y, 10)
prediction_classes = tf.contrib.lookup.index_to_string(
tf.to_int64(indices),
mapping=tf.constant([str(i) for i in xrange(10)]))
tf.add_to_collection('values', values)
tf.add_to_collection('prediction_classes', prediction_classes)
summary_op = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train',
sess.graph)
for step in range(FLAGS.training_iteration):
batch = mnist.train.next_batch(50)
_, summary = sess.run([train_step, summary_op],
feed_dict={
x: batch[0],
y_: batch[1]
})
train_writer.add_summary(summary, step)
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
print 'training accuracy %g' % sess.run(accuracy,
feed_dict={
x: mnist.test.images,
y_: mnist.test.labels
})
print 'Done training!'
train_writer.close()
checkpoint_dir = FLAGS.log_dir
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
saver = tf.train.Saver()
saver.save(sess, checkpoint_dir + "/ckpt")
print 'Done saving!'
capacity=capacity,
min_after_dequeue=min_after_dequeue,
enqueue_many=True)
else:
serialized_example = read_and_decode(filename_queue)
batch_serialized_example = tf.train.shuffle_batch(
[serialized_example],
batch_size=batch_size,
num_threads=thread_number,
capacity=capacity,
min_after_dequeue=min_after_dequeue)
features = tf.parse_example(
batch_serialized_example,
features={
"label": tf.FixedLenFeature([], tf.float32),
"ids": tf.VarLenFeature(tf.int64),
"values": tf.VarLenFeature(tf.float32),
})
batch_labels = features["label"]
batch_ids = features["ids"]
batch_values = features["values"]
init_op = tf.global_variables_initializer()
sess = tf.Session()
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
sess.run(init_op,options=run_options, run_metadata=run_metadata)
sess.run(tf.local_variables_initializer(),options=run_options, run_metadata=run_metadata)
def Export():
with tf.Session() as sess:
# Make model parameters a&b variables instead of constants to
# exercise the variable reloading mechanisms.
a = tf.Variable(0.5, name="a")
b = tf.Variable(2.0, name="b")
# Create a placeholder for serialized tensorflow.Example messages to be fed.
serialized_tf_example = tf.placeholder(tf.string, name="tf_example")
# Parse the tensorflow.Example looking for a feature named "x" with a single
# floating point value.
feature_configs = {"x": tf.FixedLenFeature([1], dtype=tf.float32),}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
# Use tf.identity() to assign name
x = tf.identity(tf_example["x"], name="x")
# Calculate, y = a*x + b
y = tf.add(tf.mul(a, x), b, name="y")
# Setup a standard Saver for our variables.
save = tf.train.Saver(
{
"a": a,
"b": b
},
sharded=True,
write_version=tf.train.SaverDef.V2 if FLAGS.use_checkpoint_v2 else
tf.train.SaverDef.V1)
# asset_path contains the base directory of assets used in training (e.g.
# vocabulary files).
original_asset_path = tf.constant("/tmp/original/export/assets")
# Ops reading asset files should reference the asset_path tensor
# which stores the original asset path at training time and the
# overridden assets directory at restore time.
asset_path = tf.Variable(original_asset_path,
name="asset_path",
trainable=False,
collections=[])
assign_asset_path = asset_path.assign(original_asset_path)
# Use a fixed global step number.
global_step_tensor = tf.Variable(123, name="global_step")
# Create a RegressionSignature for our input and output.
regression_signature = exporter.regression_signature(
input_tensor=serialized_tf_example,
# Use tf.identity here because we export two signatures here.
# Otherwise only graph for one of the signatures will be loaded
# (whichever is created first) during serving.
output_tensor=tf.identity(y))
named_graph_signature = {
"inputs": exporter.generic_signature({"x": x}),
"outputs": exporter.generic_signature({"y": y})
}
# Create two filename assets and corresponding tensors.
# TODO(b/26254158) Consider adding validation of file existance as well as
# hashes (e.g. sha1) for consistency.
original_filename1 = tf.constant("hello1.txt")
tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, original_filename1)
filename1 = tf.Variable(original_filename1,
name="filename1",
trainable=False,
collections=[])
assign_filename1 = filename1.assign(original_filename1)
original_filename2 = tf.constant("hello2.txt")
tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, original_filename2)
filename2 = tf.Variable(original_filename2,
name="filename2",
trainable=False,
collections=[])
assign_filename2 = filename2.assign(original_filename2)
# Init op contains a group of all variables that we assign.
init_op = tf.group(assign_asset_path, assign_filename1, assign_filename2)
# CopyAssets is used as a callback during export to copy files to the
# given export directory.
def CopyAssets(filepaths, export_path):
print("copying asset files to: %s" % export_path)
for filepath in filepaths:
print("copying asset file: %s" % filepath)
# Run an export.
tf.initialize_all_variables().run()
export = exporter.Exporter(save)
export.init(
sess.graph.as_graph_def(),
init_op=init_op,
default_graph_signature=regression_signature,
named_graph_signatures=named_graph_signature,
assets_collection=tf.get_collection(tf.GraphKeys.ASSET_FILEPATHS),
assets_callback=CopyAssets)
export.export(FLAGS.export_dir, global_step_tensor, sess)
def export():
'''
No 1 Sess outf of 2 : ctpn_sess
'''
cfg_from_file(os.path.join(dir_path, 'text_post.yml'))
config = tf.ConfigProto(allow_soft_placement=True)
ctpn_sess = tf.Session(config=config)
with ctpn_sess.as_default():
with tf.gfile.FastGFile('../data/ctpn.pb', 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
ctpn_sess.graph.as_default()
tf.import_graph_def(graph_def, name='')
ctpn_sess.run(tf.global_variables_initializer())
cv2img = cv2.imread("../data/demo/006.jpg", cv2.IMREAD_COLOR)
result_boxes = query_ctpn(ctpn_sess, cv2img)
print('Creating boxes done')
'''
No 2 Sess outf of 2:sess
'''
with tf.Session() as sess:
with gfile.FastGFile('../data/ctpn.pb', 'rb') as f:
restored_graph_def = tf.GraphDef()
restored_graph_def.ParseFromString(f.read())
tf.import_graph_def(restored_graph_def,
input_map=None,
return_elements=None,
name="")
'''
export_path_base = args.export_model_dir
export_path = os.path.join(tf.compat.as_bytes(export_path_base),
tf.compat.as_bytes(str(args.model_version)))
'''
builder = tf.saved_model.builder.SavedModelBuilder('../exportPo/1')
#print('Exporting trained model to', export_path)
print('Exporting trained model ')
raw_image = tf.placeholder(tf.string, name='tf_box')
feature_configs = {
'image/encoded': tf.FixedLenFeature(shape=[], dtype=tf.string),
}
tf_example = tf.parse_example(raw_image, feature_configs)
jpegs = tf_example['image/encoded']
image_string = tf.reshape(jpegs, shape=[])
jpeg = preprocess_image(image_string)
print('jpeg,jpeg.shape[]', jpeg, jpeg.shape)
output_tensor_cls_prob,output_tensor_box_pred = tf.import_graph_def\
(tf.get_default_graph().as_graph_def(),
input_map={'Placeholder:0': jpeg},
return_elements=['Reshape_2:0','rpn_bbox_pred/Reshape_1:0'])
tensor_info_input = tf.saved_model.utils.build_tensor_info(raw_image)
tensor_info_output_cls_prob = tf.saved_model.utils.build_tensor_info(
output_tensor_cls_prob)
tensor_info_output_box_pred = tf.saved_model.utils.build_tensor_info(
output_tensor_box_pred)
'''
#crop_resize_img,crop_resize_im_info = resize_im(cv2img, result_boxes)
#crop_resize_img,crop_resize_im_info = crop_resize_image(imageplaceholder_info, result_boxes)
# output_crop_resize_img = tf.saved_model.utils.build_tensor_info(crop_resize_img)
#output_crop_resize_img_info = tf.saved_model.utils.build_tensor_info(crop_resize_im_info)
#----------
'''
result_boxes = np.array(result_boxes, dtype=np.float32)
result_boxes = tf.convert_to_tensor(result_boxes)
tensor_info_output_boxes = tf.saved_model.utils.build_tensor_info(
result_boxes)
prediction_post_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={'images': tensor_info_input},
outputs={'detection_boxes': tensor_info_output_boxes},
#outputs={'detection_boxes': tensor_info_output_boxes,
# 'resize_im_info':im_info_output,
# 'crop_resize_img': output_crop_resize_img,
# 'crop_resize_im_info': output_crop_resize_img_info,},
method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME)
)
builder.add_meta_graph_and_variables(
sess,
[tf.saved_model.tag_constants.SERVING],
signature_def_map={
# 'predict_images':prediction_signature,
'predict_images_post': prediction_post_signature
})
builder.save(as_text=False)
def main():
# Get hyperparameters
if FLAGS.enable_colored_log:
import coloredlogs
coloredlogs.install()
logging.basicConfig(level=logging.INFO)
FEATURE_SIZE = FLAGS.feature_size
LABEL_SIZE = FLAGS.label_size
EPOCH_NUMBER = FLAGS.epoch_number
if EPOCH_NUMBER <= 0:
EPOCH_NUMBER = None
BATCH_THREAD_NUMBER = FLAGS.batch_thread_number
MIN_AFTER_DEQUEUE = FLAGS.min_after_dequeue
BATCH_CAPACITY = BATCH_THREAD_NUMBER * FLAGS.batch_size + MIN_AFTER_DEQUEUE
MODE = FLAGS.mode
MODEL = FLAGS.model
OPTIMIZER = FLAGS.optimizer
CHECKPOINT_PATH = FLAGS.checkpoint_path
if not CHECKPOINT_PATH.startswith("fds://") and not os.path.exists(
CHECKPOINT_PATH):
os.makedirs(CHECKPOINT_PATH)
CHECKPOINT_FILE = CHECKPOINT_PATH + "/checkpoint.ckpt"
LATEST_CHECKPOINT = tf.train.latest_checkpoint(CHECKPOINT_PATH)
OUTPUT_PATH = FLAGS.output_path
if not OUTPUT_PATH.startswith("fds://") and not os.path.exists(
OUTPUT_PATH):
os.makedirs(OUTPUT_PATH)
pprint.PrettyPrinter().pprint(FLAGS.__flags)
# Read TFRecords files for training
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
return serialized_example
# Read TFRecords files for training
filename_queue = tf.train.string_input_producer(
tf.train.match_filenames_once(FLAGS.train_tfrecords_file),
num_epochs=EPOCH_NUMBER)
serialized_example = read_and_decode(filename_queue)
batch_serialized_example = tf.train.batch(
[serialized_example],
batch_size=FLAGS.batch_size,
num_threads=BATCH_THREAD_NUMBER,
capacity=BATCH_CAPACITY,
min_after_dequeue=MIN_AFTER_DEQUEUE)
features = tf.parse_example(batch_serialized_example,
features={
"label": tf.FixedLenFeature([],
tf.float32),
"ids": tf.VarLenFeature(tf.int64),
"values": tf.VarLenFeature(tf.float32),
})
batch_labels = features["label"]
batch_ids = features["ids"]
batch_values = features["values"]
# Read TFRecords file for validation
validate_filename_queue = tf.train.string_input_producer(
tf.train.match_filenames_once(FLAGS.validate_tfrecords_file),
num_epochs=EPOCH_NUMBER)
validate_serialized_example = read_and_decode(validate_filename_queue)
validate_batch_serialized_example = tf.train.shuffle_batch(
[validate_serialized_example],
batch_size=FLAGS.validate_batch_size,
num_threads=BATCH_THREAD_NUMBER,
capacity=BATCH_CAPACITY,
min_after_dequeue=MIN_AFTER_DEQUEUE)
validate_features = tf.parse_example(validate_batch_serialized_example,
features={
"label":
tf.FixedLenFeature([],
tf.float32),
"ids":
tf.VarLenFeature(tf.int64),
"values":
tf.VarLenFeature(tf.float32),
})
validate_batch_labels = validate_features["label"]
validate_batch_ids = validate_features["ids"]
validate_batch_values = validate_features["values"]
# Define the model
input_units = FEATURE_SIZE
output_units = LABEL_SIZE
model_network_hidden_units = [int(i) for i in FLAGS.model_network.split()]
def full_connect(inputs, weights_shape, biases_shape, is_train=True):
with tf.device("/cpu:0"):
weights = tf.get_variable(
"weights",
weights_shape,
initializer=tf.random_normal_initializer())
biases = tf.get_variable(
"biases",
biases_shape,
initializer=tf.random_normal_initializer())
layer = tf.matmul(inputs, weights) + biases
if FLAGS.enable_bn and is_train:
mean, var = tf.nn.moments(layer, axes=[0])
scale = tf.get_variable(
"scale",
biases_shape,
initializer=tf.random_normal_initializer())
shift = tf.get_variable(
"shift",
biases_shape,
initializer=tf.random_normal_initializer())
layer = tf.nn.batch_normalization(layer, mean, var, shift,
scale, FLAGS.bn_epsilon)
return layer
def sparse_full_connect(sparse_ids,
sparse_values,
weights_shape,
biases_shape,
is_train=True):
weights = tf.get_variable("weights",
weights_shape,
initializer=tf.random_normal_initializer())
biases = tf.get_variable("biases",
biases_shape,
initializer=tf.random_normal_initializer())
return tf.nn.embedding_lookup_sparse(
weights, sparse_ids, sparse_values, combiner="sum") + biases
def full_connect_relu(inputs, weights_shape, biases_shape, is_train=True):
return tf.nn.relu(
full_connect(inputs, weights_shape, biases_shape, is_train))
def customized_inference(sparse_ids, sparse_values, is_train=True):
hidden1_units = 128
hidden2_units = 32
hidden3_units = 8
with tf.variable_scope("input"):
sparse_layer = sparse_full_connect(sparse_ids, sparse_values,
[input_units, hidden1_units],
[hidden1_units], is_train)
layer = tf.nn.relu(sparse_layer)
with tf.variable_scope("layer0"):
layer = full_connect_relu(layer, [hidden1_units, hidden2_units],
[hidden2_units], is_train)
with tf.variable_scope("layer1"):
layer = full_connect_relu(layer, [hidden2_units, hidden3_units],
[hidden3_units], is_train)
if FLAGS.enable_dropout and is_train:
layer = tf.nn.dropout(layer, FLAGS.dropout_keep_prob)
with tf.variable_scope("output"):
layer = full_connect(layer, [hidden3_units, output_units],
[output_units], is_train)
return layer
def dnn_inference(sparse_ids, sparse_values, is_train=True):
with tf.variable_scope("input"):
sparse_layer = sparse_full_connect(
sparse_ids, sparse_values,
[input_units, model_network_hidden_units[0]],
[model_network_hidden_units[0]], is_train)
layer = tf.nn.relu(sparse_layer)
for i in range(len(model_network_hidden_units) - 1):
with tf.variable_scope("layer{}".format(i)):
layer = full_connect_relu(layer, [
model_network_hidden_units[i],
model_network_hidden_units[i + 1]
], [model_network_hidden_units[i + 1]], is_train)
with tf.variable_scope("output"):
layer = full_connect(
layer, [model_network_hidden_units[-1], output_units],
[output_units], is_train)
return layer
def lr_inference(sparse_ids, sparse_values, is_train=True):
with tf.variable_scope("logistic_regression"):
layer = sparse_full_connect(sparse_ids, sparse_values,
[input_units, output_units],
[output_units])
return layer
def wide_and_deep_inference(sparse_ids, sparse_values, is_train=True):
return lr_inference(sparse_ids,
sparse_values, is_train) + dnn_inference(
sparse_ids, sparse_values, is_train)
def inference(sparse_ids, sparse_values, is_train=True):
if MODEL == "dnn":
return dnn_inference(sparse_ids, sparse_values, is_train)
elif MODEL == "lr":
return lr_inference(sparse_ids, sparse_values, is_train)
elif MODEL == "wide_and_deep":
return wide_and_deep_inference(sparse_ids, sparse_values, is_train)
elif MODEL == "customized":
return customized_inference(sparse_ids, sparse_values, is_train)
else:
logging.error("Unknown model, exit now")
exit(1)
logging.info("Use the model: {}, model network: {}".format(
MODEL, FLAGS.model_network))
logits = inference(batch_ids, batch_values, True)
batch_labels = tf.to_int64(batch_labels)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=batch_labels)
loss = tf.reduce_mean(cross_entropy, name="loss")
global_step = tf.Variable(0, name="global_step", trainable=False)
if FLAGS.enable_lr_decay:
logging.info("Enable learning rate decay rate: {}".format(
FLAGS.lr_decay_rate))
starter_learning_rate = FLAGS.learning_rate
learning_rate = tf.train.exponential_decay(starter_learning_rate,
global_step,
100000,
FLAGS.lr_decay_rate,
staircase=True)
else:
learning_rate = FLAGS.learning_rate
optimizer = get_optimizer(FLAGS.optimizer, learning_rate)
train_op = optimizer.minimize(loss, global_step=global_step)
tf.get_variable_scope().reuse_variables()
# Define accuracy op for train data
train_accuracy_logits = inference(batch_ids, batch_values, False)
train_softmax = tf.nn.softmax(train_accuracy_logits)
train_correct_prediction = tf.equal(tf.argmax(train_softmax, 1),
batch_labels)
train_accuracy = tf.reduce_mean(
tf.cast(train_correct_prediction, tf.float32))
# Define auc op for train data
batch_labels = tf.cast(batch_labels, tf.int32)
sparse_labels = tf.reshape(batch_labels, [-1, 1])
derived_size = tf.shape(batch_labels)[0]
indices = tf.reshape(tf.range(0, derived_size, 1), [-1, 1])
concated = tf.concat(axis=1, values=[indices, sparse_labels])
outshape = tf.stack([derived_size, LABEL_SIZE])
new_train_batch_labels = tf.sparse_to_dense(concated, outshape, 1.0, 0.0)
_, train_auc = tf.contrib.metrics.streaming_auc(train_softmax,
new_train_batch_labels)
# Define accuracy op for validate data
validate_accuracy_logits = inference(validate_batch_ids,
validate_batch_values, False)
validate_softmax = tf.nn.softmax(validate_accuracy_logits)
validate_batch_labels = tf.to_int64(validate_batch_labels)
validate_correct_prediction = tf.equal(tf.argmax(validate_softmax, 1),
validate_batch_labels)
validate_accuracy = tf.reduce_mean(
tf.cast(validate_correct_prediction, tf.float32))
# Define auc op for validate data
validate_batch_labels = tf.cast(validate_batch_labels, tf.int32)
sparse_labels = tf.reshape(validate_batch_labels, [-1, 1])
derived_size = tf.shape(validate_batch_labels)[0]
indices = tf.reshape(tf.range(0, derived_size, 1), [-1, 1])
concated = tf.concat(axis=1, values=[indices, sparse_labels])
outshape = tf.stack([derived_size, LABEL_SIZE])
new_validate_batch_labels = tf.sparse_to_dense(concated, outshape, 1.0,
0.0)
_, validate_auc = tf.contrib.metrics.streaming_auc(
validate_softmax, new_validate_batch_labels)
# Define inference op
sparse_index = tf.placeholder(tf.int64, [None, 2])
sparse_ids = tf.placeholder(tf.int64, [None])
sparse_values = tf.placeholder(tf.float32, [None])
sparse_shape = tf.placeholder(tf.int64, [2])
inference_ids = tf.SparseTensor(sparse_index, sparse_ids, sparse_shape)
inference_values = tf.SparseTensor(sparse_index, sparse_values,
sparse_shape)
inference_logits = inference(inference_ids, inference_values, False)
inference_softmax = tf.nn.softmax(inference_logits)
inference_op = tf.argmax(inference_softmax, 1)
keys_placeholder = tf.placeholder(tf.int32, shape=[None, 1])
keys = tf.identity(keys_placeholder)
model_signature = {
"inputs":
exporter.generic_signature({
"keys": keys_placeholder,
"indexs": sparse_index,
"ids": sparse_ids,
"values": sparse_values,
"shape": sparse_shape
}),
"outputs":
exporter.generic_signature({
"keys": keys,
"softmax": inference_softmax,
"prediction": inference_op
})
}
# Initialize saver and summary
saver = tf.train.Saver()
tf.summary.scalar("loss", loss)
tf.summary.scalar("train_accuracy", train_accuracy)
tf.summary.scalar("train_auc", train_auc)
tf.summary.scalar("validate_accuracy", validate_accuracy)
tf.summary.scalar("validate_auc", validate_auc)
summary_op = tf.summary.merge_all()
init_op = [
tf.global_variables_initializer(),
tf.local_variables_initializer()
]
# Create session to run
with tf.Session() as sess:
logging.info("Start to run with mode: {}".format(MODE))
writer = tf.summary.FileWriter(OUTPUT_PATH, sess.graph)
sess.run(init_op)
if MODE == "train":
# Restore session and start queue runner
restore_session_from_checkpoint(sess, saver, LATEST_CHECKPOINT)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
start_time = datetime.datetime.now()
try:
while not coord.should_stop():
if FLAGS.benchmark_mode:
sess.run(train_op)
else:
_, step = sess.run([train_op, global_step])
# Print state while training
if step % FLAGS.steps_to_validate == 0:
loss_value, train_accuracy_value, train_auc_value, validate_accuracy_value, auc_value, summary_value = sess.run(
[
loss, train_accuracy, train_auc,
validate_accuracy, validate_auc, summary_op
])
end_time = datetime.datetime.now()
logging.info(
"[{}] Step: {}, loss: {}, train_acc: {}, train_auc: {}, valid_acc: {}, valid_auc: {}"
.format(end_time - start_time, step,
loss_value, train_accuracy_value,
train_auc_value,
validate_accuracy_value, auc_value))
writer.add_summary(summary_value, step)
saver.save(sess, CHECKPOINT_FILE, global_step=step)
start_time = end_time
except tf.errors.OutOfRangeError:
if FLAGS.benchmark_mode:
print("Finish training for benchmark")
exit(0)
else:
# Export the model after training
export_model(sess, saver, model_signature,
FLAGS.model_path, FLAGS.model_version)
finally:
coord.request_stop()
coord.join(threads)
elif MODE == "export":
if not restore_session_from_checkpoint(sess, saver,
LATEST_CHECKPOINT):
logging.error("No checkpoint found, exit now")
exit(1)
# Export the model
export_model(sess, saver, model_signature, FLAGS.model_path,
FLAGS.model_version)
elif MODE == "savedmodel":
if not restore_session_from_checkpoint(sess, saver,
LATEST_CHECKPOINT):
logging.error("No checkpoint found, exit now")
exit(1)
graph_file_name = "graph.pb"
logging.info("Export the graph to: {}".format(
FLAGS.saved_model_path))
tf.train.write_graph(sess.graph_def,
FLAGS.saved_model_path,
graph_file_name,
as_text=False)
logging.info("Export the saved model to {}".format(
FLAGS.saved_model_path))
export_path_base = FLAGS.saved_model_path
export_path = os.path.join(
compat.as_bytes(export_path_base),
compat.as_bytes(str(FLAGS.model_version)))
model_signature = signature_def_utils.build_signature_def(
inputs={
"keys": utils.build_tensor_info(keys_placeholder),
"indexs": utils.build_tensor_info(sparse_index),
"ids": utils.build_tensor_info(sparse_ids),
"values": utils.build_tensor_info(sparse_values),
"shape": utils.build_tensor_info(sparse_shape)
},
outputs={
"keys": utils.build_tensor_info(keys),
"softmax": utils.build_tensor_info(inference_softmax),
"prediction": utils.build_tensor_info(inference_op)
},
method_name=signature_constants.PREDICT_METHOD_NAME)
try:
builder = saved_model_builder.SavedModelBuilder(export_path)
builder.add_meta_graph_and_variables(
sess,
[tag_constants.SERVING],
clear_devices=True,
signature_def_map={
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
model_signature,
},
#legacy_init_op=legacy_init_op)
legacy_init_op=tf.group(tf.initialize_all_tables(),
name="legacy_init_op"))
builder.save()
except Exception as e:
logging.error(
"Fail to export saved model, exception: {}".format(e))
elif MODE == "inference":
if not restore_session_from_checkpoint(sess, saver,
LATEST_CHECKPOINT):
logging.error("No checkpoint found, exit now")
exit(1)
# Load inference test data
inference_result_file_name = "./inference_result.txt"
inference_test_file_name = "./data/a8a_test.libsvm"
labels = []
feature_ids = []
feature_values = []
feature_index = []
ins_num = 0
for line in open(inference_test_file_name, "r"):
tokens = line.split(" ")
labels.append(int(tokens[0]))
feature_num = 0
for feature in tokens[1:]:
feature_id, feature_value = feature.split(":")
feature_ids.append(int(feature_id))
feature_values.append(float(feature_value))
feature_index.append([ins_num, feature_num])
feature_num += 1
ins_num += 1
# Run inference
start_time = datetime.datetime.now()
prediction, prediction_softmax = sess.run(
[inference_op, inference_softmax],
feed_dict={
sparse_index: feature_index,
sparse_ids: feature_ids,
sparse_values: feature_values,
sparse_shape: [ins_num, FEATURE_SIZE]
})
end_time = datetime.datetime.now()
# Compute accuracy
label_number = len(labels)
correct_label_number = 0
for i in range(label_number):
if labels[i] == prediction[i]:
correct_label_number += 1
accuracy = float(correct_label_number) / label_number
# Compute auc
expected_labels = np.array(labels)
predict_labels = prediction_softmax[:, 0]
fpr, tpr, thresholds = metrics.roc_curve(expected_labels,
predict_labels,
pos_label=0)
auc = metrics.auc(fpr, tpr)
logging.info("[{}] Inference accuracy: {}, auc: {}".format(
end_time - start_time, accuracy, auc))
# Save result into the file
np.savetxt(inference_result_file_name,
prediction_softmax,
delimiter=",")
logging.info(
"Save result to file: {}".format(inference_result_file_name))
elif MODE == "inference_with_tfrecords":
if not restore_session_from_checkpoint(sess, saver,
LATEST_CHECKPOINT):
logging.error("No checkpoint found, exit now")
exit(1)
# Load inference test data
inference_result_file_name = "./inference_result.txt"
inference_test_file_name = "./data/a8a/a8a_test.libsvm.tfrecords"
#inference_test_file_name = "hdfs://namenode:8020/user/tobe/deep_recommend_system/data/a8a/a8a_test.libsvm.tfrecords"
# batch_labels = features["label"]
# batch_ids = features["ids"]
# batch_values = features["values"]
batch_feature_index = []
batch_labels = []
batch_ids = []
batch_values = []
ins_num = 0
# Read from TFRecords files
for serialized_example in tf.python_io.tf_record_iterator(
inference_test_file_name):
# Get serialized example from file
example = tf.train.Example()
example.ParseFromString(serialized_example)
label = example.features.feature["label"].float_list.value
ids = example.features.feature["ids"].int64_list.value
values = example.features.feature["values"].float_list.value
#print("label: {}, features: {}".format(label, " ".join([str(id) + ":" + str(value) for id, value in zip(ids, values)])))
batch_labels.append(label)
# Notice that using extend() instead of append() to flatten the values
batch_ids.extend(ids)
batch_values.extend(values)
for i in xrange(len(ids)):
batch_feature_index.append([ins_num, i])
ins_num += 1
# Run inference
start_time = datetime.datetime.now()
prediction, prediction_softmax = sess.run(
[inference_op, inference_softmax],
feed_dict={
sparse_index: batch_feature_index,
sparse_ids: batch_ids,
sparse_values: batch_values,
sparse_shape: [ins_num, FEATURE_SIZE]
})
end_time = datetime.datetime.now()
# Compute accuracy
label_number = len(batch_labels)
correct_label_number = 0
for i in range(label_number):
if batch_labels[i] == prediction[i]:
correct_label_number += 1
accuracy = float(correct_label_number) / label_number
# Compute auc
expected_labels = np.array(batch_labels)
predict_labels = prediction_softmax[:, 0]
fpr, tpr, thresholds = metrics.roc_curve(expected_labels,
predict_labels,
pos_label=0)
auc = metrics.auc(fpr, tpr)
logging.info("[{}] Inference accuracy: {}, auc: {}".format(
end_time - start_time, accuracy, auc))
# Save result into the file
np.savetxt(inference_result_file_name,
prediction_softmax,
delimiter=",")
logging.info(
"Save result to file: {}".format(inference_result_file_name))
def Export(export_dir, use_checkpoint_v2):
with tf.Session() as sess:
# Make model parameters a&b variables instead of constants to
# exercise the variable reloading mechanisms.
a = tf.Variable(0.5, name="a")
b = tf.Variable(2.0, name="b")
# Create a placeholder for serialized tensorflow.Example messages to be fed.
serialized_tf_example = tf.placeholder(tf.string, name="tf_example")
# Parse the tensorflow.Example looking for a feature named "x" with a single
# floating point value.
feature_configs = {"x": tf.FixedLenFeature([1], dtype=tf.float32),}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
# Use tf.identity() to assign name
x = tf.identity(tf_example["x"], name="x")
# Calculate, y = a*x + b
y = tf.add(tf.multiply(a, x), b, name="y")
# Setup a standard Saver for our variables.
save = tf.train.Saver(
{
"a": a,
"b": b
},
sharded=True,
write_version=tf.train.SaverDef.V2 if use_checkpoint_v2 else
tf.train.SaverDef.V1)
# asset_path contains the base directory of assets used in training (e.g.
# vocabulary files).
original_asset_path = tf.constant("/tmp/original/export/assets")
# Ops reading asset files should reference the asset_path tensor
# which stores the original asset path at training time and the
# overridden assets directory at restore time.
asset_path = tf.Variable(original_asset_path,
name="asset_path",
trainable=False,
collections=[])
assign_asset_path = asset_path.assign(original_asset_path)
# Use a fixed global step number.
global_step_tensor = tf.Variable(123, name="global_step")
# Create a RegressionSignature for our input and output.
regression_signature = exporter.regression_signature(
input_tensor=serialized_tf_example,
# Use tf.identity here because we export two signatures here.
# Otherwise only graph for one of the signatures will be loaded
# (whichever is created first) during serving.
output_tensor=tf.identity(y))
named_graph_signature = {
"inputs": exporter.generic_signature({"x": x}),
"outputs": exporter.generic_signature({"y": y})
}
# Create two filename assets and corresponding tensors.
# TODO(b/26254158) Consider adding validation of file existence as well as
# hashes (e.g. sha1) for consistency.
original_filename1 = tf.constant("hello1.txt")
tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, original_filename1)
filename1 = tf.Variable(original_filename1,
name="filename1",
trainable=False,
collections=[])
assign_filename1 = filename1.assign(original_filename1)
original_filename2 = tf.constant("hello2.txt")
tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, original_filename2)
filename2 = tf.Variable(original_filename2,
name="filename2",
trainable=False,
collections=[])
assign_filename2 = filename2.assign(original_filename2)
# Init op contains a group of all variables that we assign.
init_op = tf.group(assign_asset_path, assign_filename1, assign_filename2)
# CopyAssets is used as a callback during export to copy files to the
# given export directory.
def CopyAssets(filepaths, export_path):
print("copying asset files to: %s" % export_path)
for filepath in filepaths:
print("copying asset file: %s" % filepath)
# Run an export.
tf.global_variables_initializer().run()
export = exporter.Exporter(save)
export.init(
sess.graph.as_graph_def(),
init_op=init_op,
default_graph_signature=regression_signature,
named_graph_signatures=named_graph_signature,
assets_collection=tf.get_collection(tf.GraphKeys.ASSET_FILEPATHS),
assets_callback=CopyAssets)
export.export(export_dir, global_step_tensor, sess)
def serving_input_fn():
serialized_tf_example = tf.placeholder(dtype=tf.string, shape=[None], name='input_tensors')
receiver_tensors = {"predictor_inputs": serialized_tf_example}
feature_spec = {"words": tf.FixedLenFeature([25], tf.int64)}
features = tf.parse_example(serialized_tf_example, feature_spec)
return tf.estimator.export.ServingInputReceiver(features, receiver_tensors)
# Exporting model
import tensorflow as tf
import numpy as np
from tensorflow.examples.tutorials.mnist import input_data
import os
work_dir = "./tmp"
model_version = 9
trainning_iteration = 1000
input_size = 784
no_class = 10
batch_size = 100
totoal_batches = 200
tf_example = tf.parse_example(tf.placeholder(tf.string, name='tf_example'),
{'x': tf.FixedLenFeature(shape=[784], dtype=tf.float32), })
x_input = tf.identity(tf_example['x'], name='x')
y_input = tf.placeholder(tf.float32, shape=[None, no_class])
weights = tf.Variable(tf.random_normal([input_size, no_class]))
bias = tf.Variable(tf.random_normal([no_class]))
logits = tf.matmul(x_input, weights) + bias
soft_max_cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_input,
logits=logits)
loss_operation = tf.reduce_mean(soft_max_cross_entropy)
optimiser = tf.train.AdamOptimizer().minimize(loss_operation)
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
def parse_tf_example(example_proto, label, feature_spec):
parsed_features = tf.parse_example(serialized=example_proto,
features=feature_spec)
target = parsed_features.pop(label)
return parsed_features, target
