def test_keyed_parse_json(self): gfile.Glob = self._orig_glob filename = self._create_temp_file( '{"features": {"feature": {"age": {"int64_list": {"value": [0]}}}}}\n' '{"features": {"feature": {"age": {"int64_list": {"value": [1]}}}}}\n' '{"features": {"feature": {"age": {"int64_list": {"value": [2]}}}}}\n' ) batch_size = 1 queue_capacity = 5 name = "my_batch" with tf.Graph().as_default() as g, self.test_session( graph=g) as session: dtypes = {"age": tf.FixedLenFeature([1], tf.int64)} parse_fn = lambda example: tf.parse_single_example( # pylint: disable=g-long-lambda tf.decode_json_example(example), dtypes) keys, inputs = tf.contrib.learn.io.read_keyed_batch_examples( filename, batch_size, reader=tf.TextLineReader, randomize_input=False, num_epochs=1, queue_capacity=queue_capacity, parse_fn=parse_fn, name=name) self.assertAllEqual((None, ), keys.get_shape().as_list()) self.assertEqual(1, len(inputs)) self.assertAllEqual((None, 1), inputs["age"].get_shape().as_list()) session.run(tf.local_variables_initializer()) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(session, coord=coord) key, age = session.run([keys, inputs["age"]]) self.assertAllEqual(age, [[0]]) self.assertAllEqual(key, [filename.encode("utf-8") + b":1"]) key, age = session.run([keys, inputs["age"]]) self.assertAllEqual(age, [[1]]) self.assertAllEqual(key, [filename.encode("utf-8") + b":2"]) key, age = session.run([keys, inputs["age"]]) self.assertAllEqual(age, [[2]]) self.assertAllEqual(key, [filename.encode("utf-8") + b":3"]) with self.assertRaises(errors.OutOfRangeError): session.run(inputs) coord.request_stop() coord.join(threads)
def _serving_input_receiver_fn(): """Applies transforms to raw data in json-example strings.""" json_example_placeholder = tf.placeholder(tf.string, shape=[None]) example_strings = tf.decode_json_example(json_example_placeholder) raw_features = tf.parse_example(example_strings, raw_serving_feature_spec) inputs = {"json_example": json_example_placeholder} _, transformed_features = ( saved_transform_io.partially_apply_saved_transform_internal( transform_savedmodel_dir, raw_features)) if convert_scalars_to_vectors: transformed_features = _convert_scalars_to_vectors(transformed_features) return tf.estimator.export.ServingInputReceiver( transformed_features, inputs)
def test_keyed_parse_json(self): gfile.Glob = self._orig_glob filename = self._create_temp_file( '{"features": {"feature": {"age": {"int64_list": {"value": [0]}}}}}\n' '{"features": {"feature": {"age": {"int64_list": {"value": [1]}}}}}\n' '{"features": {"feature": {"age": {"int64_list": {"value": [2]}}}}}\n' ) batch_size = 1 queue_capacity = 5 name = "my_batch" with tf.Graph().as_default() as g, self.test_session(graph=g) as session: dtypes = {"age": tf.FixedLenFeature([1], tf.int64)} parse_fn = lambda example: tf.parse_single_example( # pylint: disable=g-long-lambda tf.decode_json_example(example), dtypes ) keys, inputs = tf.contrib.learn.io.read_keyed_batch_examples( filename, batch_size, reader=tf.TextLineReader, randomize_input=False, num_epochs=1, queue_capacity=queue_capacity, parse_fn=parse_fn, name=name, ) session.run(tf.initialize_local_variables()) coord = tf.train.Coordinator() tf.train.start_queue_runners(session, coord=coord) key, age = session.run([keys, inputs["age"]]) self.assertAllEqual(age, [[0]]) self.assertAllEqual(key, [filename.encode("utf-8") + b":1"]) key, age = session.run([keys, inputs["age"]]) self.assertAllEqual(age, [[1]]) self.assertAllEqual(key, [filename.encode("utf-8") + b":2"]) key, age = session.run([keys, inputs["age"]]) self.assertAllEqual(age, [[2]]) self.assertAllEqual(key, [filename.encode("utf-8") + b":3"]) with self.assertRaises(errors.OutOfRangeError): session.run(inputs) coord.request_stop()
def _serving_input_fn(): """Applies transforms to raw data in json-example strings.""" json_example_placeholder = tf.placeholder(tf.string, shape=[None]) example_strings = tf.decode_json_example(json_example_placeholder) raw_features = tf.parse_example(example_strings, raw_serving_feature_spec) inputs = {"json_example": json_example_placeholder} _, transformed_features = ( saved_transform_io.partially_apply_saved_transform( transform_savedmodel_dir, raw_features)) if convert_scalars_to_vectors: transformed_features = _convert_scalars_to_vectors( transformed_features) return input_fn_utils.InputFnOps(transformed_features, None, inputs)
def _testRoundTrip(self, examples): with self.test_session() as sess: examples = np.array(examples, dtype=np.object) json_tensor = tf.constant( [json_format.MessageToJson(m) for m in examples.flatten()], shape=examples.shape, dtype=tf.string ) binary_tensor = tf.decode_json_example(json_tensor) binary_val = sess.run(binary_tensor) if examples.shape: self.assertShapeEqual(binary_val, json_tensor) for input_example, output_binary in zip(np.array(examples).flatten(), binary_val.flatten()): output_example = tf.train.Example() output_example.ParseFromString(output_binary) self.assertProtoEquals(input_example, output_example) else: output_example = tf.train.Example() output_example.ParseFromString(binary_val) self.assertProtoEquals(examples.item(), output_example)
def _testRoundTrip(self, examples): with self.test_session() as sess: examples = np.array(examples, dtype=np.object) json_tensor = tf.constant( [json_format.MessageToJson(m) for m in examples.flatten()], shape=examples.shape, dtype=tf.string) binary_tensor = tf.decode_json_example(json_tensor) binary_val = sess.run(binary_tensor) if examples.shape: self.assertShapeEqual(binary_val, json_tensor) for input_example, output_binary in zip(np.array(examples).flatten(), binary_val.flatten()): output_example = tf.train.Example() output_example.ParseFromString(output_binary) self.assertProtoEquals(input_example, output_example) else: output_example = tf.train.Example() output_example.ParseFromString(binary_val) self.assertProtoEquals(examples.item(), output_example)
def testInvalidSyntax(self): with self.test_session() as sess: json_tensor = tf.constant(["{]"]) binary_tensor = tf.decode_json_example(json_tensor) with self.assertRaisesOpError("Error while parsing JSON"): sess.run(binary_tensor)
tf.colocate_with() tf.complex() tf.cond tf.confusion_matrix() tf.conj() tf.cross() tf.cumprod() tf.cumsum() tf.constant() tf.convert_to_tensor() tf.convert_to_tensor_or_indexed_slices() tf.convert_to_tensor_or_sparse_tensor() tf.decode_base64() tf.decode_csv() tf.decode_json_example() tf.decode_raw() tf.device() tf.diag() tf.diag_part() tf.div() tf.divide() tf.batch_to_space_nd() tf.space_to_batch_nd() tf.batch_to_space() tf.space_to_batch() tf.depth_to_space() tf.space_to_depth() tf.dtypes