def testMakeOutputDictError(self):
schema = self.toSchema({'a': tf.VarLenFeature(tf.string)})
# SparseTensor that cannot be represented as VarLenFeature.
fetches = {
'a': tf.SparseTensorValue(indices=np.array([(0, 2), (0, 4), (0, 8)]),
values=np.array([10.0, 20.0, 30.0]),
dense_shape=(1, 20))
}
with self.assertRaisesRegexp(
ValueError, 'cannot be decoded by ListColumnRepresentation'):
_ = impl_helper.make_output_dict(schema, fetches)
# SparseTensor of invalid rank.
fetches = {
'a': tf.SparseTensorValue(
indices=np.array([(0, 0, 1), (0, 0, 2), (0, 0, 3)]),
values=np.array([10.0, 20.0, 30.0]),
dense_shape=(1, 10, 10))
}
with self.assertRaisesRegexp(
ValueError, 'cannot be decoded by ListColumnRepresentation'):
_ = impl_helper.make_output_dict(schema, fetches)
# SparseTensor with indices that are out of order.
fetches = {
'a': tf.SparseTensorValue(indices=np.array([(0, 2), (2, 4), (1, 8)]),
values=np.array([10.0, 20.0, 30.0]),
dense_shape=(3, 20))
}
with self.assertRaisesRegexp(
ValueError, 'Encountered out-of-order sparse index'):
_ = impl_helper.make_output_dict(schema, fetches)
def process(self, element, saved_model_dir):
"""Runs the given graph to realize the output tensors (i.e. features).
Runs the graph in a TF session for computing the output values of the
tensors, given an input row of data (input tensors). Due to the record-by
record nature of beam we are operating sess.run() on individual record
tensors vs batched tensors.
Args:
element: the element being processed by the DoFn
saved_model_dir: Directory containing saved model.
Yields:
A representation of output features as a dict mapping keys (logical column
names) to values.
"""
try:
element = element.element
except AttributeError:
pass
if saved_model_dir != self._saved_model_dir:
self._initialize_graph(saved_model_dir)
feed_dict = impl_helper.make_feed_dict(self._inputs,
self._input_schema, element)
fetched_dict = self._session.run(self._outputs, feed_dict=feed_dict)
yield impl_helper.make_output_dict(self._output_schema, fetched_dict)
def process(self, element, saved_model_dir):
"""Runs the given graph to realize the output tensors (i.e. features).
Runs the graph in a TF session for computing the output values of the
tensors, given an input row of data (input tensors). Due to the record-by
record nature of beam we are operating sess.run() on individual record
tensors vs batched tensors.
Args:
element: the element being processed by the DoFn
saved_model_dir: Directory containing saved model.
Yields:
A representation of output features as a dict mapping keys (logical column
names) to values.
"""
if (not hasattr(self._thread_local, 'graph_state')
or self._thread_local.graph_state.saved_model_dir !=
saved_model_dir):
self._num_graph_loads.inc(1)
self._thread_local.graph_state = self._GraphState(
saved_model_dir, self._input_schema, self._output_schema)
feed_dict = impl_helper.make_feed_dict(
self._thread_local.graph_state.inputs, self._input_schema, element)
fetched_dict = self._thread_local.graph_state.session.run(
self._thread_local.graph_state.outputs, feed_dict=feed_dict)
yield impl_helper.make_output_dict(self._output_schema, fetched_dict)
def testMakeOutputDict(self):
schema = self.toSchema({
'a':
tf.FixedLenFeature(None, tf.int64),
'b':
tf.FixedLenFeature([], tf.float32),
'c':
tf.FixedLenFeature([1], tf.float32),
'd':
tf.FixedLenFeature([2, 2], tf.float32),
'e':
tf.VarLenFeature(tf.string),
'f':
tf.SparseFeature('idx', 'val', tf.float32, 10)
})
fetches = {
'a':
np.array([100, 200, 300]),
'b':
np.array([10.0, 20.0, 30.0]),
'c':
np.array([[40.0], [80.0], [120.0]]),
'd':
np.array([[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]],
[[9.0, 10.0], [11.0, 12.0]]]),
'e':
tf.SparseTensorValue(
indices=np.array([(0, 0), (0, 1), (0, 2), (2, 0), (2, 1),
(2, 2)]),
values=np.array(['doe', 'a', 'deer', 'a', 'female', 'deer']),
dense_shape=(3, 3)),
'f':
tf.SparseTensorValue(indices=np.array([(0, 2), (0, 4), (0, 8),
(1, 8), (1, 4)]),
values=np.array(
[10.0, 20.0, 30.0, 40.0, 50.0]),
dense_shape=(3, 20))
}
output_dict = impl_helper.make_output_dict(schema, fetches)
self.assertSetEqual(set(six.iterkeys(output_dict)),
set(['a', 'b', 'c', 'd', 'e', 'f']))
self.assertAllEqual(output_dict['a'], [100, 200, 300])
self.assertAllEqual(output_dict['b'], [10.0, 20.0, 30.0])
self.assertAllEqual(output_dict['c'], [[40.0], [80.0], [120.0]])
self.assertAllEqual(
output_dict['d'],
[[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]],
[[9.0, 10.0], [11.0, 12.0]]])
self.assertAllEqual(output_dict['e'][0], ['doe', 'a', 'deer'])
self.assertAllEqual(output_dict['e'][1], [])
self.assertAllEqual(output_dict['e'][2], ['a', 'female', 'deer'])
self.assertEqual(len(output_dict['f']), 2)
self.assertAllEqual(output_dict['f'][0][0], [2, 4, 8])
self.assertAllEqual(output_dict['f'][0][1], [8, 4])
self.assertAllEqual(output_dict['f'][0][2], [])
self.assertAllEqual(output_dict['f'][1][0], [10.0, 20.0, 30.0])
self.assertAllEqual(output_dict['f'][1][1], [40.0, 50.0])
self.assertAllEqual(output_dict['f'][1][2], [])
def testMakeOutputDictError(self):
# SparseTensor that cannot be represented as VarLenFeature.
schema = self.toSchema({'a': tf.VarLenFeature(tf.string)})
fetches = {
'a': tf.SparseTensorValue(indices=[(0, 2), (0, 4), (0, 8)],
values=[10.0, 20.0, 30.0],
dense_shape=(1, 20))
}
with self.assertRaises(ValueError):
_ = impl_helper.make_output_dict(schema, fetches)
def testMakeOutputDict(self):
schema = self.toSchema({
'a':
tf.FixedLenFeature(None, tf.int64),
'b':
tf.FixedLenFeature([2, 2], tf.float32),
'c':
tf.VarLenFeature(tf.string),
'd':
tf.SparseFeature('idx', 'val', tf.float32, 10)
})
fetches = {
'a':
np.asarray([100, 200]),
'b':
np.asarray([[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]),
'c':
tf.SparseTensorValue(
indices=[(0, 0), (0, 1), (0, 2), (1, 0), (1, 1), (1, 2)],
values=['doe', 'a', 'deer', 'a', 'female', 'deer'],
dense_shape=(2, 3)),
'd':
tf.SparseTensorValue(indices=[(0, 2), (0, 4), (0, 8)],
values=[10.0, 20.0, 30.0],
dense_shape=(2, 20))
}
output_dicts = impl_helper.make_output_dict(schema, fetches)
self.assertEqual(2, len(output_dicts))
self.assertSetEqual(set(output_dicts[0].keys()),
set(['a', 'b', 'c', 'idx', 'val']))
self.assertAllEqual(output_dicts[0]['a'], 100)
self.assertAllEqual(output_dicts[0]['b'], [[1.0, 2.0], [3.0, 4.0]])
self.assertAllEqual(output_dicts[0]['c'], ['doe', 'a', 'deer'])
self.assertAllEqual(output_dicts[0]['idx'], [2, 4, 8])
self.assertAllEqual(output_dicts[0]['val'], [10.0, 20.0, 30.0])
self.assertAllEqual(output_dicts[1]['a'], 200)
self.assertAllEqual(output_dicts[1]['b'], [[5.0, 6.0], [7.0, 8.0]])
self.assertAllEqual(output_dicts[1]['c'], ['a', 'female', 'deer'])
self.assertAllEqual(output_dicts[1]['idx'], [])
self.assertAllEqual(output_dicts[1]['val'], [])
def convert_and_unbatch(batch_dict):
return impl_helper.to_instance_dicts(
impl_helper.make_output_dict(output_metadata.schema, batch_dict))