def test_make_list_of_column(self):
col1 = mocks.MockColumn("foo", [])
col2 = mocks.MockColumn("bar", [])
self.assertEqual([], _make_list_of_column(None))
self.assertEqual([col1], _make_list_of_column(col1))
self.assertEqual([col1], _make_list_of_column([col1]))
self.assertEqual([col1, col2], _make_list_of_column([col1, col2]))
self.assertEqual([col1, col2], _make_list_of_column((col1, col2)))
def test_set_item_column_multi(self):
df = setup_test_df()
self.assertEqual(3, len(df))
col1 = mocks.MockColumn("QuackColumn", [])
col2 = mocks.MockColumn("MooColumn", [])
df["quack", "moo"] = [col1, col2]
self.assertEqual(5, len(df))
col3 = df["quack"]
self.assertEqual(col1, col3)
col4 = df["moo"]
self.assertEqual(col2, col4)
def setup_test_df():
"""Create a dataframe populated with some test columns."""
df = learn.DataFrame()
df["a"] = learn.TransformedColumn(
[mocks.MockColumn("foobar", [])],
mocks.MockTwoOutputTransform("iue", "eui", "snt"), "out1")
df["b"] = learn.TransformedColumn(
[mocks.MockColumn("foobar", [])],
mocks.MockTwoOutputTransform("iue", "eui", "snt"), "out2")
df["c"] = learn.TransformedColumn(
[mocks.MockColumn("foobar", [])],
mocks.MockTwoOutputTransform("iue", "eui", "snt"), "out1")
return df
def test_build_multiple_output(self):
col = learn.TransformedColumn(
[mocks.MockColumn("foobar", [])],
mocks.MockTwoOutputTransform("thb", "nth", "snt"), "out2")
result = col.build()
expected = "Fake Tensor 2"
self.assertEqual(expected, result)
def test_build_single_output(self):
col = learn.TransformedColumn(
[mocks.MockColumn("foobar", [])],
mocks.MockOneOutputTransform("thb", "nth"), "out1")
result = col.build()
expected = "Fake Tensor 1"
self.assertEqual(expected, result)
def test_set_item_column(self):
df = setup_test_df()
self.assertEqual(3, len(df))
col1 = mocks.MockColumn("QuackColumn", [])
df["quack"] = col1
self.assertEqual(4, len(df))
col2 = df["quack"]
self.assertEqual(col1, col2)
def test_repr(self):
col = learn.TransformedColumn(
[mocks.MockColumn("foobar", [])],
mocks.MockTwoOutputTransform("thb", "nth", "snt"), "qux")
# note params are sorted by name
expected = ("MockTransform({'param_one': 'thb', 'param_three': 'snt', "
"'param_two': 'nth'})"
"(foobar)[qux]")
self.assertEqual(expected, repr(col))
def test_call(self):
t = mocks.MockTwoOutputTransform("a", "b", "c")
# MockTwoOutputTransform has input valency 1
input1 = mocks.MockColumn("foobar", [])
out1, out2 = t([input1]) # pylint: disable=not-callable
self.assertEqual(learn.TransformedColumn, type(out1))
# self.assertEqual(out1.transform, t)
# self.assertEqual(out1.output_name, "output1")
self.assertEqual(learn.TransformedColumn, type(out2))
def test_cache(self):
z = mocks.MockColumn("foobar", [])
t = mocks.MockTwoOutputTransform("thb", "nth", "snt")
cache = {}
t.apply_transform([z], cache)
self.assertEqual(2, len(cache))
expected_keys = [
"MockTransform("
"{'param_one': 'thb', 'param_three': 'snt', 'param_two': 'nth'})"
"(foobar)[out1]", "MockTransform("
"{'param_one': 'thb', 'param_three': 'snt', 'param_two': 'nth'})"
"(foobar)[out2]"
]
self.assertEqual(expected_keys, sorted(cache.keys()))
def testParse(self):
parser = csv_parser.CSVParser(column_names=["col0", "col1", "col2"],
default_values=["", "", 1.4])
csv_lines = ["one,two,2.5", "four,five,6.0"]
csv_input = tf.constant(csv_lines,
dtype=tf.string,
shape=[len(csv_lines)])
csv_column = mocks.MockColumn("csv", csv_input)
expected_output = [
np.array([b"one", b"four"]),
np.array([b"two", b"five"]),
np.array([2.5, 6.0])
]
output_columns = parser(csv_column)
self.assertEqual(3, len(output_columns))
cache = {}
output_tensors = [o.build(cache) for o in output_columns]
self.assertEqual(3, len(output_tensors))
with self.test_session() as sess:
output = sess.run(output_tensors)
for expected, actual in zip(expected_output, output):
np.testing.assert_array_equal(actual, expected)
def setUp(self):
super(ExampleParserTestCase, self).setUp()
self.example1 = example_pb2.Example()
text_format.Parse(
"features: { "
" feature: { "
" key: 'int_feature' "
" value: { "
" int64_list: { "
" value: [ 21, 2, 5 ] "
" } "
" } "
" } "
" feature: { "
" key: 'string_feature' "
" value: { "
" bytes_list: { "
" value: [ 'armadillo' ] "
" } "
" } "
" } "
"} ", self.example1)
self.example2 = example_pb2.Example()
text_format.Parse(
"features: { "
" feature: { "
" key: 'int_feature' "
" value: { "
" int64_list: { "
" value: [ 4, 5, 6 ] "
" } "
" } "
" } "
" feature: { "
" key: 'string_feature' "
" value: { "
" bytes_list: { "
" value: [ 'car', 'train' ] "
" } "
" } "
" } "
"} ", self.example2)
self.example_column = mocks.MockColumn(
"example",
tf.constant([
self.example1.SerializeToString(),
self.example2.SerializeToString()
],
dtype=tf.string,
shape=[2]))
self.features = (("string_feature", tf.VarLenFeature(dtype=tf.string)),
("int_feature",
tf.FixedLenFeature(shape=[3],
dtype=tf.int64,
default_value=[0, 0, 0])))
self.expected_string_values = np.array(
list(self.example1.features.feature["string_feature"].bytes_list.
value) + list(self.example2.features.
feature["string_feature"].bytes_list.value))
self.expected_string_indices = np.array([[0, 0], [1, 0], [1, 1]])
self.expected_int_feature = np.array([
list(self.example1.features.feature["int_feature"].int64_list.value
),
list(
self.example2.features.feature["int_feature"].int64_list.value)
])
def create_no_output_column():
return learn.TransformedColumn(
[mocks.MockColumn("foobar", [])],
mocks.MockZeroOutputTransform("thb", "nth"), None)