def test_convert_column_name_value_to_id_multiple(self):
ohe = Mock()
ohe.transform.return_value = np.array(
[[0, 1, 0] # one hot encoding, second dimension
])
transformer = DataTransformer()
transformer._column_transform_info_list = [
ColumnTransformInfo(
column_name='x',
column_type='continuous',
transform=None,
transform_aux=None,
output_info=[SpanInfo(1, 'tanh'),
SpanInfo(3, 'softmax')],
output_dimensions=1 + 3),
ColumnTransformInfo(column_name='y',
column_type='discrete',
transform=ohe,
transform_aux=None,
output_info=[SpanInfo(2, 'softmax')],
output_dimensions=2),
ColumnTransformInfo(column_name='z',
column_type='discrete',
transform=ohe,
transform_aux=None,
output_info=[SpanInfo(2, 'softmax')],
output_dimensions=2)
]
result = transformer.convert_column_name_value_to_id('z', 'yes')
assert result['column_id'] == 2 # this is the 3rd column
assert result[
'discrete_column_id'] == 1 # this is the 2nd discrete column
assert result[
'value_id'] == 1 # this is the 1st dimension in the one hot encoding
def test_convert_column_name_value_to_id(self):
"""Test ``convert_column_name_value_to_id`` on a simple ``_column_transform_info_list``.
Tests that the appropriate indexes are returned when a table of three columns,
discrete, continuous, discrete, is passed as '_column_transform_info_list'.
Setup:
- Mock ``_column_transform_info_list``.
Input:
- column_name = the name of a discrete column
- value = the categorical value
Output:
- dictionary containing:
- ``discrete_column_id`` = the index of the target column,
when considering only discrete columns
- ``column_id`` = the index of the target column
(e.g. 3 = the third column in the data)
- ``value_id`` = the index of the indicator value in the one-hot encoding
"""
# Setup
ohe = Mock()
ohe.transform.return_value = pd.DataFrame(
[[0, 1] # one hot encoding, second dimension
])
transformer = DataTransformer()
transformer._column_transform_info_list = [
ColumnTransformInfo(
column_name='x',
column_type='continuous',
transform=None,
output_info=[SpanInfo(1, 'tanh'),
SpanInfo(3, 'softmax')],
output_dimensions=1 + 3),
ColumnTransformInfo(column_name='y',
column_type='discrete',
transform=ohe,
output_info=[SpanInfo(2, 'softmax')],
output_dimensions=2)
]
# Run
result = transformer.convert_column_name_value_to_id('y', 'yes')
# Assert
assert result['column_id'] == 1 # this is the 2nd column
assert result[
'discrete_column_id'] == 0 # this is the 1st discrete column
assert result[
'value_id'] == 1 # this is the 2nd dimension in the one hot encoding
def test_transform(self):
"""Test 'transform' on a dataframe with one continuous and one discrete columns.
It should use the appropriate '_transform' type for each column and should return
them concanenated appropriately.
Setup:
- Mock _column_transform_info_list
- Mock _transform_discrete
- Mock _transform_continuous
Input:
- raw_data = a table with one continuous and one discrete columns.
Output:
- numpy array containing the transformed two columns
Side Effects:
- _transform_discrete and _transform_continuous should each be called once.
"""
data = pd.DataFrame({
"x": np.array([0.1, 0.3, 0.5]),
"y": np.array(["yes", "yes", "no"])
})
transformer = DataTransformer()
transformer._column_transform_info_list = [
ColumnTransformInfo(
column_name="x",
column_type="continuous",
transform=None,
transform_aux=None,
output_info=[SpanInfo(1, 'tanh'),
SpanInfo(3, 'softmax')],
output_dimensions=1 + 3),
ColumnTransformInfo(column_name="y",
column_type="discrete",
transform=None,
transform_aux=None,
output_info=[SpanInfo(2, 'softmax')],
output_dimensions=2)
]
transformer._transform_continuous = Mock()
selected_normalized_value = np.array([[0.1], [0.3], [0.5]])
selected_component_onehot = np.array([
[1, 0, 0],
[1, 0, 0],
[1, 0, 0],
])
return_value = (selected_normalized_value, selected_component_onehot)
transformer._transform_continuous.return_value = return_value
transformer._transform_discrete = Mock()
transformer._transform_discrete.return_value = [
np.array([
[0, 1],
[0, 1],
[1, 0],
])
]
result = transformer.transform(data)
transformer._transform_continuous.assert_called_once()
transformer._transform_discrete.assert_called_once()
expected = np.array([
[0.1, 1, 0, 0, 0, 1],
[0.3, 1, 0, 0, 0, 1],
[0.5, 1, 0, 0, 1, 0],
])
assert result.shape == (3, 6)
assert (result[:, 0] == expected[:, 0]).all(), "continuous-cdf"
assert (result[:, 1:4] == expected[:, 1:4]).all(), "continuous-softmax"
assert (result[:, 4:6] == expected[:, 4:6]).all(), "discrete"
def test_transform(self):
"""Test ``transform`` on a dataframe with one continuous and one discrete columns.
It should use the appropriate ``_transform`` type for each column and should return
them concanenated appropriately.
Setup:
- Initialize a ``DataTransformer`` with a ``column_transform_info`` detailing
a continuous and a discrete columns.
- Mock the ``_transform_discrete`` and ``_transform_continuous`` methods.
Input:
- A table with one continuous and one discrete columns.
Output:
- np.array containing the transformed columns.
Side Effects:
- ``_transform_discrete`` and ``_transform_continuous`` should each be called once.
"""
# Setup
data = pd.DataFrame({
'x': np.array([0.1, 0.3, 0.5]),
'y': np.array(['yes', 'yes', 'no'])
})
transformer = DataTransformer()
transformer._column_transform_info_list = [
ColumnTransformInfo(
column_name='x',
column_type='continuous',
transform=None,
output_info=[SpanInfo(1, 'tanh'),
SpanInfo(3, 'softmax')],
output_dimensions=1 + 3),
ColumnTransformInfo(column_name='y',
column_type='discrete',
transform=None,
output_info=[SpanInfo(2, 'softmax')],
output_dimensions=2)
]
transformer._transform_continuous = Mock()
selected_normalized_value = np.array([[0.1], [0.3], [0.5]])
selected_component_onehot = np.array([
[1, 0, 0],
[0, 1, 0],
[0, 1, 0],
])
return_value = np.concatenate(
(selected_normalized_value, selected_component_onehot), axis=1)
transformer._transform_continuous.return_value = return_value
transformer._transform_discrete = Mock()
transformer._transform_discrete.return_value = np.array([
[0, 1],
[0, 1],
[1, 0],
])
# Run
result = transformer.transform(data)
# Assert
transformer._transform_continuous.assert_called_once()
transformer._transform_discrete.assert_called_once()
expected = np.array([
[0.1, 1, 0, 0, 0, 1],
[0.3, 0, 1, 0, 0, 1],
[0.5, 0, 1, 0, 1, 0],
])
assert result.shape == (3, 6)
assert (result[:, 0] == expected[:, 0]).all(), 'continuous-cdf'
assert (result[:, 1:4] == expected[:, 1:4]).all(), 'continuous-softmax'
assert (result[:, 4:6] == expected[:, 4:6]).all(), 'discrete'