def test_convert_column_name_value_to_id_multiple(self): ohe = Mock() ohe.transform.return_value = np.array( [[0, 1, 0] # one hot encoding, second dimension ]) transformer = DataTransformer() transformer._column_transform_info_list = [ ColumnTransformInfo( column_name='x', column_type='continuous', transform=None, transform_aux=None, output_info=[SpanInfo(1, 'tanh'), SpanInfo(3, 'softmax')], output_dimensions=1 + 3), ColumnTransformInfo(column_name='y', column_type='discrete', transform=ohe, transform_aux=None, output_info=[SpanInfo(2, 'softmax')], output_dimensions=2), ColumnTransformInfo(column_name='z', column_type='discrete', transform=ohe, transform_aux=None, output_info=[SpanInfo(2, 'softmax')], output_dimensions=2) ] result = transformer.convert_column_name_value_to_id('z', 'yes') assert result['column_id'] == 2 # this is the 3rd column assert result[ 'discrete_column_id'] == 1 # this is the 2nd discrete column assert result[ 'value_id'] == 1 # this is the 1st dimension in the one hot encoding
def test_convert_column_name_value_to_id(self): """Test ``convert_column_name_value_to_id`` on a simple ``_column_transform_info_list``. Tests that the appropriate indexes are returned when a table of three columns, discrete, continuous, discrete, is passed as '_column_transform_info_list'. Setup: - Mock ``_column_transform_info_list``. Input: - column_name = the name of a discrete column - value = the categorical value Output: - dictionary containing: - ``discrete_column_id`` = the index of the target column, when considering only discrete columns - ``column_id`` = the index of the target column (e.g. 3 = the third column in the data) - ``value_id`` = the index of the indicator value in the one-hot encoding """ # Setup ohe = Mock() ohe.transform.return_value = pd.DataFrame( [[0, 1] # one hot encoding, second dimension ]) transformer = DataTransformer() transformer._column_transform_info_list = [ ColumnTransformInfo( column_name='x', column_type='continuous', transform=None, output_info=[SpanInfo(1, 'tanh'), SpanInfo(3, 'softmax')], output_dimensions=1 + 3), ColumnTransformInfo(column_name='y', column_type='discrete', transform=ohe, output_info=[SpanInfo(2, 'softmax')], output_dimensions=2) ] # Run result = transformer.convert_column_name_value_to_id('y', 'yes') # Assert assert result['column_id'] == 1 # this is the 2nd column assert result[ 'discrete_column_id'] == 0 # this is the 1st discrete column assert result[ 'value_id'] == 1 # this is the 2nd dimension in the one hot encoding
def test_transform(self): """Test 'transform' on a dataframe with one continuous and one discrete columns. It should use the appropriate '_transform' type for each column and should return them concanenated appropriately. Setup: - Mock _column_transform_info_list - Mock _transform_discrete - Mock _transform_continuous Input: - raw_data = a table with one continuous and one discrete columns. Output: - numpy array containing the transformed two columns Side Effects: - _transform_discrete and _transform_continuous should each be called once. """ data = pd.DataFrame({ "x": np.array([0.1, 0.3, 0.5]), "y": np.array(["yes", "yes", "no"]) }) transformer = DataTransformer() transformer._column_transform_info_list = [ ColumnTransformInfo( column_name="x", column_type="continuous", transform=None, transform_aux=None, output_info=[SpanInfo(1, 'tanh'), SpanInfo(3, 'softmax')], output_dimensions=1 + 3), ColumnTransformInfo(column_name="y", column_type="discrete", transform=None, transform_aux=None, output_info=[SpanInfo(2, 'softmax')], output_dimensions=2) ] transformer._transform_continuous = Mock() selected_normalized_value = np.array([[0.1], [0.3], [0.5]]) selected_component_onehot = np.array([ [1, 0, 0], [1, 0, 0], [1, 0, 0], ]) return_value = (selected_normalized_value, selected_component_onehot) transformer._transform_continuous.return_value = return_value transformer._transform_discrete = Mock() transformer._transform_discrete.return_value = [ np.array([ [0, 1], [0, 1], [1, 0], ]) ] result = transformer.transform(data) transformer._transform_continuous.assert_called_once() transformer._transform_discrete.assert_called_once() expected = np.array([ [0.1, 1, 0, 0, 0, 1], [0.3, 1, 0, 0, 0, 1], [0.5, 1, 0, 0, 1, 0], ]) assert result.shape == (3, 6) assert (result[:, 0] == expected[:, 0]).all(), "continuous-cdf" assert (result[:, 1:4] == expected[:, 1:4]).all(), "continuous-softmax" assert (result[:, 4:6] == expected[:, 4:6]).all(), "discrete"
def test_transform(self): """Test ``transform`` on a dataframe with one continuous and one discrete columns. It should use the appropriate ``_transform`` type for each column and should return them concanenated appropriately. Setup: - Initialize a ``DataTransformer`` with a ``column_transform_info`` detailing a continuous and a discrete columns. - Mock the ``_transform_discrete`` and ``_transform_continuous`` methods. Input: - A table with one continuous and one discrete columns. Output: - np.array containing the transformed columns. Side Effects: - ``_transform_discrete`` and ``_transform_continuous`` should each be called once. """ # Setup data = pd.DataFrame({ 'x': np.array([0.1, 0.3, 0.5]), 'y': np.array(['yes', 'yes', 'no']) }) transformer = DataTransformer() transformer._column_transform_info_list = [ ColumnTransformInfo( column_name='x', column_type='continuous', transform=None, output_info=[SpanInfo(1, 'tanh'), SpanInfo(3, 'softmax')], output_dimensions=1 + 3), ColumnTransformInfo(column_name='y', column_type='discrete', transform=None, output_info=[SpanInfo(2, 'softmax')], output_dimensions=2) ] transformer._transform_continuous = Mock() selected_normalized_value = np.array([[0.1], [0.3], [0.5]]) selected_component_onehot = np.array([ [1, 0, 0], [0, 1, 0], [0, 1, 0], ]) return_value = np.concatenate( (selected_normalized_value, selected_component_onehot), axis=1) transformer._transform_continuous.return_value = return_value transformer._transform_discrete = Mock() transformer._transform_discrete.return_value = np.array([ [0, 1], [0, 1], [1, 0], ]) # Run result = transformer.transform(data) # Assert transformer._transform_continuous.assert_called_once() transformer._transform_discrete.assert_called_once() expected = np.array([ [0.1, 1, 0, 0, 0, 1], [0.3, 0, 1, 0, 0, 1], [0.5, 0, 1, 0, 1, 0], ]) assert result.shape == (3, 6) assert (result[:, 0] == expected[:, 0]).all(), 'continuous-cdf' assert (result[:, 1:4] == expected[:, 1:4]).all(), 'continuous-softmax' assert (result[:, 4:6] == expected[:, 4:6]).all(), 'discrete'