def test_preprocessing_network(self):
feature_value_map = read_data()
normalization_parameters = {}
for name, values in feature_value_map.items():
normalization_parameters[name] = normalization.identify_parameter(
name, values, feature_type=self._feature_type_override(name))
test_features = NumpyFeatureProcessor.preprocess(
feature_value_map, normalization_parameters)
net = core.Net("PreprocessingTestNet")
C2.set_net(net)
preprocessor = PreprocessorNet()
name_preprocessed_blob_map = {}
for feature_name in feature_value_map:
workspace.FeedBlob(str(feature_name), np.array([0],
dtype=np.int32))
preprocessed_blob, _ = preprocessor.preprocess_blob(
str(feature_name), [normalization_parameters[feature_name]])
name_preprocessed_blob_map[feature_name] = preprocessed_blob
workspace.CreateNet(net)
for feature_name, feature_value in six.iteritems(feature_value_map):
feature_value = np.expand_dims(feature_value, -1)
workspace.FeedBlob(str(feature_name), feature_value)
workspace.RunNetOnce(net)
for feature_name in feature_value_map:
normalized_features = workspace.FetchBlob(
name_preprocessed_blob_map[feature_name])
if feature_name != ENUM_FEATURE_ID:
normalized_features = np.squeeze(normalized_features, -1)
tolerance = 0.01
if feature_name == BOXCOX_FEATURE_ID:
# At the limit, boxcox has some numerical instability
tolerance = 0.5
non_matching = np.where(
np.logical_not(
np.isclose(
normalized_features,
test_features[feature_name],
rtol=tolerance,
atol=tolerance,
)))
self.assertTrue(
np.all(
np.isclose(
normalized_features,
test_features[feature_name],
rtol=tolerance,
atol=tolerance,
)),
"{} does not match: {} {}".format(
feature_name,
normalized_features[non_matching].tolist(),
test_features[feature_name][non_matching].tolist(),
),
)
def test_preprocessing_network(self):
feature_value_map = read_data()
normalization_parameters = {}
name_preprocessed_blob_map = {}
for feature_name, feature_values in feature_value_map.items():
normalization_parameters[
feature_name] = normalization.identify_parameter(
feature_values,
feature_type=self._feature_type_override(feature_name))
feature_values[
0] = MISSING_VALUE # Set one entry to MISSING_VALUE to test that
preprocessor = Preprocessor(
{feature_name: normalization_parameters[feature_name]}, False)
feature_values_matrix = np.expand_dims(feature_values, -1)
normalized_feature_values = preprocessor.forward(
feature_values_matrix)
name_preprocessed_blob_map[
feature_name] = normalized_feature_values.numpy()
test_features = NumpyFeatureProcessor.preprocess(
feature_value_map, normalization_parameters)
for feature_name in feature_value_map:
normalized_features = name_preprocessed_blob_map[feature_name]
if feature_name != ENUM_FEATURE_ID:
normalized_features = np.squeeze(normalized_features, -1)
tolerance = 0.01
if feature_name == BOXCOX_FEATURE_ID:
# At the limit, boxcox has some numerical instability
tolerance = 0.5
non_matching = np.where(
np.logical_not(
np.isclose(
normalized_features.flatten(),
test_features[feature_name].flatten(),
rtol=tolerance,
atol=tolerance,
)))
self.assertTrue(
np.all(
np.isclose(
normalized_features.flatten(),
test_features[feature_name].flatten(),
rtol=tolerance,
atol=tolerance,
)),
"{} does not match: {} \n!=\n {}".format(
feature_name,
normalized_features.flatten()[non_matching],
test_features[feature_name].flatten()[non_matching],
),
)
def test_preprocessing_network(self):
feature_value_map = read_data()
normalization_parameters = {}
for name, values in feature_value_map.items():
normalization_parameters[name] = normalization.identify_parameter(
name, values, feature_type=self._feature_type_override(name)
)
test_features = NumpyFeatureProcessor.preprocess(
feature_value_map, normalization_parameters
)
net = core.Net("PreprocessingTestNet")
C2.set_net(net)
preprocessor = PreprocessorNet()
name_preprocessed_blob_map = {}
for feature_name in feature_value_map:
workspace.FeedBlob(str(feature_name), np.array([0], dtype=np.int32))
preprocessed_blob, _ = preprocessor.preprocess_blob(
str(feature_name), [normalization_parameters[feature_name]]
)
name_preprocessed_blob_map[feature_name] = preprocessed_blob
workspace.CreateNet(net)
for feature_name, feature_value in six.iteritems(feature_value_map):
feature_value = np.expand_dims(feature_value, -1)
workspace.FeedBlob(str(feature_name), feature_value)
workspace.RunNetOnce(net)
for feature_name in feature_value_map:
normalized_features = workspace.FetchBlob(
name_preprocessed_blob_map[feature_name]
)
if feature_name != ENUM_FEATURE_ID:
normalized_features = np.squeeze(normalized_features, -1)
tolerance = 0.01
if feature_name == BOXCOX_FEATURE_ID:
# At the limit, boxcox has some numerical instability
tolerance = 0.5
non_matching = np.where(
np.logical_not(
np.isclose(
normalized_features,
test_features[feature_name],
rtol=tolerance,
atol=tolerance,
)
)
)
self.assertTrue(
np.all(
np.isclose(
normalized_features,
test_features[feature_name],
rtol=tolerance,
atol=tolerance,
)
),
"{} does not match: {} {}".format(
feature_name,
normalized_features[non_matching].tolist(),
test_features[feature_name][non_matching].tolist(),
),
)
def test_preprocessing_network(self):
feature_value_map = read_data()
normalization_parameters = {}
name_preprocessed_blob_map = {}
for feature_name, feature_values in feature_value_map.items():
normalization_parameters[feature_name] = normalization.identify_parameter(
feature_name,
feature_values,
feature_type=self._feature_type_override(feature_name),
)
feature_values[
0
] = MISSING_VALUE # Set one entry to MISSING_VALUE to test that
preprocessor = Preprocessor(
{feature_name: normalization_parameters[feature_name]}, False
)
feature_values_matrix = np.expand_dims(feature_values, -1)
normalized_feature_values = preprocessor.forward(feature_values_matrix)
name_preprocessed_blob_map[feature_name] = normalized_feature_values.numpy()
test_features = NumpyFeatureProcessor.preprocess(
feature_value_map, normalization_parameters
)
for feature_name in feature_value_map:
normalized_features = name_preprocessed_blob_map[feature_name]
if feature_name != ENUM_FEATURE_ID:
normalized_features = np.squeeze(normalized_features, -1)
tolerance = 0.01
if feature_name == BOXCOX_FEATURE_ID:
# At the limit, boxcox has some numerical instability
tolerance = 0.5
non_matching = np.where(
np.logical_not(
np.isclose(
normalized_features.flatten(),
test_features[feature_name].flatten(),
rtol=tolerance,
atol=tolerance,
)
)
)
self.assertTrue(
np.all(
np.isclose(
normalized_features.flatten(),
test_features[feature_name].flatten(),
rtol=tolerance,
atol=tolerance,
)
),
"{} does not match: {} \n!=\n {}".format(
feature_name,
normalized_features.flatten()[non_matching],
test_features[feature_name].flatten()[non_matching],
),
)