def test_persistency(self):
feature_value_map = preprocessing_util.read_data()
normalization_parameters = normalization.identify_parameters(
feature_value_map
)
s = normalization.serialize(normalization_parameters)
read_parameters = normalization.deserialize(s)
self.assertEqual(read_parameters, normalization_parameters)
def test_preprocessing_network(self):
feature_value_map = preprocessing_util.read_data()
types = identify_types.identify_types_dict(feature_value_map)
normalization_parameters = normalization.identify_parameters(
feature_value_map, types)
test_features = self.preprocess(feature_value_map,
normalization_parameters)
test_features[u'186'] = 0
net = core.Net("PreprocessingTestNet")
preprocessor = PreprocessorNet(net, False)
for feature_name in feature_value_map:
workspace.FeedBlob(feature_name, np.array([0], dtype=np.int32))
preprocessor.preprocess_blob(
feature_name, normalization_parameters[feature_name])
workspace.CreateNet(net)
for feature_name in feature_value_map:
if feature_name != u'186':
workspace.FeedBlob(
feature_name,
feature_value_map[feature_name].astype(np.float32))
else:
workspace.FeedBlob(
feature_name,
normalization.MISSING_VALUE * np.ones(1, dtype=np.float32))
workspace.RunNetOnce(net)
for feature_name in feature_value_map:
normalized_features = workspace.FetchBlob(feature_name +
"_preprocessed")
self.assertTrue(
np.all(
np.isclose(normalized_features,
test_features[feature_name])))
for feature_name in feature_value_map:
if feature_name != u'186':
workspace.FeedBlob(
feature_name,
feature_value_map[feature_name].astype(np.float32))
else:
workspace.FeedBlob(
feature_name,
normalization.MISSING_VALUE * np.ones(1, dtype=np.float32))
workspace.RunNetOnce(net)
for feature_name in feature_value_map:
normalized_features = workspace.FetchBlob(feature_name +
"_preprocessed")
self.assertTrue(
np.all(
np.isclose(normalized_features,
test_features[feature_name])))
def test_preprocessing_network(self):
feature_value_map = preprocessing_util.read_data()
normalization_parameters = normalization.identify_parameters(
feature_value_map
)
test_features = self.preprocess(
feature_value_map, normalization_parameters
)
net = core.Net("PreprocessingTestNet")
preprocessor = PreprocessorNet(net, False)
for feature_name in feature_value_map:
workspace.FeedBlob(feature_name, np.array([0], dtype=np.int32))
preprocessor.preprocess_blob(
feature_name, normalization_parameters[feature_name]
)
workspace.CreateNet(net)
for feature_name in feature_value_map:
workspace.FeedBlob(feature_name, feature_value_map[feature_name])
workspace.RunNetOnce(net)
for feature_name in feature_value_map:
normalized_features = workspace.FetchBlob(
feature_name + "_preprocessed"
)
tolerance = 0.01
if feature_name == 'boxcox':
# At the limit, boxcox has some numerical instability
tolerance = 0.1
non_matching = np.where(
np.logical_not(
np.isclose(
normalized_features,
test_features[feature_name],
rtol=tolerance,
atol=tolerance,
)
)
)
self.assertTrue(
np.all(
np.isclose(
normalized_features,
test_features[feature_name],
rtol=tolerance,
atol=tolerance,
)
), '{} does not match: {} {}'.format(
feature_name, normalized_features[non_matching].tolist(),
test_features[feature_name][non_matching].tolist()
)
)
def test_persistency(self):
feature_value_map = preprocessing_util.read_data()
types = identify_types.identify_types_dict(feature_value_map)
normalization_parameters = normalization.identify_parameters(
feature_value_map, types)
with io.StringIO() as f:
normalization.write_parameters(f, normalization_parameters)
f.seek(0)
read_parameters = normalization.load_parameters(f)
self.assertEqual(read_parameters, normalization_parameters)
def test_prepare_normalization_and_normalize(self):
feature_value_map = preprocessing_util.read_data()
types = identify_types.identify_types(feature_value_map)
types_dict = identify_types.identify_types_dict(feature_value_map)
normalization_parameters = normalization.identify_parameters(
feature_value_map, types_dict)
features = list(feature_value_map.keys())
norm_net = core.Net("net")
blobname_template = '{}_blob'
blob_map = prepare_normalization(norm_net, normalization_parameters,
features, blobname_template, False)
normalized_features = normalize_feature_map(feature_value_map,
norm_net, features,
blob_map,
blobname_template)
self.assertTrue(
all([
np.isfinite(parameter.stddev) and np.isfinite(parameter.mean)
for parameter in normalization_parameters.values()
]))
for k, v in six.iteritems(normalized_features):
self.assertTrue(np.all(np.isfinite(v)))
feature_type = normalization_parameters[k].feature_type
if feature_type == identify_types.PROBABILITY:
sigmoidv = special.expit(v)
self.assertTrue(
np.all(
np.logical_and(np.greater(sigmoidv, 0),
np.less(sigmoidv, 1))))
elif feature_type == identify_types.ENUM:
possible_values = normalization_parameters[k].possible_values
self.assertEqual(v.shape[0], len(feature_value_map[k]))
self.assertEqual(v.shape[1], len(possible_values))
possible_value_map = {}
for i, possible_value in enumerate(possible_values):
possible_value_map[possible_value] = i
for i, row in enumerate(v):
original_feature = feature_value_map[k][i]
self.assertEqual(possible_value_map[original_feature],
np.where(row == 1)[0][0])
else:
one_stddev = np.isclose(np.std(v, ddof=1), 1, atol=0.00001)
zero_stddev = np.isclose(np.std(v, ddof=1), 0, atol=0.00001)
zero_mean = np.isclose(np.mean(v), 0, atol=0.00001)
is_binary = types[k] == identify_types.BINARY
self.assertTrue(np.all(np.logical_or(zero_mean, is_binary)))
self.assertTrue(
np.all(
np.logical_or(np.logical_or(one_stddev, zero_stddev),
is_binary)))
has_boxcox = normalization_parameters[
k].boxcox_lambda is not None
is_ctd = types[k] == identify_types.CONTINUOUS
# This should be true at the moment
self.assertTrue(is_ctd == has_boxcox)
def test_prepare_normalization_and_normalize(self):
feature_value_map = preprocessing_util.read_data()
normalization_parameters = normalization.identify_parameters(
feature_value_map, 10
)
for k, v in normalization_parameters.items():
if k == 'normal':
self.assertEqual(v.feature_type, 'CONTINUOUS')
self.assertIs(v.boxcox_lambda, None)
self.assertIs(v.boxcox_shift, None)
elif k == 'boxcox':
self.assertEqual(v.feature_type, 'CONTINUOUS')
self.assertIsNot(v.boxcox_lambda, None)
self.assertIsNot(v.boxcox_shift, None)
else:
self.assertEqual(v.feature_type, k)
features = list(feature_value_map.keys())
norm_net = core.Net("net")
blobname_template = '{}_blob'
blob_map = prepare_normalization(
norm_net, normalization_parameters, features, blobname_template,
False
)
normalized_features = normalize_feature_map(
feature_value_map, norm_net, features, blob_map, blobname_template
)
self.assertTrue(
all(
[
np.isfinite(parameter.stddev) and
np.isfinite(parameter.mean)
for parameter in normalization_parameters.values()
]
)
)
for k, v in six.iteritems(normalized_features):
self.assertTrue(np.all(np.isfinite(v)))
feature_type = normalization_parameters[k].feature_type
if feature_type == identify_types.PROBABILITY:
sigmoidv = special.expit(v)
self.assertTrue(
np.all(
np.logical_and(
np.greater(sigmoidv, 0), np.less(sigmoidv, 1)
)
)
)
elif feature_type == identify_types.ENUM:
possible_values = normalization_parameters[k].possible_values
self.assertEqual(v.shape[0], len(feature_value_map[k]))
self.assertEqual(v.shape[1], len(possible_values))
possible_value_map = {}
for i, possible_value in enumerate(possible_values):
possible_value_map[possible_value] = i
for i, row in enumerate(v):
original_feature = feature_value_map[k][i]
self.assertEqual(
possible_value_map[original_feature],
np.where(row == 1)[0][0]
)
elif feature_type == identify_types.QUANTILE:
quantiles = normalization_parameters[k].quantiles
for i, feature in enumerate(v[0]):
original_feature = feature_value_map[k][i]
count = 0
for quantile in quantiles:
if original_feature >= quantile:
count += 1
count /= float(len(quantiles))
self.assertAlmostEqual(feature, count, 2)
elif feature_type == identify_types.BINARY:
pass
elif feature_type == identify_types.CONTINUOUS:
one_stddev = np.isclose(np.std(v, ddof=1), 1, atol=0.01)
zero_stddev = np.isclose(np.std(v, ddof=1), 0, atol=0.01)
zero_mean = np.isclose(np.mean(v), 0, atol=0.01)
self.assertTrue(
np.all(zero_mean),
'mean of feature {} is {}, not 0'.format(k, np.mean(v))
)
self.assertTrue(np.all(np.logical_or(one_stddev, zero_stddev)))
else:
raise NotImplementedError()