def read_norm_file(path): path = os.path.expanduser(path) if path.split(".")[-1] == "gz": with gzip.open(path) as f: norm_json = json.load(f) else: with open(path) as f: norm_json = json.load(f) return normalization.deserialize(norm_json)
def test_persistency(self): _, feature_value_map = preprocessing_util.read_data() normalization_parameters = {} for name, values in feature_value_map.items(): normalization_parameters[name] = normalization.identify_parameter(values) s = normalization.serialize(normalization_parameters) read_parameters = normalization.deserialize(s) self.assertEqual(read_parameters, normalization_parameters)
def read_norm_file(path) -> Dict[int, NormalizationParameters]: path = os.path.expanduser(path) if path.split(".")[-1] == "gz": with gzip.open(path) as f: norm_json = json.load(f) else: with open(path) as f: norm_json = json.load(f) return normalization.deserialize(norm_json)
def test_persistency(self): feature_value_map = read_data() normalization_parameters = {} for name, values in feature_value_map.items(): normalization_parameters[name] = normalization.identify_parameter( values, feature_type=self._feature_type_override(name)) s = normalization.serialize(normalization_parameters) read_parameters = normalization.deserialize(s) self.assertEqual(read_parameters, normalization_parameters)
def test_persistency(self): feature_value_map = preprocessing_util.read_data() normalization_parameters = normalization.identify_parameters( feature_value_map ) s = normalization.serialize(normalization_parameters) read_parameters = normalization.deserialize(s) self.assertEqual(read_parameters, normalization_parameters)
def test_persistency(self): feature_value_map = read_data() normalization_parameters = {} for name, values in feature_value_map.items(): normalization_parameters[name] = normalization.identify_parameter( values) values[ 0] = MISSING_VALUE # Set one entry to MISSING_VALUE to test that s = normalization.serialize(normalization_parameters) read_parameters = normalization.deserialize(s) self.assertEqual(read_parameters, normalization_parameters)
def test_persistency(self): feature_value_map = read_data() normalization_parameters = {} for name, values in feature_value_map.items(): normalization_parameters[name] = normalization.identify_parameter( name, values, feature_type=self._feature_type_override(name)) values[ 0] = MISSING_VALUE # Set one entry to MISSING_VALUE to test that s = normalization.serialize(normalization_parameters) read_parameters = normalization.deserialize(s) # Unfortunately, Thrift serializatin seems to lose a bit of precision. # Using `==` will be false. self.assertEqual(read_parameters.keys(), normalization_parameters.keys()) for k in normalization_parameters: self.assertEqual( read_parameters[k].feature_type, normalization_parameters[k].feature_type, ) self.assertEqual( read_parameters[k].possible_values, normalization_parameters[k].possible_values, ) for field in [ "boxcox_lambda", "boxcox_shift", "mean", "stddev", "quantiles", "min_value", "max_value", ]: if getattr(normalization_parameters[k], field) is None: self.assertEqual( getattr(read_parameters[k], field), getattr(normalization_parameters[k], field), ) else: npt.assert_allclose( getattr(read_parameters[k], field), getattr(normalization_parameters[k], field), )
def test_persistency(self): feature_value_map = read_data() normalization_parameters = {} for name, values in feature_value_map.items(): normalization_parameters[name] = normalization.identify_parameter( name, values, feature_type=self._feature_type_override(name) ) values[0] = MISSING_VALUE # Set one entry to MISSING_VALUE to test that s = normalization.serialize(normalization_parameters) read_parameters = normalization.deserialize(s) # Unfortunately, Thrift serializatin seems to lose a bit of precision. # Using `==` will be false. self.assertEqual(read_parameters.keys(), normalization_parameters.keys()) for k in normalization_parameters: self.assertEqual( read_parameters[k].feature_type, normalization_parameters[k].feature_type, ) self.assertEqual( read_parameters[k].possible_values, normalization_parameters[k].possible_values, ) for field in [ "boxcox_lambda", "boxcox_shift", "mean", "stddev", "quantiles", "min_value", "max_value", ]: if getattr(normalization_parameters[k], field) is None: self.assertEqual( getattr(read_parameters[k], field), getattr(normalization_parameters[k], field), ) else: npt.assert_allclose( getattr(read_parameters[k], field), getattr(normalization_parameters[k], field), )
def read_norm_file(path): path = os.path.expanduser(path) with open(path) as f: norm_json = json.load(f) return normalization.deserialize(norm_json)
def read_norm_params(table_output): norm_data = dict( zip(table_output["feature"], table_output["normalization"])) return normalization.deserialize(norm_data)