def test__analyze(self):
"""Test _analyze"""
# Setup
hp = HyperTransformer(dtype_transformers={'O': 'one_hot_encoding'})
# Run
data = pd.DataFrame({
'int': [1, 2, None],
'float': [1.0, 2.0, None],
'object': ['foo', 'bar', None],
'category': [1, 2, None],
'bool': [True, False, None],
'datetime': pd.to_datetime(['1965-05-23', None, '1997-10-17']),
})
data['category'] = data['category'].astype('category')
result = hp._analyze(data)
# Asserts
assert isinstance(result, dict)
assert set(result.keys()) == {'int', 'float', 'object', 'category', 'bool', 'datetime'}
assert isinstance(result['int'], NumericalTransformer)
assert isinstance(result['float'], NumericalTransformer)
assert isinstance(result['object'], OneHotEncodingTransformer)
assert isinstance(result['category'], OneHotEncodingTransformer)
assert isinstance(result['bool'], BooleanTransformer)
assert isinstance(result['datetime'], DatetimeTransformer)
def _fit(self, table_data):
"""Fit the model to the table.
Args:
table_data (pandas.DataFrame):
Data to be learned.
"""
distributions = self._field_distributions
fields = self._metadata.get_fields()
transformers = {}
for field in table_data:
field_name = field.replace('.value', '')
if field_name in fields and fields.get(
field_name,
dict(),
).get('type') != 'categorical':
transformers[field] = GaussianCopulaTransformer(
distribution=distributions.get(field_name,
self._default_distribution))
self._ht = HyperTransformer(field_transformers=transformers)
table_data = self._ht.fit_transform(table_data)
super()._fit(table_data)
def test_fit_transform(self):
"""Test call fit_transform"""
# Run
transformer = Mock()
HyperTransformer.fit_transform(transformer, pd.DataFrame())
# Asserts
expect_call_count_fit = 1
expect_call_count_transform = 1
expect_call_args_fit = pd.DataFrame()
expect_call_args_transform = pd.DataFrame()
self.assertEqual(
transformer.fit.call_count,
expect_call_count_fit
)
pd.testing.assert_frame_equal(
transformer.fit.call_args[0][0],
expect_call_args_fit
)
self.assertEqual(
transformer.transform.call_count,
expect_call_count_transform
)
pd.testing.assert_frame_equal(
transformer.transform.call_args[0][0],
expect_call_args_transform
)
def test_fit_with_analyze(self):
"""Test fit and analyze the transformers"""
# Setup
data = pd.DataFrame({
'integers': [1, 2, 3, 4],
'floats': [1.1, 2.2, 3.3, 4.4],
'booleans': [True, False, False, True]
})
int_mock = Mock()
float_mock = Mock()
bool_mock = Mock()
analyzed_data = {
'integers': int_mock,
'floats': float_mock,
'booleans': bool_mock
}
# Run
transformer = Mock()
transformer.transformers = None
transformer._analyze.return_value = analyzed_data
HyperTransformer.fit(transformer, data)
# Asserts
expect_int_call_count = 1
expect_float_call_count = 1
expect_bool_call_count = 1
self.assertEqual(int_mock.fit.call_count, expect_int_call_count)
self.assertEqual(float_mock.fit.call_count, expect_float_call_count)
self.assertEqual(bool_mock.fit.call_count, expect_bool_call_count)
def compute(cls, real_data, synthetic_data, metadata=None):
"""Compute this metric.
Args:
real_data (pandas.DataFrame):
The values from the real dataset.
synthetic_data (pandas.DataFrame):
The values from the synthetic dataset.
metadata (dict):
Table metadata dict.
Returns:
Union[float, tuple[float]]:
Metric output.
"""
metadata = cls._validate_inputs(real_data, synthetic_data, metadata)
transformer = HyperTransformer()
fields = cls._select_fields(metadata, cls.field_types)
real_data = transformer.fit_transform(real_data[fields])
synthetic_data = transformer.transform(synthetic_data[fields])
values = []
for column_name, real_column in real_data.items():
real_column = real_column.values
synthetic_column = synthetic_data[column_name].values
score = cls.single_column_metric.compute(real_column,
synthetic_column)
values.append(score)
return np.nanmean(values)
def test__analyze_invalid_dtype(self):
"""Test _analyze when a list of dtypes containing an invalid dtype is passed."""
# Setup
hp = HyperTransformer(dtypes=['int', 'complex'])
# Run
data = pd.DataFrame({
'int': [1, 2, None],
'complex': [1.0 + 0j, 2.0 + 1j, None],
})
with pytest.raises(ValueError):
hp._analyze(data)
def test__analyze_raise_error(self):
"""Test _analyze raise error"""
# Setup
data = Mock()
data.columns = ['foo']
dtypes = [Mock()]
# Run
transformer = Mock()
transformer.dtypes = dtypes
with self.assertRaises(ValueError):
HyperTransformer._analyze(transformer, data)
def compute(cls, real_data, synthetic_data, metadata=None):
"""Compute this metric.
This builds a Machine Learning Classifier that learns to tell the synthetic
data apart from the real data, which later on is evaluated using Cross Validation.
The output of the metric is one minus the average ROC AUC score obtained.
Args:
real_data (Union[numpy.ndarray, pandas.DataFrame]):
The values from the real dataset.
synthetic_data (Union[numpy.ndarray, pandas.DataFrame]):
The values from the synthetic dataset.
metadata (dict):
Table metadata dict. If not passed, it is build based on the
real_data fields and dtypes.
Returns:
float:
One minus the ROC AUC Cross Validation Score obtained by the classifier.
"""
metadata = cls._validate_inputs(real_data, synthetic_data, metadata)
transformer = HyperTransformer(
default_data_type_transformers={
'categorical': OneHotEncodingTransformer(
error_on_unknown=False),
})
real_data = transformer.fit_transform(real_data).to_numpy()
synthetic_data = transformer.transform(synthetic_data).to_numpy()
X = np.concatenate([real_data, synthetic_data])
y = np.hstack([np.ones(len(real_data)), np.zeros(len(synthetic_data))])
if np.isin(X, [np.inf, -np.inf]).any():
X[np.isin(X, [np.inf, -np.inf])] = np.nan
try:
scores = []
kf = StratifiedKFold(n_splits=3, shuffle=True)
for train_index, test_index in kf.split(X, y):
y_pred = cls._fit_predict(X[train_index], y[train_index],
X[test_index])
roc_auc = roc_auc_score(y[test_index], y_pred)
scores.append(max(0.5, roc_auc) * 2 - 1)
return 1 - np.mean(scores)
except ValueError as err:
raise IncomputableMetricError(
f'DetectionMetric: Unable to be fit with error {err}')
def test__get_columns_one(self):
data = pd.DataFrame({
'a': [1, 2, 3],
})
returned = HyperTransformer._get_columns(data, 'a')
np.testing.assert_equal(returned, np.array([1, 2, 3]))
def test__get_columns_none(self):
data = pd.DataFrame({
'a': [1, 2, 3],
})
returned = HyperTransformer._get_columns(data, 'b')
assert returned.empty
def test___init__(self):
"""Test create new instance of HyperTransformer"""
# Run
ht = HyperTransformer()
# Asserts
self.assertTrue(ht.copy)
self.assertEqual(ht.dtypes, None)
def fit(self, table_data):
"""Fit the ``ColumnsModel``.
Fit a ``GaussianUnivariate`` model to the ``self.constraint_column`` columns in the
``table_data`` in order to sample those columns when missing.
Args:
table_data (pandas.DataFrame):
Table data.
"""
data_to_model = table_data[self.constraint_columns]
self._hyper_transformer = HyperTransformer(
default_data_type_transformers={
'categorical': 'OneHotEncodingTransformer'
})
transformed_data = self._hyper_transformer.fit_transform(data_to_model)
self._model = GaussianMultivariate(distribution=GaussianUnivariate)
self._model.fit(transformed_data)
def test__get_columns_two(self):
data = pd.DataFrame({
'b': [4, 5, 6],
'b#1': [7, 8, 9],
})
returned = HyperTransformer._get_columns(data, 'b')
expected = np.array([[4, 7], [5, 8], [6, 9]])
np.testing.assert_equal(returned, expected)
def _fit(self, table_data):
"""Fit the model to the table.
Args:
table_data (pandas.DataFrame):
Data to be learned.
"""
distributions = self._field_distributions
default = self._default_distribution
fields = self._metadata.get_fields()
transformers = {
field: GaussianCopulaTransformer(
distribution=distributions.get(field, default))
for field in table_data.columns
if fields.get(field, dict()).get('type') != 'categorical'
}
self._ht = HyperTransformer(transformers=transformers)
table_data = self._ht.fit_transform(table_data)
super()._fit(table_data)
def test_hypertransformer_with_transformers(faker_mock):
faker_mock.return_value.first_name.side_effect = [
'Jaime', 'Cersei', 'Tywin', 'Tyrion'
]
data = get_input_data()
transformers = get_transformers()
ht = HyperTransformer(transformers)
ht.fit(data)
transformed = ht.transform(data)
expected = get_transformed_data()
np.testing.assert_allclose(
transformed.sort_index(axis=1).values,
expected.sort_index(axis=1).values)
reversed_data = ht.reverse_transform(transformed)
original_names = data.pop('names')
reversed_names = reversed_data.pop('names')
pd.testing.assert_frame_equal(data.sort_index(axis=1),
reversed_data.sort_index(axis=1))
for name in original_names:
assert name not in reversed_names
def _compute_auroc(self, real_table, synthetic_table):
transformer = HyperTransformer()
real_table = transformer.fit_transform(real_table).values
synthetic_table = transformer.transform(synthetic_table).values
X = np.concatenate([real_table, synthetic_table])
y = np.hstack([np.ones(len(real_table)), np.zeros(len(synthetic_table))])
X[np.isnan(X)] = 0.0
if len(X) < 20:
warnings.warn("Not enough data, skipping the detection tests.")
scores = []
kf = StratifiedKFold(n_splits=3, shuffle=True)
for train_index, test_index in kf.split(X, y):
self.fit(X[train_index], y[train_index])
y_pred = self.predict_proba(X[test_index])
auroc = roc_auc_score(y[test_index], y_pred)
if auroc < 0.5:
auroc = 1.0 - auroc
scores.append(auroc)
return np.mean(scores)
def test__get_columns_regex(self):
data = pd.DataFrame({
'a(b)': [4, 5, 6],
'a(b)#1': [7, 8, 9],
'b(b)': [4, 5, 6],
'b(b)#1': [7, 8, 9],
})
returned = HyperTransformer._get_columns(data, 'a(b)')
expected = np.array([
'a(b)',
'a(b)#1',
])
np.testing.assert_equal(returned, expected)
def fit(self, table_data):
"""Fit ``Constraint`` class to data.
If ``fit_columns_model`` is True, then this method will fit
a ``GaussianCopula`` model to the relevant columns in ``table_data``.
Subclasses can overwrite this method, or overwrite the ``_fit`` method
if they will not be needing the model to handle conditional sampling.
Args:
table_data (pandas.DataFrame):
Table data.
"""
self._fit(table_data)
if self.fit_columns_model and len(self.constraint_columns) > 1:
data_to_model = table_data[list(self.constraint_columns)]
self._hyper_transformer = HyperTransformer(dtype_transformers={
'O': 'one_hot_encoding',
})
transformed_data = self._hyper_transformer.fit_transform(
data_to_model)
self._columns_model = GaussianMultivariate(
distribution=GaussianUnivariate)
self._columns_model.fit(transformed_data)
def test_single_category():
ht = HyperTransformer(transformers={'a': OneHotEncodingTransformer()})
data = pd.DataFrame({'a': ['a', 'a', 'a']})
ht.fit(data)
transformed = ht.transform(data)
reverse = ht.reverse_transform(transformed)
pd.testing.assert_frame_equal(data, reverse)
def test_dtype_category():
df = pd.DataFrame({'a': ['a', 'b', 'c']}, dtype='category')
ht = HyperTransformer()
ht.fit(df)
trans = ht.transform(df)
rever = ht.reverse_transform(trans)
pd.testing.assert_frame_equal(df, rever)
def test__analyze_bool(self):
"""Test _analyze bool dtype"""
# Setup
data = pd.DataFrame({'booleans': [True, False, None, False, True]})
dtypes = [bool]
# Run
transformer = Mock()
transformer.dtypes = dtypes
result = HyperTransformer._analyze(transformer, data)
# Asserts
expect_class = BooleanTransformer
self.assertIsInstance(result['booleans'], expect_class)
def test__analyze_int(self):
"""Test _analyze int dtype"""
# Setup
data = pd.DataFrame({'integers': [1, 2, 3, 4, 5, None, 6, 7, 8, 9, 0]})
dtypes = [int]
# Run
transformer = Mock()
transformer.dtypes = dtypes
result = HyperTransformer._analyze(transformer, data)
# Asserts
expect_class = NumericalTransformer
self.assertIsInstance(result['integers'], expect_class)
def test__analyze_object(self):
"""Test _analyze object dtype"""
# Setup
data = pd.DataFrame({'objects': ['foo', 'bar', None, 'tar']})
dtypes = [np.object]
# Run
transformer = Mock()
transformer.dtypes = dtypes
result = HyperTransformer._analyze(transformer, data)
# Asserts
expect_class = CategoricalTransformer
self.assertIsInstance(result['objects'], expect_class)
def test_empty_transformers_nan_data():
"""If transformers is an empty dict, do nothing."""
data = get_input_data_with_nan()
ht = HyperTransformer(transformers={})
ht.fit(data)
transformed = ht.transform(data)
reverse = ht.reverse_transform(transformed)
pd.testing.assert_frame_equal(data, transformed)
pd.testing.assert_frame_equal(data, reverse)
def _load_hyper_transformer(self, table_name):
"""Create and return a new ``rdt.HyperTransformer`` instance for a table.
First get the ``dtypes`` and ``pii fields`` from a given table, then use
those to build a transformer dictionary to be used by the ``HyperTransformer``.
Args:
table_name (str):
Table name for which to load the HyperTransformer.
Returns:
rdt.HyperTransformer:
Instance of ``rdt.HyperTransformer`` for the given table.
"""
dtypes = self.get_dtypes(table_name)
pii_fields = self._get_pii_fields(table_name)
transformers_dict = self._get_transformers(dtypes, pii_fields)
return HyperTransformer(transformers=transformers_dict)
def test__analyze_float(self):
"""Test _analyze float dtype"""
# Setup
data = pd.DataFrame({
'floats': [1.1, 2.2, 3.3, 4.4, 5.5, None, 6.6, 7.7, 8.8, 9.9, 0.0]
})
dtypes = [float]
# Run
transformer = Mock()
transformer.dtypes = dtypes
result = HyperTransformer._analyze(transformer, data)
# Asserts
expect_class = NumericalTransformer
self.assertIsInstance(result['floats'], expect_class)
def test_subset_of_columns_nan_data():
"""HyperTransform should be able to transform a subset of the training columns.
See https://github.com/sdv-dev/RDT/issues/152
"""
data = get_input_data_with_nan()
ht = HyperTransformer()
ht.fit(data)
subset = data[[data.columns[0]]]
transformed = ht.transform(subset)
reverse = ht.reverse_transform(transformed)
pd.testing.assert_frame_equal(subset, reverse)
def test__analyze_datetime64(self):
"""Test _analyze datetime64 dtype"""
# Setup
data = pd.DataFrame({'datetimes': ['1965-05-23', None, '1997-10-17']})
data['datetimes'] = pd.to_datetime(data['datetimes'],
format='%Y-%m-%d',
errors='coerce')
dtypes = [np.datetime64]
# Run
transformer = Mock()
transformer.dtypes = dtypes
result = HyperTransformer._analyze(transformer, data)
# Asserts
expect_class = DatetimeTransformer
self.assertIsInstance(result['datetimes'], expect_class)
def test_hypertransformer_without_transformers_nan_data():
data = get_input_data_with_nan()
ht = HyperTransformer()
ht.fit(data)
transformed = ht.transform(data)
expected = get_transformed_nan_data()
np.testing.assert_allclose(
transformed.sort_index(axis=1).values,
expected.sort_index(axis=1).values)
reversed_data = ht.reverse_transform(transformed)
original_names = data.pop('names')
reversed_names = reversed_data.pop('names')
pd.testing.assert_frame_equal(data.sort_index(axis=1),
reversed_data.sort_index(axis=1))
for name in original_names:
assert name not in reversed_names
class CopulaGAN(CTGAN):
"""Combination of GaussianCopula transformation and GANs.
This model extends the ``CTGAN`` model to add the flexibility of the GaussianCopula
transformations provided by the ``GaussianCopulaTransformer`` from ``RDT``.
Overall, the fitting process consists of the following steps:
1. Transform each non categorical variable from the input
data using a ``GaussianCopulaTransformer``:
i. If not specified, find out the distribution which each one
of the variables from the input dataset has.
ii. Transform each variable to a standard normal space by applying
the CDF of the corresponding distribution and later on applying
an inverse CDF from a standard normal distribution.
2. Fit CTGAN with the transformed table.
And the process of sampling is:
1. Sample using CTGAN
2. Reverse the previous transformation by applying the CDF of a standard normal
distribution and then inverting the CDF of the distribution that correpsonds
to each variable.
The arguments of this model are the same as for CTGAN except for two additional
arguments, ``field_distributions`` and ``default_distribution`` that give the
ability to define specific transformations for individual fields as well as
which distribution to use by default if no specific distribution has been selected.
Distributions can be passed as a ``copulas`` univariate instance or as one
of the following string values:
* ``univariate``: Let ``copulas`` select the optimal univariate distribution.
This may result in non-parametric models being used.
* ``parametric``: Let ``copulas`` select the optimal univariate distribution,
but restrict the selection to parametric distributions only.
* ``bounded``: Let ``copulas`` select the optimal univariate distribution,
but restrict the selection to bounded distributions only.
This may result in non-parametric models being used.
* ``semi_bounded``: Let ``copulas`` select the optimal univariate distribution,
but restrict the selection to semi-bounded distributions only.
This may result in non-parametric models being used.
* ``parametric_bounded``: Let ``copulas`` select the optimal univariate
distribution, but restrict the selection to parametric and bounded distributions
only.
* ``parametric_semi_bounded``: Let ``copulas`` select the optimal univariate
distribution, but restrict the selection to parametric and semi-bounded
distributions only.
* ``gaussian``: Use a Gaussian distribution.
* ``gamma``: Use a Gamma distribution.
* ``beta``: Use a Beta distribution.
* ``student_t``: Use a Student T distribution.
* ``gaussian_kde``: Use a GaussianKDE distribution. This model is non-parametric,
so using this will make ``get_parameters`` unusable.
* ``truncated_gaussian``: Use a Truncated Gaussian distribution.
Args:
field_names (list[str]):
List of names of the fields that need to be modeled
and included in the generated output data. Any additional
fields found in the data will be ignored and will not be
included in the generated output.
If ``None``, all the fields found in the data are used.
field_types (dict[str, dict]):
Dictinary specifying the data types and subtypes
of the fields that will be modeled. Field types and subtypes
combinations must be compatible with the SDV Metadata Schema.
field_transformers (dict[str, str]):
Dictinary specifying which transformers to use for each field.
Available transformers are:
* ``integer``: Uses a ``NumericalTransformer`` of dtype ``int``.
* ``float``: Uses a ``NumericalTransformer`` of dtype ``float``.
* ``categorical``: Uses a ``CategoricalTransformer`` without gaussian noise.
* ``categorical_fuzzy``: Uses a ``CategoricalTransformer`` adding gaussian noise.
* ``one_hot_encoding``: Uses a ``OneHotEncodingTransformer``.
* ``label_encoding``: Uses a ``LabelEncodingTransformer``.
* ``boolean``: Uses a ``BooleanTransformer``.
* ``datetime``: Uses a ``DatetimeTransformer``.
anonymize_fields (dict[str, str]):
Dict specifying which fields to anonymize and what faker
category they belong to.
primary_key (str):
Name of the field which is the primary key of the table.
constraints (list[Constraint, dict]):
List of Constraint objects or dicts.
table_metadata (dict or metadata.Table):
Table metadata instance or dict representation.
If given alongside any other metadata-related arguments, an
exception will be raised.
If not given at all, it will be built using the other
arguments or learned from the data.
log_frequency (boolean):
Whether to use log frequency of categorical levels in conditional
sampling. Defaults to ``True``.
embedding_dim (int):
Size of the random sample passed to the Generator. Defaults to 128.
generator_dim (tuple or list of ints):
Size of the output samples for each one of the Residuals. A Resiudal Layer
will be created for each one of the values provided. Defaults to (256, 256).
discriminator_dim (tuple or list of ints):
Size of the output samples for each one of the Discriminator Layers. A Linear
Layer will be created for each one of the values provided. Defaults to (256, 256).
batch_size (int):
Number of data samples to process in each step.
verbose (bool):
Whether to print fit progress on stdout. Defaults to ``False``.
epochs (int):
Number of training epochs. Defaults to 300.
cuda (bool or str):
If ``True``, use CUDA. If an ``str``, use the indicated device.
If ``False``, do not use cuda at all.
field_distributions (dict):
Optionally specify a dictionary that maps the name of each field to the distribution
that must be used in it. Fields that are not specified in the input ``dict`` will
be modeled using the default distribution. Defaults to ``None``.
default_distribution (copulas.univariate.Univariate or str):
Distribution to use on the fields for which no specific distribution has been given.
Defaults to ``parametric``.
"""
DEFAULT_DISTRIBUTION = 'parametric'
def __init__(self,
field_names=None,
field_types=None,
field_transformers=None,
anonymize_fields=None,
primary_key=None,
constraints=None,
table_metadata=None,
embedding_dim=128,
generator_dim=(256, 256),
discriminator_dim=(256, 256),
generator_lr=2e-4,
generator_decay=1e-6,
discriminator_lr=2e-4,
discriminator_decay=1e-6,
batch_size=500,
discriminator_steps=1,
log_frequency=True,
verbose=False,
epochs=300,
cuda=True,
field_distributions=None,
default_distribution=None):
super().__init__(field_names=field_names,
primary_key=primary_key,
field_types=field_types,
field_transformers=field_transformers,
anonymize_fields=anonymize_fields,
constraints=constraints,
table_metadata=table_metadata,
embedding_dim=embedding_dim,
generator_dim=generator_dim,
discriminator_dim=discriminator_dim,
generator_lr=generator_lr,
generator_decay=generator_decay,
discriminator_lr=discriminator_lr,
discriminator_decay=discriminator_decay,
batch_size=batch_size,
discriminator_steps=discriminator_steps,
log_frequency=log_frequency,
verbose=verbose,
epochs=epochs,
cuda=cuda)
self._field_distributions = field_distributions or dict()
self._default_distribution = default_distribution or self.DEFAULT_DISTRIBUTION
def get_distributions(self):
"""Get the marginal distributions used by this CopulaGAN.
Returns:
dict:
Dictionary containing the distributions used or detected
for each column.
"""
return {
field: transformer._univariate.to_dict()['type']
for field, transformer in self._ht.transformers.items()
}
def _fit(self, table_data):
"""Fit the model to the table.
Args:
table_data (pandas.DataFrame):
Data to be learned.
"""
distributions = self._field_distributions
default = self._default_distribution
fields = self._metadata.get_fields()
transformers = {
field: GaussianCopulaTransformer(
distribution=distributions.get(field, default))
for field in table_data.columns
if fields.get(field, dict()).get('type') != 'categorical'
}
self._ht = HyperTransformer(transformers=transformers)
table_data = self._ht.fit_transform(table_data)
super()._fit(table_data)
def _sample(self, num_rows, conditions=None):
"""Sample the indicated number of rows from the model.
Args:
num_rows (int):
Amount of rows to sample.
conditions (dict):
If specified, this dictionary maps column names to the column
value. Then, this method generates `num_rows` samples, all of
which are conditioned on the given variables.
Returns:
pandas.DataFrame:
Sampled data.
"""
sampled = super()._sample(num_rows, conditions)
return self._ht.reverse_transform(sampled)
