def test_sample_random_state(self):
"""When random_state is set the samples are the same."""
# Setup
instance = GaussianMultivariate(GaussianUnivariate, random_seed=0)
data = pd.DataFrame([
{'A': 25, 'B': 75, 'C': 100},
{'A': 30, 'B': 60, 'C': 250},
{'A': 10, 'B': 65, 'C': 350},
{'A': 20, 'B': 80, 'C': 150},
{'A': 25, 'B': 70, 'C': 500}
])
instance.fit(data)
expected_result = pd.DataFrame(
np.array([
[25.19031668, 61.96527251, 543.43595269],
[31.50262306, 49.70971698, 429.06537124],
[20.31636799, 64.3492326, 384.27561823],
[25.00302427, 72.06019812, 415.85215123],
[23.07525773, 66.70901743, 390.8226672]
]),
columns=['A', 'B', 'C']
)
# Run
result = instance.sample(5)
# Check
pd.testing.assert_frame_equal(result, expected_result, check_less_precise=True)
def test_to_dict(self):
"""To_dict returns the parameters to replicate the copula."""
# Setup
copula = GaussianMultivariate()
data = pd.read_csv('data/iris.data.csv')
copula.fit(data)
covariance = [[
1.006711409395973, -0.11010327176239865, 0.8776048563471857,
0.823443255069628
],
[
-0.11010327176239865, 1.006711409395972,
-0.4233383520816991, -0.3589370029669186
],
[
0.8776048563471857, -0.4233383520816991,
1.006711409395973, 0.9692185540781536
],
[
0.823443255069628, -0.3589370029669186,
0.9692185540781536, 1.0067114093959735
]]
expected_result = {
'covariance': covariance,
'fitted': True,
'type': 'copulas.multivariate.gaussian.GaussianMultivariate',
'distribution': 'copulas.univariate.gaussian.GaussianUnivariate',
'distribs': {
'feature_01': {
'type': 'copulas.univariate.gaussian.GaussianUnivariate',
'mean': 5.843333333333334,
'std': 0.8253012917851409,
'fitted': True,
},
'feature_02': {
'type': 'copulas.univariate.gaussian.GaussianUnivariate',
'mean': 3.0540000000000003,
'std': 0.4321465800705435,
'fitted': True,
},
'feature_03': {
'type': 'copulas.univariate.gaussian.GaussianUnivariate',
'mean': 3.758666666666666,
'std': 1.7585291834055212,
'fitted': True,
},
'feature_04': {
'type': 'copulas.univariate.gaussian.GaussianUnivariate',
'mean': 1.1986666666666668,
'std': 0.7606126185881716,
'fitted': True,
}
}
}
# Run
result = copula.to_dict()
# Check
compare_nested_dicts(result, expected_result)
def test__transform_to_normal_numpy_1d(self):
# Setup
gm = GaussianMultivariate()
dist_a = Mock()
dist_a.cdf.return_value = np.array([0])
dist_b = Mock()
dist_b.cdf.return_value = np.array([0.3])
gm.columns = ['a', 'b']
gm.univariates = [dist_a, dist_b]
# Run
data = np.array([
[3, 5],
])
returned = gm._transform_to_normal(data)
# Check
# Failures may occurr on different cpytonn implementations
# with different float precision values.
# If that happens, atol might need to be increased
expected = np.array([
[-5.166579, -0.524401],
])
np.testing.assert_allclose(returned, expected, atol=1e-6)
assert dist_a.cdf.call_count == 1
expected = np.array([3])
passed = dist_a.cdf.call_args[0][0]
np.testing.assert_allclose(expected, passed)
assert dist_b.cdf.call_count == 1
expected = np.array([5])
passed = dist_b.cdf.call_args[0][0]
np.testing.assert_allclose(expected, passed)
def test_sample(self):
"""Generated samples keep the same mean and deviation as the original data."""
copula = GaussianMultivariate()
stats = [{
'mean': 10000,
'std': 15
}, {
'mean': 150,
'std': 10
}, {
'mean': -50,
'std': 0.1
}]
data = pd.DataFrame(
[np.random.normal(x['mean'], x['std'], 100) for x in stats]).T
copula.fit(data)
# Run
result = copula.sample(1000000)
# Check
assert result.shape == (1000000, 3)
for i, stat in enumerate(stats):
expected_mean = np.mean(data[i])
expected_std = np.std(data[i])
result_mean = np.mean(result[i])
result_std = np.std(result[i])
assert abs(expected_mean - result_mean) < abs(expected_mean / 100)
assert abs(expected_std - result_std) < abs(expected_std / 100)
def test__transform_to_normal_dataframe(self):
# Setup
gm = GaussianMultivariate()
dist_a = Mock()
dist_a.cdf.return_value = np.array([0, 0.5, 1])
dist_b = Mock()
dist_b.cdf.return_value = np.array([0.3, 0.5, 0.7])
gm.distribs = OrderedDict((
('a', dist_a),
('b', dist_b),
))
# Run
data = pd.DataFrame({'a': [3, 4, 5], 'b': [5, 6, 7]})
returned = gm._transform_to_normal(data)
# Check
# Failures may occurr on different cpytonn implementations
# with different float precision values.
# If that happens, atol might need to be increased
expected = np.array([[-5.166579, -0.524401], [0.0, 0.0],
[5.166579, 0.524401]])
np.testing.assert_allclose(returned, expected, atol=1e-6)
assert dist_a.cdf.call_count == 1
expected = np.array([3, 4, 5])
passed = dist_a.cdf.call_args[0][0]
np.testing.assert_allclose(expected, passed)
assert dist_b.cdf.call_count == 1
expected = np.array([5, 6, 7])
passed = dist_b.cdf.call_args[0][0]
np.testing.assert_allclose(expected, passed)
def test_sample_constant_column(self):
"""Gaussian copula can sample after being fit with a constant column.
This process will raise warnings when computing the covariance matrix
"""
# Setup
instance = GaussianMultivariate()
X = np.array([
[1.0, 2.0],
[1.0, 3.0],
[1.0, 4.0],
[1.0, 5.0]
])
instance.fit(X)
# Run
result = instance.sample(5)
# Check
assert result.shape == (5, 2)
assert result[~result.isnull()].all().all()
assert result.loc[:, 0].equals(pd.Series([1.0, 1.0, 1.0, 1.0, 1.0], name=0))
# This is to check that the samples on the non constant column are not constant too.
assert len(result.loc[:, 1].unique()) > 1
covariance = instance.covariance
assert (~pd.isnull(covariance)).all().all()
def test_sample_random_state(self):
"""When random_state is set the samples are the same."""
# Setup
instance = GaussianMultivariate(
random_seed=0,
distribution='copulas.univariate.gaussian.GaussianUnivariate')
data = pd.DataFrame([{
'A': 25,
'B': 75,
'C': 100
}, {
'A': 30,
'B': 60,
'C': 250
}, {
'A': 10,
'B': 65,
'C': 350
}, {
'A': 20,
'B': 80,
'C': 150
}, {
'A': 25,
'B': 70,
'C': 500
}])
instance.fit(data)
expected_result = pd.DataFrame([{
'A': 25.566882482769294,
'B': 61.01690157277244,
'C': 575.71068885087790
}, {
'A': 32.624255560452110,
'B': 47.31477394460025,
'C': 447.84049148268970
}, {
'A': 20.117642182744806,
'B': 63.68224998298797,
'C': 397.76402526341593
}, {
'A': 25.357483201156676,
'B': 72.30337152729443,
'C': 433.06766240515134
}, {
'A': 23.202174689737113,
'B': 66.32056962524452,
'C': 405.08384853948280
}])
# Run
result = instance.sample(5)
# Check
assert result.equals(expected_result)
def test_get_lower_bounds(self):
"""get_lower_bounds returns the point from where cut the tail of the infinite integral."""
# Setup
copula = GaussianMultivariate()
copula.fit(self.data)
expected_result = -3.104256111232535
# Run
result = copula.get_lower_bound()
# Check
assert result == expected_result
def test_probability_density(self):
"""Probability_density computes probability for the given values."""
# Setup
copula = GaussianMultivariate(GaussianUnivariate)
copula.fit(self.data)
X = np.array([2000., 200., 0.])
expected_result = 0.032245296420409846
# Run
result = copula.probability_density(X)
# Check
self.assertAlmostEqual(result, expected_result)
def test_probability_density(self):
"""Probability_density computes probability for the given values."""
# Setup
copula = GaussianMultivariate()
copula.fit(self.data)
X = np.array([[0., 0., 0.]])
expected_result = 0.059566912334560594
# Run
result = copula.probability_density(X)
# Check
assert result == expected_result
def test_cumulative_distribution_fit_call_pd(self):
"""Cumulative_density integrates the probability density along the given values."""
# Setup
copula = GaussianMultivariate(GaussianUnivariate)
copula.fit(self.data.values)
X = np.array([2000., 200., 1.])
expected_result = 0.4550595153746892
# Run
result = copula.cumulative_distribution(X)
# Check
assert np.isclose(result, expected_result, atol=1e-5).all().all()
def test_cumulative_distribution_fit_call_pd(self):
"""Cumulative_density integrates the probability density along the given values."""
# Setup
copula = GaussianMultivariate()
copula.fit(self.data.values)
X = pd.Series([1., 1., 1.])
expected_result = 0.5822020991592192
# Run
result = copula.cumulative_distribution(X)
# Check
assert np.isclose(result, expected_result).all().all()
def test_fit_default_distribution(self):
"""On fit, a distribution is created for each column along the covariance and means"""
copula = GaussianMultivariate(GaussianUnivariate)
copula.fit(self.data)
for i, key in enumerate(self.data.columns):
assert copula.columns[i] == key
assert copula.univariates[i].__class__ == GaussianUnivariate
assert copula.univariates[i]._params['loc'] == self.data[key].mean()
assert copula.univariates[i]._params['scale'] == np.std(self.data[key])
expected_covariance = copula._get_covariance(self.data)
assert (copula.covariance == expected_covariance).all().all()
def test_probability_density(self):
"""Probability_density computes probability for the given values."""
# Setup
copula = GaussianMultivariate(
distribution='copulas.univariate.gaussian.GaussianUnivariate')
copula.fit(self.data)
X = np.array([2000., 200., 0.])
expected_result = 0.031163598715950383
# Run
result = copula.probability_density(X)
# Check
self.assertAlmostEqual(result, expected_result)
def test_deprecation_warnings(self):
"""After fitting, Gaussian copula can produce new samples warningless."""
# Setup
copula = GaussianMultivariate()
data = pd.read_csv('data/iris.data.csv')
# Run
with warnings.catch_warnings(record=True) as warns:
copula.fit(data)
result = copula.sample(10)
# Check
assert len(warns) == 0
assert len(result) == 10
def test_cumulative_distribution_fit_call_np_array(self):
"""Cumulative_density integrates the probability density along the given values."""
# Setup
copula = GaussianMultivariate(
distribution='copulas.univariate.gaussian.GaussianUnivariate')
copula.fit(self.data.values)
X = np.array([2000., 200., 1.])
expected_result = 0.4460456536217443
# Run
result = copula.cumulative_distribution(X)
# Check
assert np.isclose(result, expected_result, atol=1e-5).all().all()
def test__get_covariance(self):
"""_get_covariance computes the covariance matrix of normalized values."""
# Setup
copula = GaussianMultivariate(GaussianUnivariate)
copula.fit(self.data)
expected_covariance = np.array([[1., -0.01261819, -0.19821644],
[-0.01261819, 1., -0.16896087],
[-0.19821644, -0.16896087, 1.]])
# Run
covariance = copula._get_covariance(self.data)
# Check
assert np.isclose(covariance, expected_covariance).all().all()
def test__get_covariance_numpy_array(self):
"""_get_covariance computes the covariance matrix of normalized values."""
# Setup
copula = GaussianMultivariate()
copula.fit(self.data.values)
expected_covariance = np.array([[1.04347826, -0.01316681, -0.20683455],
[-0.01316681, 1.04347826, -0.176307],
[-0.20683455, -0.176307, 1.04347826]])
# Run
covariance = copula._get_covariance(self.data.values)
# Check
assert np.isclose(covariance, expected_covariance).all().all()
def test_fit_numpy_array(self):
"""Fit should work indistinctly with numpy arrays and pandas dataframes """
# Setup
copula = GaussianMultivariate(
distribution='copulas.univariate.gaussian.GaussianUnivariate')
# Run
copula.fit(self.data.values)
# Check
for key, (column, univariate) in enumerate(zip(self.data.columns, copula.univariates)):
assert univariate._params['loc'] == np.mean(self.data[column])
assert univariate._params['scale'] == np.std(self.data[column])
expected_covariance = copula._get_covariance(pd.DataFrame(self.data.values))
assert (copula.covariance == expected_covariance).all().all()
def test_fit_numpy_array(self):
"""Fit should work indistinctly with numpy arrays and pandas dataframes """
# Setup
copula = GaussianMultivariate()
# Run
copula.fit(self.data.values)
# Check
for key, column in enumerate(self.data.columns):
assert copula.distribs[key]
assert copula.distribs[key].mean == np.mean(self.data[column])
assert copula.distribs[key].std == np.std(self.data[column])
expected_covariance = copula._get_covariance(
pd.DataFrame(self.data.values))
assert (copula.covariance == expected_covariance).all().all()
def test_fit_distribution_selector(self):
"""
On fit, it should use the correct distributions for those that are
specified and default to using the base class otherwise.
"""
copula = GaussianMultivariate(distribution={
'column1': 'copulas.univariate.beta.BetaUnivariate',
'column2': 'copulas.univariate.gaussian_kde.GaussianKDE',
})
copula.fit(self.data)
assert get_qualified_name(
copula.univariates[0].__class__) == 'copulas.univariate.beta.BetaUnivariate'
assert get_qualified_name(
copula.univariates[1].__class__) == 'copulas.univariate.gaussian_kde.GaussianKDE'
assert get_qualified_name(
copula.univariates[2].__class__) == 'copulas.univariate.base.Univariate'
def test_fit(self):
"""On fit, a distribution is created for each column along the covariance and means"""
# Setup
copula = GaussianMultivariate()
# Run
copula.fit(self.data)
# Check
for key in self.data.columns:
assert copula.distribs[key]
assert copula.distribs[key].mean == self.data[key].mean()
assert copula.distribs[key].std == np.std(self.data[key])
expected_covariance = copula._get_covariance(self.data)
assert (copula.covariance == expected_covariance).all().all()
def fit(self, table_data):
"""Fit the ``ColumnsModel``.
Fit a ``GaussianUnivariate`` model to the ``self.constraint_column`` columns in the
``table_data`` in order to sample those columns when missing.
Args:
table_data (pandas.DataFrame):
Table data.
"""
data_to_model = table_data[self.constraint_columns]
self._hyper_transformer = HyperTransformer(
default_data_type_transformers={
'categorical': 'OneHotEncodingTransformer'
})
transformed_data = self._hyper_transformer.fit_transform(data_to_model)
self._model = GaussianMultivariate(distribution=GaussianUnivariate)
self._model.fit(transformed_data)
def test___init__(self):
"""On init an instance with None on all attributes except distribs is returned."""
# Run
copula = GaussianMultivariate()
# Check
assert copula.distribs == {}
assert copula.covariance is None
assert copula.means is None
def test_fit_distribution_arg(self):
"""On fit, the distributions for each column use instances of copula.distribution."""
# Setup
distribution = 'copulas.univariate.gaussian_kde.GaussianKDE'
copula = GaussianMultivariate(distribution=distribution)
# Run
copula.fit(self.data)
# Check
assert copula.distribution == 'copulas.univariate.gaussian_kde.GaussianKDE'
for i, key in enumerate(self.data.columns):
assert copula.columns[i] == key
assert get_qualified_name(copula.univariates[i].__class__) == copula.distribution
expected_covariance = copula._get_covariance(self.data)
assert (copula.covariance == expected_covariance).all().all()
def test_fit_distribution_arg(self):
"""On fit, the distributions for each column use instances of copula.distribution."""
# Setup
distribution = 'copulas.univariate.kde.KDEUnivariate'
copula = GaussianMultivariate(distribution=distribution)
# Run
copula.fit(self.data)
# Check
assert copula.distribution == 'copulas.univariate.kde.KDEUnivariate'
for key in self.data.columns:
assert key in copula.distribs
assert get_qualified_name(
copula.distribs[key].__class__) == copula.distribution
expected_covariance = copula._get_covariance(self.data)
assert (copula.covariance == expected_covariance).all().all()
def test___init__default_args(self):
"""On init an instance with None on all attributes except distribs is returned."""
# Run
copula = GaussianMultivariate()
# Check
assert copula.distribs == {}
assert copula.covariance is None
assert copula.means is None
assert copula.distribution == 'copulas.univariate.gaussian.GaussianUnivariate'
def test__get_conditional_distribution(self):
gm = GaussianMultivariate()
gm.covariance = pd.DataFrame({
'a': [1, 0.2, 0.3],
'b': [0.2, 1, 0.4],
'c': [0.3, 0.4, 1],
}, index=['a', 'b', 'c'])
conditions = pd.Series({
'b': 1
})
means, covariance, columns = gm._get_conditional_distribution(conditions)
np.testing.assert_allclose(means, [0.2, 0.4])
np.testing.assert_allclose(covariance, [
[0.96, 0.22],
[0.22, 0.84]
])
assert columns.tolist() == ['a', 'c']
def test__init__distribution_arg(self):
"""On init the distribution argument is set as attribute."""
# Setup
distribution = 'full.qualified.name.of.distribution'
# Run
copula = GaussianMultivariate(distribution)
# Check
assert copula.distribs == {}
assert copula.covariance is None
assert copula.distribution == 'full.qualified.name.of.distribution'
def test_fit_default_distribution(self):
"""On fit, a distribution is created for each column along the covariance and means"""
# Setup
copula = GaussianMultivariate()
# Run
copula.fit(self.data)
# Check
assert copula.distribution == 'copulas.univariate.gaussian.GaussianUnivariate'
for key in self.data.columns:
assert key in copula.distribs
assert get_qualified_name(
copula.distribs[key].__class__) == copula.distribution
assert copula.distribs[key].mean == self.data[key].mean()
assert copula.distribs[key].std == np.std(self.data[key])
expected_covariance = copula._get_covariance(self.data)
assert (copula.covariance == expected_covariance).all().all()
