def test_corr_nearest_factor_sparse(self, dm):
# Test that result is the same if the input is dense or sparse
d = 200
# Generate a test matrix of factors
X = np.zeros((d, dm), dtype=np.float64)
x = np.linspace(0, 2 * np.pi, d)
rs = np.random.RandomState(10)
for j in range(dm):
X[:, j] = np.sin(x * (j + 1)) + rs.randn(d)
# Get the correlation matrix
_project_correlation_factors(X)
X *= 0.7
mat = np.dot(X, X.T)
np.fill_diagonal(mat, 1)
# Threshold it
mat.flat[np.abs(mat.flat) < 0.35] = 0.0
smat = sparse.csr_matrix(mat)
dense_rslt = corr_nearest_factor(mat, dm, maxiter=10000)
sparse_rslt = corr_nearest_factor(smat, dm, maxiter=10000)
mat_dense = dense_rslt.corr.to_matrix()
mat_sparse = sparse_rslt.corr.to_matrix()
assert dense_rslt.Converged is sparse_rslt.Converged
assert dense_rslt.Converged is True
assert_allclose(mat_dense, mat_sparse, rtol=.25, atol=1e-3)
def test_corr_nearest_factor_sparse(self):
d = 100
for dm in 1, 2:
# Generate a test matrix of factors
X = np.zeros((d, dm), dtype=np.float64)
x = np.linspace(0, 2 * np.pi, d)
np.random.seed(10)
for j in range(dm):
X[:, j] = np.sin(x * (j + 1)) + 1e-10 * np.random.randn(d)
# Get the correlation matrix
_project_correlation_factors(X)
X *= 0.7
mat = np.dot(X, X.T)
np.fill_diagonal(mat, 1)
# Threshold it
mat *= (np.abs(mat) >= 0.4)
smat = sparse.csr_matrix(mat)
rslt = corr_nearest_factor(smat, dm)
assert_equal(rslt.Converged, True)
mat_dense = rslt.corr.to_matrix()
rslt = corr_nearest_factor(smat, dm)
assert_equal(rslt.Converged, True)
mat_sparse = rslt.corr.to_matrix()
assert_allclose(mat_dense, mat_sparse, rtol=0.25, atol=1e-3)
def test_corr_nearest_factor(self, dm):
objvals = [
np.array([6241.8, 6241.8, 579.4, 264.6, 264.3]),
np.array([2104.9, 2104.9, 710.5, 266.3, 286.1])
]
d = 100
# Construct a test matrix with exact factor structure
X = np.zeros((d, dm), dtype=np.float64)
x = np.linspace(0, 2 * np.pi, d)
np.random.seed(10)
for j in range(dm):
X[:, j] = np.sin(x * (j + 1)) + 1e-10 * np.random.randn(d)
_project_correlation_factors(X)
assert np.isfinite(X).all()
X *= 0.7
mat = np.dot(X, X.T)
np.fill_diagonal(mat, 1.)
# Try to recover the structure
rslt = corr_nearest_factor(mat, dm, maxiter=10000)
err_msg = 'rank=%d, niter=%d' % (dm, len(rslt.objective_values))
assert_allclose(rslt.objective_values[:5],
objvals[dm - 1],
rtol=0.5,
err_msg=err_msg)
assert rslt.Converged
mat1 = rslt.corr.to_matrix()
assert_allclose(mat, mat1, rtol=0.25, atol=1e-3, err_msg=err_msg)
def test_corr_nearest_factor_sparse(self):
d = 100
for dm in 1,2:
# Generate a test matrix of factors
X = np.zeros((d,dm), dtype=np.float64)
x = np.linspace(0, 2*np.pi, d)
for j in range(dm):
X[:,j] = np.sin(x*(j+1))
# Get the correlation matrix
_project_correlation_factors(X)
X *= 0.7
mat = np.dot(X, X.T)
np.fill_diagonal(mat, 1)
# Threshold it
mat *= (np.abs(mat) >= 0.4)
smat = sparse.csr_matrix(mat)
fac_dense = corr_nearest_factor(smat, dm).corr
mat_dense = fac_dense.to_matrix()
fac_sparse = corr_nearest_factor(smat, dm).corr
mat_sparse = fac_sparse.to_matrix()
assert_allclose(mat_dense, mat_sparse, rtol=0.25,
atol=1e-3)
def test_corr_nearest_factor(self, dm):
objvals = [np.array([6241.8, 6241.8, 579.4, 264.6, 264.3]),
np.array([2104.9, 2104.9, 710.5, 266.3, 286.1])]
d = 100
# Construct a test matrix with exact factor structure
X = np.zeros((d, dm), dtype=np.float64)
x = np.linspace(0, 2 * np.pi, d)
np.random.seed(10)
for j in range(dm):
X[:, j] = np.sin(x * (j + 1)) + 1e-10 * np.random.randn(d)
_project_correlation_factors(X)
assert np.isfinite(X).all()
X *= 0.7
mat = np.dot(X, X.T)
np.fill_diagonal(mat, 1.)
# Try to recover the structure
rslt = corr_nearest_factor(mat, dm, maxiter=10000)
err_msg = 'rank=%d, niter=%d' % (dm, len(rslt.objective_values))
assert_allclose(rslt.objective_values[:5], objvals[dm - 1],
rtol=0.5, err_msg=err_msg)
assert rslt.Converged
mat1 = rslt.corr.to_matrix()
assert_allclose(mat, mat1, rtol=0.25, atol=1e-3, err_msg=err_msg)
def test_corr_nearest_factor_sparse(self, dm):
# Test that result is the same if the input is dense or sparse
d = 100
# Generate a test matrix of factors
X = np.zeros((d, dm), dtype=np.float64)
x = np.linspace(0, 2 * np.pi, d)
np.random.seed(10)
for j in range(dm):
X[:, j] = np.sin(x * (j + 1)) + 1e-10 * np.random.randn(d)
# Get the correlation matrix
_project_correlation_factors(X)
X *= 0.7
mat = np.dot(X, X.T)
np.fill_diagonal(mat, 1)
# Threshold it
mat *= (np.abs(mat) >= 0.35)
smat = sparse.csr_matrix(mat)
try:
rslt = corr_nearest_factor(mat, dm, maxiter=10000)
assert rslt.Converged is True
mat_dense = rslt.corr.to_matrix()
rslt = corr_nearest_factor(smat, dm, maxiter=10000)
assert rslt.Converged is True
mat_sparse = rslt.corr.to_matrix()
assert_allclose(mat_dense, mat_sparse, rtol=.25, atol=1e-3)
except AssertionError as err:
if PLATFORM_WIN32:
pytest.xfail('Known to randomly fail on Win32')
raise err
def test_corr_nearest_factor_sparse(self):
d = 100
for dm in 1,2:
# Generate a test matrix of factors
X = np.zeros((d,dm), dtype=np.float64)
x = np.linspace(0, 2*np.pi, d)
np.random.seed(10)
for j in range(dm):
X[:,j] = np.sin(x*(j+1)) + 1e-10 * np.random.randn(d)
# Get the correlation matrix
_project_correlation_factors(X)
X *= 0.7
mat = np.dot(X, X.T)
np.fill_diagonal(mat, 1)
# Threshold it
mat *= (np.abs(mat) >= 0.4)
smat = sparse.csr_matrix(mat)
rslt = corr_nearest_factor(smat, dm)
assert_equal(rslt.Converged, True)
mat_dense = rslt.corr.to_matrix()
rslt = corr_nearest_factor(smat, dm)
assert_equal(rslt.Converged, True)
mat_sparse = rslt.corr.to_matrix()
assert_allclose(mat_dense, mat_sparse, rtol=0.25,
atol=1e-3)
def test_corr_nearest_factor_sparse(self):
d = 100
for dm in 1,2:
# Generate a test matrix of factors
X = np.zeros((d,dm), dtype=np.float64)
x = np.linspace(0, 2*np.pi, d)
for j in range(dm):
X[:,j] = np.sin(x*(j+1))
# Get the correlation matrix
_project_correlation_factors(X)
X *= 0.7
mat = np.dot(X, X.T)
np.fill_diagonal(mat, 1)
# Threshold it
mat *= (np.abs(mat) >= 0.4)
smat = sparse.csr_matrix(mat)
fac_dense = corr_nearest_factor(smat, dm).corr
mat_dense = fac_dense.to_matrix()
fac_sparse = corr_nearest_factor(smat, dm).corr
mat_sparse = fac_sparse.to_matrix()
assert_allclose(mat_dense, mat_sparse, rtol=0.25,
atol=1e-3)
def test_corr_nearest_factor_sparse(self, dm):
# Test that result is the same if the input is dense or sparse
d = 100
# Generate a test matrix of factors
X = np.zeros((d, dm), dtype=np.float64)
x = np.linspace(0, 2 * np.pi, d)
np.random.seed(10)
for j in range(dm):
X[:, j] = np.sin(x * (j + 1)) + 1e-10 * np.random.randn(d)
# Get the correlation matrix
_project_correlation_factors(X)
X *= 0.7
mat = np.dot(X, X.T)
np.fill_diagonal(mat, 1)
# Threshold it
mat *= (np.abs(mat) >= 0.35)
smat = sparse.csr_matrix(mat)
try:
rslt = corr_nearest_factor(mat, dm, maxiter=10000)
assert rslt.Converged is True
mat_dense = rslt.corr.to_matrix()
rslt = corr_nearest_factor(smat, dm, maxiter=10000)
assert rslt.Converged is True
mat_sparse = rslt.corr.to_matrix()
assert_allclose(mat_dense, mat_sparse, rtol=.25, atol=1e-3)
except AssertionError as err:
if PLATFORM_WIN32:
pytest.xfail('Known to randomly fail on Win32')
raise err
def test_corr_nearest_factor_arrpack(self):
# regression results for svds call
u2 = np.array([[
6.39407581e-19, 9.15225947e-03, 1.82631698e-02, 2.72917181e-02,
3.61975557e-02, 4.49413101e-02, 5.34848732e-02, 6.17916613e-02,
6.98268388e-02, 7.75575058e-02, 8.49528448e-02, 9.19842264e-02,
9.86252769e-02, 1.04851906e-01, 1.10642305e-01, 1.15976906e-01,
1.20838331e-01, 1.25211306e-01, 1.29082570e-01, 1.32440778e-01,
1.35276397e-01, 1.37581605e-01, 1.39350201e-01, 1.40577526e-01,
1.41260396e-01, 1.41397057e-01, 1.40987160e-01, 1.40031756e-01,
1.38533306e-01, 1.36495727e-01, 1.33924439e-01, 1.30826443e-01,
1.27210404e-01, 1.23086750e-01, 1.18467769e-01, 1.13367717e-01,
1.07802909e-01, 1.01791811e-01, 9.53551023e-02, 8.85157320e-02,
8.12989329e-02, 7.37322125e-02, 6.58453049e-02, 5.76700847e-02,
4.92404406e-02, 4.05921079e-02, 3.17624629e-02, 2.27902803e-02,
1.37154584e-02, 4.57871801e-03, -4.57871801e-03, -1.37154584e-02,
-2.27902803e-02, -3.17624629e-02, -4.05921079e-02, -4.92404406e-02,
-5.76700847e-02, -6.58453049e-02, -7.37322125e-02, -8.12989329e-02,
-8.85157320e-02, -9.53551023e-02, -1.01791811e-01, -1.07802909e-01,
-1.13367717e-01, -1.18467769e-01, -1.23086750e-01, -1.27210404e-01,
-1.30826443e-01, -1.33924439e-01, -1.36495727e-01, -1.38533306e-01,
-1.40031756e-01, -1.40987160e-01, -1.41397057e-01, -1.41260396e-01,
-1.40577526e-01, -1.39350201e-01, -1.37581605e-01, -1.35276397e-01,
-1.32440778e-01, -1.29082570e-01, -1.25211306e-01, -1.20838331e-01,
-1.15976906e-01, -1.10642305e-01, -1.04851906e-01, -9.86252769e-02,
-9.19842264e-02, -8.49528448e-02, -7.75575058e-02, -6.98268388e-02,
-6.17916613e-02, -5.34848732e-02, -4.49413101e-02, -3.61975557e-02,
-2.72917181e-02, -1.82631698e-02, -9.15225947e-03, -3.51829569e-17
]]).T
s2 = np.array([24.88812183])
d = 100
dm = 1
# Construct a test matrix with exact factor structure
X = np.zeros((d, dm), dtype=np.float64)
x = np.linspace(0, 2 * np.pi, d)
for j in range(dm):
X[:, j] = np.sin(x * (j + 1))
_project_correlation_factors(X)
X *= 0.7
mat = np.dot(X, X.T)
np.fill_diagonal(mat, 1.)
from scipy.sparse.linalg import svds
u, s, vt = svds(mat, dm)
#difference in sign
dsign = np.sign(u[1]) * np.sign(u2[1])
assert_allclose(u, dsign * u2, rtol=1e-6, atol=1e-14)
assert_allclose(s, s2, rtol=1e-6)
def test_corr_nearest_factor_sparse(self, dm):
# Test that result is the same if the input is dense or sparse
d = 100
# Generate a test matrix of factors
X = np.zeros((d, dm), dtype=np.float64)
x = np.linspace(0, 2 * np.pi, d)
rs = np.random.RandomState(10)
for j in range(dm):
X[:, j] = np.sin(x * (j + 1)) + 1e-10 * rs.randn(d)
# Get the correlation matrix
_project_correlation_factors(X)
X *= 0.7
mat = np.dot(X, X.T)
np.fill_diagonal(mat, 1)
# Threshold it
mat.flat[np.abs(mat.flat) < 0.35] = 0.0
# Replace line below which left signed 0
# mat *= (np.abs(mat) >= 0.35)
smat = sparse.csr_matrix(mat)
try:
dense_rslt = corr_nearest_factor(mat, dm, maxiter=10000)
sparse_rslt = corr_nearest_factor(smat, dm, maxiter=10000)
mat_dense = dense_rslt.corr.to_matrix()
mat_sparse = sparse_rslt.corr.to_matrix()
assert dense_rslt.Converged is True
assert sparse_rslt.Converged is True
assert_allclose(mat_dense, mat_sparse, rtol=.25, atol=1e-3)
except AssertionError as err:
if PLATFORM_WIN32:
pytest.xfail('Known to randomly fail on Win32')
# Some debugging information for CI runs that randomly fail
print(dense_rslt.objective_values)
print(sparse_rslt.objective_values)
locs = np.where(~np.isclose(mat_dense, mat_sparse, rtol=.25, atol=1e-3))
print(mat_sparse[locs])
print(mat_dense[locs])
raise err
def test_corr_nearest_factor(self):
d = 100
for dm in 1,2:
# Construct a test matrix with exact factor structure
X = np.zeros((d,dm), dtype=np.float64)
x = np.linspace(0, 2*np.pi, d)
for j in range(dm):
X[:,j] = np.sin(x*(j+1))
_project_correlation_factors(X)
X *= 0.7
mat = np.dot(X, X.T)
np.fill_diagonal(mat, 1.)
# Try to recover the structure
rslt = corr_nearest_factor(mat, dm)
assert_equal(rslt.Converged, True)
mat1 = rslt.corr.to_matrix()
assert_allclose(mat, mat1, rtol=0.25, atol=1e-3)
def test_corr_nearest_factor(self):
d = 100
for dm in 1,2:
# Construct a test matrix with exact factor structure
X = np.zeros((d,dm), dtype=np.float64)
x = np.linspace(0, 2*np.pi, d)
for j in range(dm):
X[:,j] = np.sin(x*(j+1))
_project_correlation_factors(X)
X *= 0.7
mat = np.dot(X, X.T)
np.fill_diagonal(mat, 1.)
# Try to recover the structure
rslt = corr_nearest_factor(mat, dm)
C = rslt.corr
mat1 = C.to_matrix()
assert(np.abs(mat - mat1).max() < 1e-3)
def test_corr_nearest_factor_arrpack(self):
# regression results for svds call
u2 = np.array([[
6.39407581e-19, 9.15225947e-03, 1.82631698e-02,
2.72917181e-02, 3.61975557e-02, 4.49413101e-02,
5.34848732e-02, 6.17916613e-02, 6.98268388e-02,
7.75575058e-02, 8.49528448e-02, 9.19842264e-02,
9.86252769e-02, 1.04851906e-01, 1.10642305e-01,
1.15976906e-01, 1.20838331e-01, 1.25211306e-01,
1.29082570e-01, 1.32440778e-01, 1.35276397e-01,
1.37581605e-01, 1.39350201e-01, 1.40577526e-01,
1.41260396e-01, 1.41397057e-01, 1.40987160e-01,
1.40031756e-01, 1.38533306e-01, 1.36495727e-01,
1.33924439e-01, 1.30826443e-01, 1.27210404e-01,
1.23086750e-01, 1.18467769e-01, 1.13367717e-01,
1.07802909e-01, 1.01791811e-01, 9.53551023e-02,
8.85157320e-02, 8.12989329e-02, 7.37322125e-02,
6.58453049e-02, 5.76700847e-02, 4.92404406e-02,
4.05921079e-02, 3.17624629e-02, 2.27902803e-02,
1.37154584e-02, 4.57871801e-03, -4.57871801e-03,
-1.37154584e-02, -2.27902803e-02, -3.17624629e-02,
-4.05921079e-02, -4.92404406e-02, -5.76700847e-02,
-6.58453049e-02, -7.37322125e-02, -8.12989329e-02,
-8.85157320e-02, -9.53551023e-02, -1.01791811e-01,
-1.07802909e-01, -1.13367717e-01, -1.18467769e-01,
-1.23086750e-01, -1.27210404e-01, -1.30826443e-01,
-1.33924439e-01, -1.36495727e-01, -1.38533306e-01,
-1.40031756e-01, -1.40987160e-01, -1.41397057e-01,
-1.41260396e-01, -1.40577526e-01, -1.39350201e-01,
-1.37581605e-01, -1.35276397e-01, -1.32440778e-01,
-1.29082570e-01, -1.25211306e-01, -1.20838331e-01,
-1.15976906e-01, -1.10642305e-01, -1.04851906e-01,
-9.86252769e-02, -9.19842264e-02, -8.49528448e-02,
-7.75575058e-02, -6.98268388e-02, -6.17916613e-02,
-5.34848732e-02, -4.49413101e-02, -3.61975557e-02,
-2.72917181e-02, -1.82631698e-02, -9.15225947e-03,
-3.51829569e-17]]).T
s2 = np.array([ 24.88812183])
d = 100
dm = 1
# Construct a test matrix with exact factor structure
X = np.zeros((d,dm), dtype=np.float64)
x = np.linspace(0, 2*np.pi, d)
for j in range(dm):
X[:,j] = np.sin(x*(j+1))
_project_correlation_factors(X)
X *= 0.7
mat = np.dot(X, X.T)
np.fill_diagonal(mat, 1.)
from scipy.sparse.linalg import svds
u, s, vt = svds(mat, dm)
#difference in sign
dsign = np.sign(u[1]) * np.sign(u2[1])
assert_allclose(u, dsign * u2, rtol=1e-6, atol=1e-14)
assert_allclose(s, s2, rtol=1e-6)
