def test_helper(A,
M,
N,
R,
sketch,
measures,
MPI,
num_repeats=5,
direction="columnwise"):
"""
Test if the singular values of the original (M x N) and sketched matrix
(R x N) are fulfilling some measurement criteria. The test is repeated
num_repeats times.
"""
results = []
for i in range(num_repeats):
if direction == "columnwise":
S = sketch(M, R)
SA = El.DistMatrix(El.dTag, El.STAR, El.STAR)
El.Uniform(SA, R, N)
else:
S = sketch(N, R)
SA = El.DistMatrix(El.dTag, El.STAR, El.STAR)
El.Uniform(SA, M, R)
S.apply(A, SA, direction)
for m in measures:
results.append(m(SA.Matrix().ToNumPy()))
return results
def _read_elemental_dense(self, colDist=El.MC, rowDist=El.MR):
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
# only the root process touches the filesystem
if rank == 0:
f = h5py.File(self.fpath, 'r')
dataset_obj = f[self.dataset]
shape = dataset_obj.shape if rank == 0 else None
shape = comm.bcast(shape, root=0)
height = shape[0]
width = shape[1]
num_entries = height * width
# max memory capacity per process assumed/hardcoded to 10 blocks
# XXX should this number be passed as a parameter?
max_blocks_per_process = 10
max_block_entries = int(
(1.0 * num_entries) / (max_blocks_per_process * size))
# XXX We could set up a different block generating scheme, e.g. more
# square-ish blocks
block_height = int(numpy.sqrt(max_block_entries))
while max_block_entries % block_height != 0:
block_height = block_height + 1
block_width = max_block_entries / block_height
num_height_blocks = int(numpy.ceil(height / (1.0 * block_height)))
num_width_blocks = int(numpy.ceil(width / (1.0 * block_width)))
num_blocks = num_height_blocks * num_width_blocks
A = El.DistMatrix(colDist=colDist, rowDist=rowDist)
for block in range(num_blocks):
# the global coordinates of the block corners
i_start = (block / num_width_blocks) * block_height
j_start = (block % num_width_blocks) * block_width
i_end = min(height, i_start + block_height)
j_end = min(width, j_start + block_width)
# the block size
local_height = i_end - i_start
local_width = j_end - j_start
# [CIRC, CIRC] matrix is populated by the reader process (i.e. the root)...
A_block = El.DistMatrix(colDist=El.CIRC, rowDist=El.CIRC)
A_block.Resize(local_height, local_width)
if rank == 0:
for j in range(j_start, j_end):
for i in range(i_start, i_end):
A_block.SetLocal(i - i_start, j - j_start,
dataset_obj[i, j])
# ... then a view into the full matrix A is constructed...
A_block_view = A[i_start:i_end, j_start:j_end]
# ... and finally this view is updated by redistribution of the [CIRC, CIRC] block
El.Copy(A_block, A_block_view)
if rank == 0:
f.close()
return A
def load_libsvm_file(fpath, col_row):
"""
Read from a libsv file
:param fpath: path of the file
:param col_row: 0 to read in col mode, 1 to read in row mode
:returns: Returns X, Y matrices
"""
# Read the data
X = El.DistMatrix()
Y = El.DistMatrix()
return sl_io.readlibsvm(fpath, X, Y, 0)
def ConcatFD2D(N0,N1):
A = El.DistMatrix(El.zTag)
height = N0*N1
width = 2*N0*N1
El.Zeros(A,height,width)
localHeight = A.LocalHeight()
A.Reserve(11*localHeight)
for iLoc in xrange(localHeight):
i = A.GlobalRow(iLoc)
x0 = i % N0
x1 = i / N0
iRel = i + N0*N1
A.Update( i, i, El.ComplexDouble(1,1) )
A.Update( i, iRel, El.ComplexDouble(20,2) )
if x0 > 0:
A.Update( i, i-1, El.ComplexDouble(-1,3) )
A.Update( i, iRel-1, El.ComplexDouble(-17,4) )
if x0+1 < N0:
A.Update( i, i+1, El.ComplexDouble(2,5) )
A.Update( i, iRel+1, El.ComplexDouble(-20,6) )
if x1 > 0:
A.Update( i, i-N0, El.ComplexDouble(-30,7) )
A.Update( i, iRel-N0, El.ComplexDouble(-3,8) )
if x1+1 < N1:
A.Update( i, i+N0, El.ComplexDouble(4,9) )
A.Update( i, iRel+N0, El.ComplexDouble(3,10) )
# The dense last column
A.Update( i, width-1, El.ComplexDouble(-10/height) );
return A
def test_approximate_symmetric_svd(self):
"""Compute the SVD of symmetric **A** such that **SVD(A) = V S V^T**"""
n = 100
A = El.DistMatrix()
El.Uniform(A, n, n)
A = A.Matrix()
# Make A symmetric
for i in xrange(0, A.Height()):
for j in xrange(0, i + 1):
A.Set(j, i, A.Get(i, j))
# Usign symmetric SVD
SA = El.Matrix()
VA = El.Matrix()
sl_nla.approximate_symmetric_svd(A, SA, VA, k=n)
# Check result
VAT = El.Matrix()
El.Copy(VA, VAT)
RESULT = El.Matrix()
El.Zeros(RESULT, n, n)
El.DiagonalScale(El.RIGHT, El.NORMAL, SA, VAT)
El.Gemm(El.NORMAL, El.ADJOINT, 1, VAT, VA, 1, RESULT)
self.assertTrue(utils.equal(A, RESULT))
def test_approximate_svd(self):
"""Compute the SVD of **A** such that **SVD(A) = U S V^T**."""
n = 100
# Generate random matrix
A = El.DistMatrix()
El.Uniform(A, n, n)
A = A.Matrix()
# Dimension to apply along.
k = n
U = El.Matrix()
S = El.Matrix()
V = El.Matrix()
sl_nla.approximate_svd(A, U, S, V, k=k)
# Check result
RESULT = El.Matrix()
El.Zeros(RESULT, n, n)
El.DiagonalScale(El.RIGHT, El.NORMAL, S, U)
El.Gemm(El.NORMAL, El.ADJOINT, 1, U, V, 1, RESULT)
self.assertTrue(utils.equal(A, RESULT))
def test_Linear_kernel(self):
"""Test Linear kernel."""
X, Y = sl_test_utils.load_libsvm_file(fpath, 0)
K = El.DistMatrix()
Linear_kernel = sl_kernels.Linear(X.Height())
Linear_kernel.gram(X, K, 0, 0)
self.assertTrue(sl_test_utils.is_kernel(K))
def test_Gaussian_kernel(self):
"""Test Gaussian kernel."""
X, Y = sl_test_utils.load_libsvm_file(fpath, 0)
K = El.DistMatrix()
Gaussian_kernel = sl_kernels.Gaussian(X.Height(), 10.0)
Gaussian_kernel.gram(X, K, 0, 0)
self.assertTrue(sl_test_utils.is_kernel(K))
def test_Laplacian_kernel(self):
"""Test Laplacian kernel."""
X, Y = sl_test_utils.load_libsvm_file(fpath, 0)
K = El.DistMatrix()
Laplacian_kernel = sl_kernels.Laplacian(X.Height(), 2.0)
Laplacian_kernel.gram(X, K, 0, 0)
self.assertTrue(sl_test_utils.is_kernel(K))
def test_Polynomial_kernel(self):
"""Test Polynomial kernel."""
X, Y = sl_test_utils.load_libsvm_file(fpath, 0)
K = El.DistMatrix()
Polynomial_kernel = sl_kernels.Polynomial(X.Height(), 1, 1, 1)
Polynomial_kernel.gram(X, K, 0, 0)
self.assertTrue(sl_test_utils.is_kernel(K, positive=False))
def test_ExpSemiGroup_kernel(self):
"""Test ExpSemiGroup kernel."""
X, Y = sl_test_utils.load_libsvm_file(fpath, 0)
K = El.DistMatrix()
ExpSemiGroup_kernel = sl_kernels.ExpSemiGroup(X.Height(), 0.1)
ExpSemiGroup_kernel.gram(X, K, 0, 0)
self.assertTrue(sl_test_utils.is_kernel(K))
def _write_elemental_dense(self, A):
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
# XXX currently gathers at root
A_CIRC_CIRC = El.DistMatrix(colDist=El.CIRC, rowDist=El.CIRC)
El.Copy(A, A_CIRC_CIRC)
if rank == 0:
A_numpy_dense = A_CIRC_CIRC.Matrix().ToNumPy()
self._write_numpy_dense(A_numpy_dense)
def RectangSparse(height,width):
A = El.DistMatrix()
El.Zeros( A, height, width )
for s in xrange(height):
if s < width: A.Update( s, s, 11 )
if s >= 1 and s-1 < width: A.Update( s, s-1, -1 )
if s+1 < width: A.Update( s, s+1, 2 )
if s >= height and s-height < width: A.Update( s, s-height, -3 )
if s+height < width: A.Update( s, s+height, 4 )
# The dense last column
A.Update( s, width-1, -5/height );
return A
def test_apply_colwise(self):
A = El.DistMatrix(El.dTag, El.VR, El.STAR)
#FIXME: Christos, use your matrix problem factory here
El.Uniform(A, _M, _N)
#FIXME: A.Matrix will not work in parallel
self.sv = np.linalg.svd(A.Matrix().ToNumPy(),
full_matrices=1,
compute_uv=0)
for sketch in self.sketches:
results = test_helper(A, _M, _N, _R, sketch, [self.svd_bound], MPI)
self.check_result(results, str(sketch))
def StackedFD2D(N0, N1):
A = El.DistMatrix()
height = 2 * N0 * N1
width = N0 * N1
A.Resize(height, width)
blocksize = height // worldSize
myStart = blocksize * worldRank
if worldRank == worldSize - 1:
myHeight = height - myStart
else:
myHeight = blocksize
A.Reserve(6 * myHeight)
for sLoc in xrange(localHeight):
s = A.GlobalRow(sLoc)
if s < N0 * N1:
x0 = s % N0
x1 = s / N0
A.QueueUpdate(sLoc, s, 11)
if x0 > 0:
A.QueueUpdate(sLoc, s - 1, -1)
if x0 + 1 < N0:
A.QueueUpdate(sLoc, s + 1, 2)
if x1 > 0:
A.QueueUpdate(sLoc, s - N0, -30)
if x1 + 1 < N1:
A.QueueUpdate(sLoc, s + N0, 4)
else:
sRel = s - N0 * N1
x0 = sRel % N0
x1 = sRel / N0
A.QueueUpdate(sLoc, sRel, -20)
if x0 > 0:
A.QueueUpdate(sLoc, sRel - 1, -17)
if x0 + 1 < N0:
A.QueueUpdate(sLoc, sRel + 1, -20)
if x1 > 0:
A.QueueUpdate(sLoc, sRel - N0, -3)
if x1 + 1 < N1:
A.QueueUpdate(sLoc, sRel + N0, 3)
# The dense last column
A.QueueUpdate(sLoc, width - 1, -10 / height)
A.ProcessQueues()
return A
def _read_elemental_dense_parallel(self, colDist=El.MC, rowDist=El.MR):
f = h5py.File(fpath, 'r', driver='mpio', comm=MPI.COMM_WORLD)
height, width = int(f['shape'][0]), int(f['shape'][1])
A = El.DistMatrix(colDist=colDist, rowDist=colDist)
A.Resize(height, width)
local_height, local_width = A.LocalHeight(), A.LocalWidth()
indices = elemental_dense.get_indices(A)
local_data = f[self.dataset][indices]
local_buf = A.Matrix().Buffer()
for i in range(len(indices)):
local_buf[i] = local_data[i]
f.close()
return A
#
# Copyright (c) 2009-2016, Jack Poulson
# All rights reserved.
#
# This file is part of Elemental and is under the BSD 2-Clause License,
# which can be found in the LICENSE file in the root directory, or at
# http://opensource.org/licenses/BSD-2-Clause
#
import math, El
n = 100 # matrix size
realRes = imagRes = 100 # grid resolution
# Display an instance of the Fox-Li/Landau matrix
A = El.DistMatrix(El.zTag)
El.FoxLi(A,n)
El.Display(A,"Fox-Li/Landau matrix")
# Display its spectral portrait
portrait, box = El.SpectralPortrait(A,realRes,imagRes)
El.DisplayPortrait(portrait,box,"spectral portrait of Fox-Li/Landau matrix")
# Display its singular values
s = El.SingularValues(A)
El.EntrywiseMap(s,math.log10)
El.Display(s,"log10(svd(A))")
# Require the user to press a button before the figures are closed
worldSize = El.mpi.WorldSize()
El.Finalize()
if worldSize == 1:
raw_input('Press Enter to exit')
display = False progress = True worldRank = El.mpi.WorldRank() worldSize = El.mpi.WorldSize() # Make a dense matrix def RectangDense(height,width): A = El.DistMatrix() El.Gaussian( A, height, width ) return A A = RectangDense(m,n) # Generate a b which implies a primal feasible x # ============================================== xGen = El.DistMatrix() El.Uniform(xGen,n,1,0.5,0.4999) b = El.DistMatrix() El.Zeros( b, m, 1 ) El.Gemv( El.NORMAL, 1., A, xGen, 0., b ) # Generate a c which implies a dual feasible (y,z) # ================================================ yGen = El.DistMatrix() El.Gaussian(yGen,m,1) c = El.DistMatrix() El.Uniform(c,n,1,0.5,0.5) El.Gemv( El.TRANSPOSE, -1., A, yGen, 1., c ) if display: El.Display( A, "A" )
# # Copyright (c) 2009-2015, Jack Poulson # All rights reserved. # # This file is part of Elemental and is under the BSD 2-Clause License, # which can be found in the LICENSE file in the root directory, or at # http://opensource.org/licenses/BSD-2-Clause # import math, El k = 140 # matrix size realRes = imagRes = 100 # grid resolution # Display an instance of the pathological example A = El.DistMatrix() El.DruinskyToledo(A, k) El.Display(A, "Bunch-Kaufman growth matrix") # Display the spectral portrait portrait, box = El.SpectralPortrait(A, realRes, imagRes) El.DisplayPortrait(portrait, box, "spectral portrait of BK growth matrix") # Make a copy before overwriting with LDL factorization A_LU = El.DistMatrix() El.Copy(A, A_LU) # Display the relevant pieces of pivoted LDL factorization dSub, p = El.LDL(A, False, El.BUNCH_KAUFMAN_A) El.MakeTrapezoidal(El.LOWER, A) El.Display(dSub, "Subdiagonal of D from LDL") #P = El.DistMatrix(iTag,MC,MR,A.Grid()) # TODO: Construct P from p
def RectangSparse(height,width):
A = El.DistMatrix()
El.Zeros( A, height, width )
for s in xrange(height):
if s < width: A.Update( s, s, 11 )
if s >= 1 and s-1 < width: A.Update( s, s-1, -1 )
if s+1 < width: A.Update( s, s+1, 2 )
if s >= height and s-height < width: A.Update( s, s-height, -3 )
if s+height < width: A.Update( s, s+height, 4 )
# The dense last column
A.Update( s, width-1, -5/height );
return A
# Define a random (affine) hyperplane
wGen = El.DistMatrix()
El.Gaussian( wGen, n, 1 )
wGenNorm = El.FrobeniusNorm( wGen )
El.Scale( 1./wGenNorm, wGen )
El.Print( wGen, "wGen" )
# TODO: Add support for mpi::Broadcast and randomly generate this
offset = 0.3147
# Define a random set of points
A = RectangSparse(m,n)
# Label the points based upon their location relative to the hyperplane
d = El.DistMatrix()
El.Ones( d, m, 1 )
El.Gemv( El.NORMAL, 1., A, wGen, -offset, d )
El.EntrywiseMap( d, lambda alpha : 1. if alpha > 0 else -1. )
m = 2000
n = 1000
display = True
worldRank = El.mpi.WorldRank()
worldSize = El.mpi.WorldSize()
def Rectang(height, width):
A = El.DistMatrix()
El.Uniform(A, height, width)
return A
A = Rectang(m, n)
b = El.DistMatrix()
El.Gaussian(b, m, 1)
if display:
El.Display(A, "A")
El.Display(b, "b")
ctrl = El.LPAffineCtrl_d()
ctrl.mehrotraCtrl.progress = True
startCP = El.mpi.Time()
x = El.CP(A, b, ctrl)
endCP = El.mpi.Time()
if worldRank == 0:
print "CP time:", endCP - startCP, "seconds"
if display:
El.Display(x, "x")
rank = comm.Get_rank()
size = comm.Get_size()
# If on rank 0 read data
if rank == 0:
print "Loading training data..."
trnfile = skylark.io.libsvm(sys.argv[1])
data = trnfile.read()
# Now broadcast the sizes
shape_X = data[0].shape if rank == 0 else None
shape_X = MPI.COMM_WORLD.bcast(shape_X, root=0)
# Transfer to Elemental format and redistribute the matrix and labels
if rank == 0: print "Distributing the matrix..."
X_cc = El.DistMatrix(colDist=El.CIRC, rowDist=El.CIRC)
El.Zeros(X_cc, shape_X[0], shape_X[1])
Y_cc = El.DistMatrix(colDist=El.CIRC, rowDist=El.CIRC)
El.Zeros(Y_cc, shape_X[0], 1)
if rank == 0:
X_ll = X_cc.Matrix()
s = 0
row = 0
for e in data[0].indptr[1:]:
for idx in range(s, e):
col = data[0].indices[idx]
val = data[0].data[idx]
X_ll.Set(row, col, val)
s = e
row = row + 1
# # Copyright (c) 2009-2016, Jack Poulson # All rights reserved. # # This file is part of Elemental and is under the BSD 2-Clause License, # which can be found in the LICENSE file in the root directory, or at # http://opensource.org/licenses/BSD-2-Clause # import math, El n = 100 # matrix size realRes = imagRes = 100 # grid resolution # Display an instance of the pathological example A = El.DistMatrix() El.GEPPGrowth(A,n) El.Display(A,"GEPP growth matrix") # Display the spectral portrait portrait, box = El.SpectralPortrait(A,realRes,imagRes) El.DisplayPortrait(portrait,box,"spectral portrait of GEPP growth matrix") # Display the relevant pieces of pivoted LU factorization p = El.LU(A) El.Display(p,"LU permutation") El.EntrywiseMap(A,lambda x:math.log10(max(abs(x),1))) El.Display(A,"Logarithmically-scaled LU factors") El.Display(A[0:n,n-1],"Last column of logarithmic U") # Require the user to press a button before the figures are closed worldSize = El.mpi.WorldSize() El.Finalize()
m = 200
n = 400
numLambdas = 5
startLambda = 0.01
endLambda = 1
display = True
worldRank = El.mpi.WorldRank()
worldSize = El.mpi.WorldSize()
def Rectang(height,width):
A = El.DistMatrix()
El.Uniform( A, height, width )
return A
A = Rectang(m,n)
b = El.DistMatrix()
El.Gaussian( b, m, 1 )
if display:
El.Display( A, "A" )
El.Display( b, "b" )
ctrl = El.LPAffineCtrl_d()
ctrl.mehrotraCtrl.progress = True
for j in xrange(0,numLambdas):
lambd = startLambda + j*(endLambda-startLambda)/(numLambdas-1.)
if worldRank == 0:
print "lambda =", lambd
startDS = El.mpi.Time()
x = El.DS( A, b, lambd, ctrl )
def RectangDense(height,width): A = El.DistMatrix() El.Gaussian( A, height, width ) return A
progress = True worldRank = El.mpi.WorldRank() worldSize = El.mpi.WorldSize() # Make a dense matrix def RectangDense(height,width): A = El.DistMatrix() El.Gaussian( A, height, width ) return A A = RectangDense(m,n) G = RectangDense(k,n) # Generate a (b,h) which implies a primal feasible (x,s) # ====================================================== xGen = El.DistMatrix() # b := A xGen # ----------- El.Gaussian(xGen,n,1) b = El.DistMatrix() El.Zeros( b, m, 1 ) El.Gemv( El.NORMAL, 1., A, xGen, 0., b ) # h := G xGen + sGen # ------------------ sGen = El.DistMatrix() El.Uniform(sGen,k,1,0.5,0.5) h = El.DistMatrix() El.Copy( sGen, h ) El.Gemv( El.NORMAL, 1., G, xGen, 1., h ) # Generate a c which implies a dual feasible (y,z)
numLambdas = 7
startLambda = 0
endLambda = 1
display = True
worldRank = El.mpi.WorldRank()
worldSize = El.mpi.WorldSize()
def Rectang(height, width):
A = El.DistMatrix()
El.Uniform(A, height, width)
return A
A = Rectang(m, n)
b = El.DistMatrix()
El.Gaussian(b, m, 1)
if display:
El.Display(A, "A")
El.Display(b, "b")
ctrl = El.BPDNCtrl_d()
ctrl.ipmCtrl.mehrotraCtrl.progress = True
for j in xrange(0, numLambdas):
lambd = startLambda + j * (endLambda - startLambda) / (numLambdas - 1.)
if worldRank == 0:
print "lambda =", lambd
startBPDN = El.mpi.Time()
x = El.BPDN(A, b, lambd, ctrl)
def Semidefinite(height):
Q = El.DistMatrix()
El.Identity(Q, height, height)
return Q
def Rectang(height, width):
A = El.DistMatrix()
El.Uniform(A, height, width)
return A
def _usage_tests(usps_path='./datasets/usps.t'):
'''
Various simple example scenaria for showing the usage of the IO facilities
'''
############################################################
# libsvm
############################################################
fpath = usps_path
# read features matrix and labels vector
try:
store = libsvm(fpath)
except ImportError:
print 'Please provide the path to usps.t as an argument'
import sys
sys.exit()
features_matrix, labels_matrix = store.read()
matrix_info = features_matrix.shape, features_matrix.nnz, labels_matrix.shape
# stream features matrix and labels vector
#store = libsvm(fpath)
#for features_matrix, labels_matrix in store.stream(num_features=400, block_size=100):
# matrix_info = features_matrix.shape, features_matrix.nnz, labels_matrix.shape
print 'libsvm OK'
############################################################
# mtx
############################################################
features_fpath = '/tmp/test_features.mtx'
labels_fpath = '/tmp/test_labels.mtx'
# write features and labels
store = mtx(features_fpath)
store.write(features_matrix)
store = mtx(labels_fpath)
store.write(labels_matrix)
# read back features as 'scipy-sparse' and 'combblas-sparse'
store = mtx(features_fpath)
A = store.read('scipy-sparse')
store = mtx(features_fpath)
B = store.read('combblas-sparse')
print 'mtx OK'
############################################################
# hdf5
############################################################
fpath = '/tmp/test_matrix.h5'
# write a random 'numpy-dense' to HDF5 file
store = hdf5(fpath)
A = numpy.random.random((20, 65))
store.write(A)
# read HDF5 file as:
# - 'numpy-dense'
# - 'elemental-dense' (default [MC, MR] distribution)
# - 'elemental-dense' ([VC, STAR] distribution)
B = store.read('numpy-dense')
C = store.read('elemental-dense')
D = store.read('elemental-dense', colDist=El.VC, rowDist=El.STAR)
print 'hdf OK'
############################################################
# txt
############################################################
fpath = '/tmp/test_matrix.txt'
# write a uniform random 'elemental-dense', 'MC_MR' distribution
A = El.DistMatrix()
El.Uniform(A, 10, 30)
store = txt(fpath)
store.write(A)
# read the matrix back as 'numpy-dense'
store = txt(fpath)
A = store.read('numpy-dense')
print 'txt OK'
