def __init__(self, rows, numRows=0, numCols=0): """ Note: This docstring is not shown publicly. Create a wrapper over a Java RowMatrix. Publicly, we require that `rows` be an RDD. However, for internal usage, `rows` can also be a Java RowMatrix object, in which case we can wrap it directly. This assists in clean matrix conversions. >>> rows = sc.parallelize([[1, 2, 3], [4, 5, 6]]) >>> mat = RowMatrix(rows) >>> mat_diff = RowMatrix(rows) >>> (mat_diff._java_matrix_wrapper._java_model == ... mat._java_matrix_wrapper._java_model) False >>> mat_same = RowMatrix(mat._java_matrix_wrapper._java_model) >>> (mat_same._java_matrix_wrapper._java_model == ... mat._java_matrix_wrapper._java_model) True """ if isinstance(rows, RDD): rows = rows.map(_convert_to_vector) java_matrix = callMLlibFunc("createRowMatrix", rows, long(numRows), int(numCols)) elif (isinstance(rows, JavaObject) and rows.getClass().getSimpleName() == "RowMatrix"): java_matrix = rows else: raise TypeError("rows should be an RDD of vectors, got %s" % type(rows)) self._java_matrix_wrapper = JavaModelWrapper(java_matrix)
def __init__(self, blocks, rowsPerBlock, colsPerBlock, numRows=0, numCols=0): """ Note: This docstring is not shown publicly. Create a wrapper over a Java BlockMatrix. Publicly, we require that `blocks` be an RDD. However, for internal usage, `blocks` can also be a Java BlockMatrix object, in which case we can wrap it directly. This assists in clean matrix conversions. >>> blocks = sc.parallelize([((0, 0), Matrices.dense(3, 2, [1, 2, 3, 4, 5, 6])), ... ((1, 0), Matrices.dense(3, 2, [7, 8, 9, 10, 11, 12]))]) >>> mat = BlockMatrix(blocks, 3, 2) >>> mat_diff = BlockMatrix(blocks, 3, 2) >>> (mat_diff._java_matrix_wrapper._java_model == ... mat._java_matrix_wrapper._java_model) False >>> mat_same = BlockMatrix(mat._java_matrix_wrapper._java_model, 3, 2) >>> (mat_same._java_matrix_wrapper._java_model == ... mat._java_matrix_wrapper._java_model) True """ if isinstance(blocks, RDD): blocks = blocks.map(_convert_to_matrix_block_tuple) # We use DataFrames for serialization of sub-matrix blocks # from Python, so first convert the RDD to a DataFrame on # this side. This will convert each sub-matrix block # tuple to a Row containing the 'blockRowIndex', # 'blockColIndex', and 'subMatrix' values, which can # each be easily serialized. We will convert back to # ((blockRowIndex, blockColIndex), sub-matrix) tuples on # the Scala side. java_matrix = callMLlibFunc("createBlockMatrix", blocks.toDF(), int(rowsPerBlock), int(colsPerBlock), long(numRows), long(numCols)) elif (isinstance(blocks, JavaObject) and blocks.getClass().getSimpleName() == "BlockMatrix"): java_matrix = blocks else: raise TypeError("blocks should be an RDD of sub-matrix blocks as " "((int, int), matrix) tuples, got %s" % type(blocks)) self._java_matrix_wrapper = JavaModelWrapper(java_matrix)
def __init__(self, entries, numRows=0, numCols=0): """ Note: This docstring is not shown publicly. Create a wrapper over a Java CoordinateMatrix. Publicly, we require that `rows` be an RDD. However, for internal usage, `rows` can also be a Java CoordinateMatrix object, in which case we can wrap it directly. This assists in clean matrix conversions. >>> entries = sc.parallelize([MatrixEntry(0, 0, 1.2), ... MatrixEntry(6, 4, 2.1)]) >>> mat = CoordinateMatrix(entries) >>> mat_diff = CoordinateMatrix(entries) >>> (mat_diff._java_matrix_wrapper._java_model == ... mat._java_matrix_wrapper._java_model) False >>> mat_same = CoordinateMatrix(mat._java_matrix_wrapper._java_model) >>> (mat_same._java_matrix_wrapper._java_model == ... mat._java_matrix_wrapper._java_model) True """ if isinstance(entries, RDD): entries = entries.map(_convert_to_matrix_entry) # We use DataFrames for serialization of MatrixEntry entries # from Python, so first convert the RDD to a DataFrame on # this side. This will convert each MatrixEntry to a Row # containing the 'i', 'j', and 'value' values, which can # each be easily serialized. We will convert back to # MatrixEntry inputs on the Scala side. java_matrix = callMLlibFunc("createCoordinateMatrix", entries.toDF(), long(numRows), long(numCols)) elif (isinstance(entries, JavaObject) and entries.getClass().getSimpleName() == "CoordinateMatrix"): java_matrix = entries else: raise TypeError( "entries should be an RDD of MatrixEntry entries or " "(long, long, float) tuples, got %s" % type(entries)) self._java_matrix_wrapper = JavaModelWrapper(java_matrix)
def __init__(self, rows, numRows=0, numCols=0): """ Note: This docstring is not shown publicly. Create a wrapper over a Java IndexedRowMatrix. Publicly, we require that `rows` be an RDD. However, for internal usage, `rows` can also be a Java IndexedRowMatrix object, in which case we can wrap it directly. This assists in clean matrix conversions. >>> rows = sc.parallelize([IndexedRow(0, [1, 2, 3]), ... IndexedRow(1, [4, 5, 6])]) >>> mat = IndexedRowMatrix(rows) >>> mat_diff = IndexedRowMatrix(rows) >>> (mat_diff._java_matrix_wrapper._java_model == ... mat._java_matrix_wrapper._java_model) False >>> mat_same = IndexedRowMatrix(mat._java_matrix_wrapper._java_model) >>> (mat_same._java_matrix_wrapper._java_model == ... mat._java_matrix_wrapper._java_model) True """ if isinstance(rows, RDD): rows = rows.map(_convert_to_indexed_row) # We use DataFrames for serialization of IndexedRows from # Python, so first convert the RDD to a DataFrame on this # side. This will convert each IndexedRow to a Row # containing the 'index' and 'vector' values, which can # both be easily serialized. We will convert back to # IndexedRows on the Scala side. java_matrix = callMLlibFunc("createIndexedRowMatrix", rows.toDF(), long(numRows), int(numCols)) elif (isinstance(rows, JavaObject) and rows.getClass().getSimpleName() == "IndexedRowMatrix"): java_matrix = rows else: raise TypeError( "rows should be an RDD of IndexedRows or (long, vector) tuples, " "got %s" % type(rows)) self._java_matrix_wrapper = JavaModelWrapper(java_matrix)
def tallSkinnyQR(self, computeQ=False): """ Compute the QR decomposition of this RowMatrix. The implementation is designed to optimize the QR decomposition (factorization) for the RowMatrix of a tall and skinny shape. Reference: Paul G. Constantine, David F. Gleich. "Tall and skinny QR factorizations in MapReduce architectures" ([[https://doi.org/10.1145/1996092.1996103]]) :param: computeQ: whether to computeQ :return: QRDecomposition(Q: RowMatrix, R: Matrix), where Q = None if computeQ = false. >>> rows = sc.parallelize([[3, -6], [4, -8], [0, 1]]) >>> mat = RowMatrix(rows) >>> decomp = mat.tallSkinnyQR(True) >>> Q = decomp.Q >>> R = decomp.R >>> # Test with absolute values >>> absQRows = Q.rows.map(lambda row: abs(row.toArray()).tolist()) >>> absQRows.collect() [[0.6..., 0.0], [0.8..., 0.0], [0.0, 1.0]] >>> # Test with absolute values >>> abs(R.toArray()).tolist() [[5.0, 10.0], [0.0, 1.0]] """ decomp = JavaModelWrapper( self._java_matrix_wrapper.call("tallSkinnyQR", computeQ)) if computeQ: java_Q = decomp.call("Q") Q = RowMatrix(java_Q) else: Q = None R = decomp.call("R") return QRDecomposition(Q, R)