def test_ind_to_sub_array(self): data_local = map(lambda x: (x, array([1.0])), range(1, 13)) dims = [2, 3, 2] subs = map(lambda x: x[0], indtosub(data_local, dims)) assert (allclose( subs, array([(1, 1, 1), (2, 1, 1), (1, 2, 1), (2, 2, 1), (1, 3, 1), (2, 3, 1), (1, 1, 2), (2, 1, 2), (1, 2, 2), (2, 2, 2), (1, 3, 2), (2, 3, 2)])))
def test_ind_to_sub_rdd(self): data_local = map(lambda x: (x, array([1.0])), range(1, 13)) data = self.sc.parallelize(data_local) dims = [2, 3, 2] #subs = data.map(lambda (k, _): indtosub(k, dims)).collect() subs = indtosub(data, dims).map(lambda (k, _): k).collect() assert(allclose(subs, array([(1, 1, 1), (2, 1, 1), (1, 2, 1), (2, 2, 1), (1, 3, 1), (2, 3, 1), (1, 1, 2), (2, 1, 2), (1, 2, 2), (2, 2, 2), (1, 3, 2), (2, 3, 2)])))
def test_ind_to_sub_rdd(self): data_local = map(lambda x: (x, array([1.0])), range(1, 13)) data = self.sc.parallelize(data_local) dims = [2, 3, 2] #subs = data.map(lambda (k, _): indtosub(k, dims)).collect() subs = indtosub(data, dims).map(lambda (k, _): k).collect() assert (allclose( subs, array([(1, 1, 1), (2, 1, 1), (1, 2, 1), (2, 2, 1), (1, 3, 1), (2, 3, 1), (1, 1, 2), (2, 1, 2), (1, 2, 2), (2, 2, 2), (1, 3, 2), (2, 3, 2)])))
def test_ind_to_sub_array(self): data_local = map(lambda x: (x, array([1.0])), range(1, 13)) dims = [2, 3, 2] subs = map(lambda x: x[0], indtosub(data_local, dims)) assert(allclose(subs, array([(1, 1, 1), (2, 1, 1), (1, 2, 1), (2, 2, 1), (1, 3, 1), (2, 3, 1), (1, 1, 2), (2, 1, 2), (1, 2, 2), (2, 2, 2), (1, 3, 2), (2, 3, 2)])))