def test_convert_toiter(self): data = np.arange(40) rdd = self.sc.parallelize(data, 4) X = ArrayRDD(rdd, 5) X_iter = X.toiter() assert_is_instance(X_iter, collections.Iterator) assert_array_equal(list(X_iter), X.collect())
def test_transform(self): data = np.arange(400).reshape((100, 4)) rdd = self.sc.parallelize(data, 4) X = ArrayRDD(rdd, 5) fn = lambda x: x ** 2 X1 = map(fn, X.collect()) X2 = X.transform(fn).collect() assert_array_equal(X1, X2)