def test_map(self): for start, stop, step in self.TEST_RANGES: l1 = range(start, stop, step) l2 = map(RDDTests.square, l1) rdd = RDD(list(l1), self.SPARK_CONTEXT) rdd = rdd.map(RDDTests.square) self.assertEquals(rdd.collect(), list(l2))
def test_distinct(self): for start, stop, step in self.TEST_RANGES: l = range(start, stop, step) rdd = RDD(list(l), self.SPARK_CONTEXT) rdd = rdd.map(RDDTests.return_one) rdd = rdd.distinct() if len(l) > 0: self.assertEquals(rdd.collect(), [1]) else: self.assertEquals(rdd.collect(), [])