def test_s3_textFile_loop():
random.seed()
fn = '{}/pysparkling_test_{:d}.txt'.format(S3_TEST_PATH, random.random() * 999999.0)
rdd = Context().parallelize('Line {0}'.format(n) for n in range(200))
rdd.saveAsTextFile(fn)
rdd_check = Context().textFile(fn)
assert rdd.count() == rdd_check.count() and all(e1 == e2 for e1, e2 in zip(rdd.collect(), rdd_check.collect()))
def test_hdfs_textFile_loop():
random.seed()
fn = '{}/pysparkling_test_{:d}.txt'.format(HDFS_TEST_PATH, random.random() * 999999.0)
print('HDFS test file: {0}'.format(fn))
rdd = Context().parallelize('Hello World {0}'.format(x) for x in range(10))
rdd.saveAsTextFile(fn)
read_rdd = Context().textFile(fn)
print(rdd.collect())
print(read_rdd.collect())
assert rdd.count() == read_rdd.count() and all(r1 == r2 for r1, r2 in zip(rdd.collect(), read_rdd.collect()))
from __future__ import print_function
from fast_pyspark_tester import Context
my_rdd = Context().textFile('tests/*.py')
print('In tests/*.py: all lines={0}, with import={1}'.format(
my_rdd.count(),
my_rdd.filter(lambda l: l.startswith('import ')).count(),
))