def test_hdfs_file_exists():
random.seed()
fn1 = f'{HDFS_TEST_PATH}/pysparkling_test_{random.random() * 999999.0:d}.txt'
fn2 = f'{HDFS_TEST_PATH}/pysparkling_test_{random.random() * 999999.0:d}.txt'
rdd = Context().parallelize(f'Hello World {x}' for x in range(10))
rdd.saveAsTextFile(fn1)
assert File(fn1).exists() and not File(fn2).exists()
def test_s3_textFile_loop():
random.seed()
fn = f'{S3_TEST_PATH}/pysparkling_test_{random.random() * 999999.0:d}.txt'
rdd = Context().parallelize(f'Line {n}' for n in range(200))
rdd.saveAsTextFile(fn)
rdd_check = Context().textFile(fn)
assert (rdd.count() == rdd_check.count()
and all(e1 == e2
for e1, e2 in zip(rdd.collect(), rdd_check.collect())))
def test_hdfs_file_exists():
random.seed()
fn1 = '{}/pysparkling_test_{:d}.txt'.format(HDFS_TEST_PATH,
random.random() * 999999.0)
fn2 = '{}/pysparkling_test_{:d}.txt'.format(HDFS_TEST_PATH,
random.random() * 999999.0)
rdd = Context().parallelize('Hello World {0}'.format(x) for x in range(10))
rdd.saveAsTextFile(fn1)
assert File(fn1).exists() and not File(fn2).exists()
def test_hdfs_file_exists():
random.seed()
fn1 = '{}/pysparkling_test_{:d}.txt'.format(
HDFS_TEST_PATH, random.random() * 999999.0)
fn2 = '{}/pysparkling_test_{:d}.txt'.format(
HDFS_TEST_PATH, random.random() * 999999.0)
rdd = Context().parallelize('Hello World {0}'.format(x) for x in range(10))
rdd.saveAsTextFile(fn1)
assert File(fn1).exists() and not File(fn2).exists()
def test_gs_textFile_loop():
random.seed()
fn = '{}/pysparkling_test_{:d}.txt'.format(GS_TEST_PATH,
random.random() * 999999.0)
rdd = Context().parallelize('Line {0}'.format(n) for n in range(200))
rdd.saveAsTextFile(fn)
rdd_check = Context().textFile(fn)
assert (rdd.count() == rdd_check.count()
and all(e1 == e2
for e1, e2 in zip(rdd.collect(), rdd_check.collect())))
def test_hdfs_textFile_loop():
random.seed()
fn = f'{HDFS_TEST_PATH}/pysparkling_test_{random.random() * 999999.0:d}.txt'
print(f'HDFS test file: {fn}')
rdd = Context().parallelize(f'Hello World {x}' for x in range(10))
rdd.saveAsTextFile(fn)
read_rdd = Context().textFile(fn)
print(rdd.collect())
print(read_rdd.collect())
assert (rdd.count() == read_rdd.count()
and all(r1 == r2
for r1, r2 in zip(rdd.collect(), read_rdd.collect())))
def test_gs_textFile_loop():
random.seed()
fn = '{}/pysparkling_test_{:d}.txt'.format(
GS_TEST_PATH, random.random() * 999999.0)
rdd = Context().parallelize('Line {0}'.format(n) for n in range(200))
rdd.saveAsTextFile(fn)
rdd_check = Context().textFile(fn)
assert (
rdd.count() == rdd_check.count() and
all(e1 == e2 for e1, e2 in zip(rdd.collect(), rdd_check.collect()))
)
def test_hdfs_textFile_loop():
random.seed()
fn = '{}/pysparkling_test_{:d}.txt'.format(HDFS_TEST_PATH,
random.random() * 999999.0)
print('HDFS test file: {0}'.format(fn))
rdd = Context().parallelize('Hello World {0}'.format(x) for x in range(10))
rdd.saveAsTextFile(fn)
read_rdd = Context().textFile(fn)
print(rdd.collect())
print(read_rdd.collect())
assert (rdd.count() == read_rdd.count()
and all(r1 == r2
for r1, r2 in zip(rdd.collect(), read_rdd.collect())))
def test_hdfs_file_exists():
if not HDFS_TEST_PATH:
raise SkipTest
random.seed()
fn1 = HDFS_TEST_PATH + '/pysparkling_test_{0}.txt'.format(
int(random.random() * 999999.0))
fn2 = HDFS_TEST_PATH + '/pysparkling_test_{0}.txt'.format(
int(random.random() * 999999.0))
rdd = Context().parallelize('Hello World {0}'.format(x) for x in range(10))
rdd.saveAsTextFile(fn1)
assert File(fn1).exists() and not File(fn2).exists()
def test_s3_textFile_loop():
if not AWS_ACCESS_KEY_ID or not S3_TEST_PATH:
raise SkipTest
random.seed()
fn = '{}/pysparkling_test_{0}.txt'.format(S3_TEST_PATH,
int(random.random() * 999999.0))
rdd = Context().parallelize("Line {0}".format(n) for n in range(200))
rdd.saveAsTextFile(fn)
rdd_check = Context().textFile(fn)
assert (rdd.count() == rdd_check.count()
and all(e1 == e2
for e1, e2 in zip(rdd.collect(), rdd_check.collect())))
def test_hdfs_textFile_loop():
random.seed()
fn = '{}/pysparkling_test_{:d}.txt'.format(
HDFS_TEST_PATH, random.random() * 999999.0)
print('HDFS test file: {0}'.format(fn))
rdd = Context().parallelize('Hello World {0}'.format(x) for x in range(10))
rdd.saveAsTextFile(fn)
read_rdd = Context().textFile(fn)
print(rdd.collect())
print(read_rdd.collect())
assert (
rdd.count() == read_rdd.count() and
all(r1 == r2 for r1, r2 in zip(rdd.collect(), read_rdd.collect()))
)
def test_gs_textFile_loop():
if not OAUTH2_CLIENT_ID or not GS_TEST_PATH:
raise SkipTest
random.seed()
fn = GS_TEST_PATH + '/pysparkling_test_{0}.txt'.format(
int(random.random() * 999999.0))
rdd = Context().parallelize("Line {0}".format(n) for n in range(200))
rdd.saveAsTextFile(fn)
rdd_check = Context().textFile(fn)
assert (rdd.count() == rdd_check.count()
and all(e1 == e2
for e1, e2 in zip(rdd.collect(), rdd_check.collect())))
def test_hdfs_textFile_loop():
if not HDFS_TEST_PATH:
raise SkipTest
random.seed()
fn = HDFS_TEST_PATH+'/pysparkling_test_{0}.txt'.format(
int(random.random()*999999.0)
)
rdd = Context().parallelize('Hello World {0}'.format(x) for x in range(10))
rdd.saveAsTextFile(fn)
read_rdd = Context().textFile(fn)
assert (
rdd.count() == read_rdd.count() and
all(r1 == r2 for r1, r2 in zip(rdd.collect(), read_rdd.collect()))
)
def test_hdfs_file_exists():
if not HDFS_TEST_PATH:
raise SkipTest
random.seed()
fn1 = HDFS_TEST_PATH+'/pysparkling_test_{0}.txt'.format(
int(random.random()*999999.0)
)
fn2 = HDFS_TEST_PATH+'/pysparkling_test_{0}.txt'.format(
int(random.random()*999999.0)
)
rdd = Context().parallelize('Hello World {0}'.format(x) for x in range(10))
rdd.saveAsTextFile(fn1)
assert File(fn1).exists() and not File(fn2).exists()
def test_s3_textFile_loop():
if not AWS_ACCESS_KEY_ID or not S3_TEST_PATH:
raise SkipTest
random.seed()
fn = S3_TEST_PATH+'/pysparkling_test_{0}.txt'.format(
int(random.random()*999999.0)
)
rdd = Context().parallelize("Line {0}".format(n) for n in range(200))
rdd.saveAsTextFile(fn)
rdd_check = Context().textFile(fn)
assert (
rdd.count() == rdd_check.count() and
all(e1 == e2 for e1, e2 in zip(rdd.collect(), rdd_check.collect()))
)
def test_gs_textFile_loop():
if not OAUTH2_CLIENT_ID or not GS_TEST_PATH:
raise SkipTest
random.seed()
fn = '{}/pysparkling_test_{0}.txt'.format(
GS_TEST_PATH, int(random.random() * 999999.0)
)
rdd = Context().parallelize("Line {0}".format(n) for n in range(200))
rdd.saveAsTextFile(fn)
rdd_check = Context().textFile(fn)
assert (
rdd.count() == rdd_check.count() and
all(e1 == e2 for e1, e2 in zip(rdd.collect(), rdd_check.collect()))
)
