def test_end_to_end(self):
n_file_path = os.path.join(self.tmp_dir, 'n_file')
with open(n_file_path, 'w') as f:
f.write('3')
os.environ['LOCAL_N_FILE_PATH'] = n_file_path
stdin = ['0\n', '1\n', '2\n']
# use local runner so that the file is actually sent somewhere
mr_job = MRTowerOfPowers(
['--no-conf', '-v', '--cleanup=NONE', '--n-file', n_file_path,
'--runner=local'])
self.assertEqual(len(mr_job.steps()), 3)
mr_job.sandbox(stdin=stdin)
with logger_disabled('mrjob.local'):
with mr_job.make_runner() as runner:
# make sure our file gets "uploaded"
assert [fd for fd in runner._files
if fd['path'] == n_file_path]
runner.run()
output = set()
for line in runner.stream_output():
_, value = mr_job.parse_output_line(line)
output.add(value)
self.assertEqual(set(output), set([0, 1, ((2 ** 3) ** 3) ** 3]))
def test_end_to_end(self):
n_file_path = os.path.join(self.tmp_dir, 'n_file')
with open(n_file_path, 'w') as f:
f.write('3')
os.environ['LOCAL_N_FILE_PATH'] = n_file_path
stdin = [b'0\n', b'1\n', b'2\n']
# use local runner so that the file is actually sent somewhere
mr_job = MRTowerOfPowers(
['-v', '--cleanup=NONE', '--n-file', n_file_path,
'--runner=local'])
self.assertEqual(len(mr_job.steps()), 3)
mr_job.sandbox(stdin=stdin)
with logger_disabled('mrjob.local'):
with mr_job.make_runner() as runner:
# make sure our file gets placed in the working dir
self.assertIn(n_file_path, runner._working_dir_mgr.paths())
runner.run()
output = set()
for _, value in mr_job.parse_output(runner.cat_output()):
output.add(value)
self.assertEqual(set(output), set([0, 1, ((2 ** 3) ** 3) ** 3]))
def test_end_to_end(self):
n_file_path = os.path.join(self.tmp_dir, 'n_file')
with open(n_file_path, 'w') as f:
f.write('3')
os.environ['LOCAL_N_FILE_PATH'] = n_file_path
stdin = ['0\n', '1\n', '2\n']
mr_job = MRTowerOfPowers(
['--no-conf', '-v', '--cleanup=NONE', '--n-file', n_file_path])
assert_equal(len(mr_job.steps()), 3)
mr_job.sandbox(stdin=stdin)
with mr_job.make_runner() as runner:
assert isinstance(runner, LocalMRJobRunner)
# make sure our file gets "uploaded"
assert [fd for fd in runner._files if fd['path'] == n_file_path]
runner.run()
output = set()
for line in runner.stream_output():
_, value = mr_job.parse_output_line(line)
output.add(value)
assert_equal(set(output), set([0, 1, ((2**3)**3)**3]))
def test_end_to_end(self):
n_file_path = os.path.join(self.tmp_dir, 'n_file')
with open(n_file_path, 'w') as f:
f.write('3')
os.environ['LOCAL_N_FILE_PATH'] = n_file_path
stdin = ['0\n', '1\n', '2\n']
mr_job = MRTowerOfPowers(['--no-conf', '-v', '--cleanup=NONE', '--n-file', n_file_path])
assert_equal(len(mr_job.steps()), 3)
mr_job.sandbox(stdin=stdin)
with mr_job.make_runner() as runner:
assert isinstance(runner, LocalMRJobRunner)
# make sure our file gets "uploaded"
assert [fd for fd in runner._files if fd['path'] == n_file_path]
runner.run()
output = set()
for line in runner.stream_output():
_, value = mr_job.parse_output_line(line)
output.add(value)
assert_equal(set(output), set([0, 1, ((2**3)**3)**3]))
def test_end_to_end(self):
n_file_path = os.path.join(self.tmp_dir, 'n_file')
with open(n_file_path, 'w') as f:
f.write('3')
os.environ['LOCAL_N_FILE_PATH'] = n_file_path
stdin = [b'0\n', b'1\n', b'2\n']
# use local runner so that the file is actually sent somewhere
mr_job = MRTowerOfPowers(
['-v', '--cleanup=NONE', '--n-file', n_file_path,
'--runner=local'])
self.assertEqual(len(mr_job.steps()), 3)
mr_job.sandbox(stdin=stdin)
with logger_disabled('mrjob.local'):
with mr_job.make_runner() as runner:
# make sure our file gets placed in the working dir
self.assertIn(n_file_path, runner._working_dir_mgr.paths())
runner.run()
output = set()
for _, value in mr_job.parse_output(runner.cat_output()):
output.add(value)
self.assertEqual(set(output), set([0, 1, ((2 ** 3) ** 3) ** 3]))
def test_end_to_end(self):
n_file_path = os.path.join(self.tmp_dir, 'n_file')
with open(n_file_path, 'w') as f:
f.write('3')
os.environ['LOCAL_N_FILE_PATH'] = n_file_path
stdin = ['0\n', '1\n', '2\n']
# use local runner so that the file is actually sent somewhere
mr_job = MRTowerOfPowers(
['--no-conf', '-v', '--cleanup=NONE', '--n-file', n_file_path,
'--runner=local'])
self.assertEqual(len(mr_job.steps()), 3)
mr_job.sandbox(stdin=stdin)
with logger_disabled('mrjob.local'):
with mr_job.make_runner() as runner:
# make sure our file gets "uploaded"
assert [fd for fd in runner._files
if fd['path'] == n_file_path]
runner.run()
output = set()
for line in runner.stream_output():
_, value = mr_job.parse_output_line(line)
output.add(value)
self.assertEqual(set(output), set([0, 1, ((2 ** 3) ** 3) ** 3]))
def test_end_to_end(self):
n_file_path = os.path.join(self.tmp_dir, "n_file")
with open(n_file_path, "w") as f:
f.write("3")
os.environ["LOCAL_N_FILE_PATH"] = n_file_path
stdin = ["0\n", "1\n", "2\n"]
# use local runner so that the file is actually sent somewhere
mr_job = MRTowerOfPowers(["-v", "--cleanup=NONE", "--n-file", n_file_path, "--runner=local"])
self.assertEqual(len(mr_job.steps()), 3)
mr_job.sandbox(stdin=stdin)
with logger_disabled("mrjob.local"):
with mr_job.make_runner() as runner:
# make sure our file gets placed in the working dir
self.assertIn(n_file_path, runner._working_dir_mgr.paths())
runner.run()
output = set()
for line in runner.stream_output():
_, value = mr_job.parse_output_line(line)
output.add(value)
self.assertEqual(set(output), set([0, 1, ((2 ** 3) ** 3) ** 3]))
def _test_file_upload_args_loaded_at_init(self, spark_master):
# can we simulate a MRJob that expects to load files in its
# constructor?
# sanity-check that our test; if MRTowerOfPowers can initialize
# without its --n-file, we're not testing anything
self.assertFalse(exists('n_file'))
self.assertRaises(IOError, MRTowerOfPowers, ['--n-file', 'n_file'])
n_file_path = self.makefile('n_file', b'3')
input_bytes = b'0\n1\n2\n'
# this should work because we can see n_file_path
job = MRTowerOfPowers(['-r', 'spark',
'--spark-master', spark_master,
'--n-file', n_file_path])
job.sandbox(stdin=BytesIO(input_bytes))
with job.make_runner() as runner:
runner.run()
output = {n for _, n in job.parse_output(runner.cat_output())}
self.assertEqual(output,
{0, 1, (((2 ** 3) ** 3) ** 3)})
def _test_file_upload_args_loaded_at_init(self, spark_master):
# can we simulate a MRJob that expects to load files in its
# constructor?
# sanity-check that our test; if MRTowerOfPowers can initialize
# without its --n-file, we're not testing anything
self.assertFalse(exists('n_file'))
self.assertRaises(IOError, MRTowerOfPowers, ['--n-file', 'n_file'])
n_file_path = self.makefile('n_file', b'3')
input_bytes = b'0\n1\n2\n'
# this should work because we can see n_file_path
job = MRTowerOfPowers([
'-r', 'spark', '--spark-master', spark_master, '--n-file',
n_file_path
])
job.sandbox(stdin=BytesIO(input_bytes))
with job.make_runner() as runner:
runner.run()
output = {n for _, n in job.parse_output(runner.cat_output())}
self.assertEqual(output, {0, 1, (((2**3)**3)**3)})
