def test_end_to_end(self): n_file_path = os.path.join(self.tmp_dir, 'n_file') with open(n_file_path, 'w') as f: f.write('3') os.environ['LOCAL_N_FILE_PATH'] = n_file_path stdin = ['0\n', '1\n', '2\n'] mr_job = MRTowerOfPowers(['--no-conf', '-v', '--cleanup=NONE', '--n-file', n_file_path]) assert_equal(len(mr_job.steps()), 3) mr_job.sandbox(stdin=stdin) with mr_job.make_runner() as runner: assert isinstance(runner, LocalMRJobRunner) # make sure our file gets "uploaded" assert [fd for fd in runner._files if fd['path'] == n_file_path] runner.run() output = set() for line in runner.stream_output(): _, value = mr_job.parse_output_line(line) output.add(value) assert_equal(set(output), set([0, 1, ((2**3)**3)**3]))
def test_end_to_end(self): n_file_path = os.path.join(self.tmp_dir, 'n_file') with open(n_file_path, 'w') as f: f.write('3') os.environ['LOCAL_N_FILE_PATH'] = n_file_path stdin = [b'0\n', b'1\n', b'2\n'] # use local runner so that the file is actually sent somewhere mr_job = MRTowerOfPowers( ['-v', '--cleanup=NONE', '--n-file', n_file_path, '--runner=local']) self.assertEqual(len(mr_job.steps()), 3) mr_job.sandbox(stdin=stdin) with logger_disabled('mrjob.local'): with mr_job.make_runner() as runner: # make sure our file gets placed in the working dir self.assertIn(n_file_path, runner._working_dir_mgr.paths()) runner.run() output = set() for _, value in mr_job.parse_output(runner.cat_output()): output.add(value) self.assertEqual(set(output), set([0, 1, ((2 ** 3) ** 3) ** 3]))
def test_end_to_end(self): n_file_path = os.path.join(self.tmp_dir, 'n_file') with open(n_file_path, 'w') as f: f.write('3') os.environ['LOCAL_N_FILE_PATH'] = n_file_path stdin = ['0\n', '1\n', '2\n'] # use local runner so that the file is actually sent somewhere mr_job = MRTowerOfPowers( ['--no-conf', '-v', '--cleanup=NONE', '--n-file', n_file_path, '--runner=local']) self.assertEqual(len(mr_job.steps()), 3) mr_job.sandbox(stdin=stdin) with logger_disabled('mrjob.local'): with mr_job.make_runner() as runner: # make sure our file gets "uploaded" assert [fd for fd in runner._files if fd['path'] == n_file_path] runner.run() output = set() for line in runner.stream_output(): _, value = mr_job.parse_output_line(line) output.add(value) self.assertEqual(set(output), set([0, 1, ((2 ** 3) ** 3) ** 3]))
def test_end_to_end(self): n_file_path = os.path.join(self.tmp_dir, "n_file") with open(n_file_path, "w") as f: f.write("3") os.environ["LOCAL_N_FILE_PATH"] = n_file_path stdin = ["0\n", "1\n", "2\n"] # use local runner so that the file is actually sent somewhere mr_job = MRTowerOfPowers(["-v", "--cleanup=NONE", "--n-file", n_file_path, "--runner=local"]) self.assertEqual(len(mr_job.steps()), 3) mr_job.sandbox(stdin=stdin) with logger_disabled("mrjob.local"): with mr_job.make_runner() as runner: # make sure our file gets placed in the working dir self.assertIn(n_file_path, runner._working_dir_mgr.paths()) runner.run() output = set() for line in runner.stream_output(): _, value = mr_job.parse_output_line(line) output.add(value) self.assertEqual(set(output), set([0, 1, ((2 ** 3) ** 3) ** 3]))
def test_end_to_end(self): n_file_path = os.path.join(self.tmp_dir, 'n_file') with open(n_file_path, 'w') as f: f.write('3') os.environ['LOCAL_N_FILE_PATH'] = n_file_path stdin = ['0\n', '1\n', '2\n'] mr_job = MRTowerOfPowers( ['--no-conf', '-v', '--cleanup=NONE', '--n-file', n_file_path]) assert_equal(len(mr_job.steps()), 3) mr_job.sandbox(stdin=stdin) with mr_job.make_runner() as runner: assert isinstance(runner, LocalMRJobRunner) # make sure our file gets "uploaded" assert [fd for fd in runner._files if fd['path'] == n_file_path] runner.run() output = set() for line in runner.stream_output(): _, value = mr_job.parse_output_line(line) output.add(value) assert_equal(set(output), set([0, 1, ((2**3)**3)**3]))
def _test_file_upload_args_loaded_at_init(self, spark_master): # can we simulate a MRJob that expects to load files in its # constructor? # sanity-check that our test; if MRTowerOfPowers can initialize # without its --n-file, we're not testing anything self.assertFalse(exists('n_file')) self.assertRaises(IOError, MRTowerOfPowers, ['--n-file', 'n_file']) n_file_path = self.makefile('n_file', b'3') input_bytes = b'0\n1\n2\n' # this should work because we can see n_file_path job = MRTowerOfPowers([ '-r', 'spark', '--spark-master', spark_master, '--n-file', n_file_path ]) job.sandbox(stdin=BytesIO(input_bytes)) with job.make_runner() as runner: runner.run() output = {n for _, n in job.parse_output(runner.cat_output())} self.assertEqual(output, {0, 1, (((2**3)**3)**3)})
def _test_file_upload_args_loaded_at_init(self, spark_master): # can we simulate a MRJob that expects to load files in its # constructor? # sanity-check that our test; if MRTowerOfPowers can initialize # without its --n-file, we're not testing anything self.assertFalse(exists('n_file')) self.assertRaises(IOError, MRTowerOfPowers, ['--n-file', 'n_file']) n_file_path = self.makefile('n_file', b'3') input_bytes = b'0\n1\n2\n' # this should work because we can see n_file_path job = MRTowerOfPowers(['-r', 'spark', '--spark-master', spark_master, '--n-file', n_file_path]) job.sandbox(stdin=BytesIO(input_bytes)) with job.make_runner() as runner: runner.run() output = {n for _, n in job.parse_output(runner.cat_output())} self.assertEqual(output, {0, 1, (((2 ** 3) ** 3) ** 3)})