def test_standard_run(self, rath_job, cpa):
job_runner = HadoopJobRunner('jar_path', end_job_with_atomic_move_dir=False)
job_runner.run_job(MockStreamingJob())
self.assertEqual(1, cpa.call_count)
self.assertEqual(1, rath_job.call_count)
mr_args = rath_job.call_args[0][0]
mr_args_pairs = zip(mr_args, mr_args[1:])
self.assertIn(('-mapper', 'python mrrunner.py map'), mr_args_pairs)
self.assertIn(('-file', mrrunner.__file__.rstrip('c')), mr_args_pairs)
def test_run_with_extra_arguments(self, rath_job, cpa):
job_runner = HadoopJobRunner('jar_path', end_job_with_atomic_move_dir=False)
job_runner.run_job(MockStreamingJobWithExtraArguments())
self.assertEqual(1, cpa.call_count)
self.assertEqual(1, rath_job.call_count)
mr_args = rath_job.call_args[0][0]
mr_args_pairs = list(zip(mr_args, mr_args[1:]))
self.assertIn(('-myargument', '/path/to/coolvalue'), mr_args_pairs)
self.assertIn(('-archives', '/path/to/myarchive.zip,/path/to/other_archive.zip'), mr_args_pairs)
def test_standard_run(self, rath_job, cpa):
job_runner = HadoopJobRunner('jar_path', end_job_with_atomic_move_dir=False)
job_runner.run_job(MockStreamingJob())
self.assertEqual(1, cpa.call_count)
self.assertEqual(1, rath_job.call_count)
mr_args = rath_job.call_args[0][0]
mr_args_pairs = zip(mr_args, mr_args[1:])
self.assertIn(('-mapper', 'python mrrunner.py map'), mr_args_pairs)
self.assertIn(('-file', mrrunner.__file__.rstrip('c')), mr_args_pairs)
def test_run_with_extra_arguments(self, rath_job, cpa):
job_runner = HadoopJobRunner('jar_path', end_job_with_atomic_move_dir=False)
job_runner.run_job(MockStreamingJobWithExtraArguments())
self.assertEqual(1, cpa.call_count)
self.assertEqual(1, rath_job.call_count)
mr_args = rath_job.call_args[0][0]
mr_args_pairs = list(zip(mr_args, mr_args[1:]))
self.assertIn(('-myargument', '/path/to/coolvalue'), mr_args_pairs)
self.assertIn(('-archives', '/path/to/myarchive.zip,/path/to/other_archive.zip'), mr_args_pairs)
def test_package_binary_run(self, rath_job, shutil):
job_runner = HadoopJobRunner('jar_path', end_job_with_atomic_move_dir=False)
job_runner.run_job(MockStreamingJob(package_binary='test_bin.pex'))
self.assertEqual(1, shutil.copy.call_count)
pex_src, pex_dest = shutil.copy.call_args[0]
runner_fname = os.path.basename(pex_dest)
self.assertEqual('test_bin.pex', pex_src)
self.assertEqual('mrrunner.pex', runner_fname)
self.assertEqual(1, rath_job.call_count)
mr_args = rath_job.call_args[0][0]
mr_args_pairs = zip(mr_args, mr_args[1:])
self.assertIn(('-mapper', 'python mrrunner.pex map'), mr_args_pairs)
self.assertIn(('-file', pex_dest), mr_args_pairs)
def test_package_binary_run(self, rath_job, shutil):
job_runner = HadoopJobRunner('jar_path', end_job_with_atomic_move_dir=False)
job_runner.run_job(MockStreamingJob(package_binary='test_bin.pex'))
self.assertEqual(1, shutil.copy.call_count)
pex_src, pex_dest = shutil.copy.call_args[0]
runner_fname = os.path.basename(pex_dest)
self.assertEqual('test_bin.pex', pex_src)
self.assertEqual('mrrunner.pex', runner_fname)
self.assertEqual(1, rath_job.call_count)
mr_args = rath_job.call_args[0][0]
mr_args_pairs = zip(mr_args, mr_args[1:])
self.assertIn(('-mapper', 'python mrrunner.pex map'), mr_args_pairs)
self.assertIn(('-file', pex_dest), mr_args_pairs)
