def test_spark_script_interpolation(self): job = MRSparkScript(['--script-arg', OUTPUT, '--script-arg', INPUT]) job.sandbox() with job.make_runner() as runner: self.assertEqual(runner._spark_script_args(0), ['<step 0 output>', '<step 0 input>'])
def test_spark_script(self): job = MRSparkScript(['--script-arg', 'foo', '--script-arg', 'bar']) job.sandbox() with job.make_runner() as runner: self.assertEqual( runner._spark_script_args(0), ['foo', 'bar'])
def test_spark_script_interpolation(self): job = MRSparkScript(['--script-arg', OUTPUT, '--script-arg', INPUT]) job.sandbox() with job.make_runner() as runner: self.assertEqual( runner._spark_script_args(0), ['<step 0 output>', '<step 0 input>'])
def test_spark_script(self): job = MRSparkScript(['--script-arg', 'foo', '--script-arg', 'bar']) job.sandbox() with job.make_runner() as runner: self.assertEqual( runner._spark_script_args(0), ['foo', 'bar'])
def test_spark_script_step_without_mr_job_script(self): spark_script_path = self.makefile('a_spark_script.py') steps = MRSparkScript(['--script', spark_script_path])._steps_desc() runner = EMRJobRunner(steps=steps, stdin=BytesIO()) runner.run() runner.cleanup()