def test_spark_script_interpolation(self):
job = MRSparkScript(['--script-arg', OUTPUT, '--script-arg', INPUT])
job.sandbox()
with job.make_runner() as runner:
self.assertEqual(runner._spark_script_args(0),
['<step 0 output>', '<step 0 input>'])
def test_spark_script(self):
job = MRSparkScript(['--script-arg', 'foo', '--script-arg', 'bar'])
job.sandbox()
with job.make_runner() as runner:
self.assertEqual(
runner._spark_script_args(0),
['foo', 'bar'])
def test_spark_script_interpolation(self):
job = MRSparkScript(['--script-arg', OUTPUT, '--script-arg', INPUT])
job.sandbox()
with job.make_runner() as runner:
self.assertEqual(
runner._spark_script_args(0),
['<step 0 output>', '<step 0 input>'])
def test_spark_script(self):
job = MRSparkScript(['--script-arg', 'foo', '--script-arg', 'bar'])
job.sandbox()
with job.make_runner() as runner:
self.assertEqual(
runner._spark_script_args(0),
['foo', 'bar'])
def test_spark_script_step_without_mr_job_script(self):
spark_script_path = self.makefile('a_spark_script.py')
steps = MRSparkScript(['--script', spark_script_path])._steps_desc()
runner = EMRJobRunner(steps=steps, stdin=BytesIO())
runner.run()
runner.cleanup()