def test_spark_jar_interpolation(self):
job = MRSparkJar(['--jar-arg', OUTPUT, '--jar-arg', INPUT])
job.sandbox()
with job.make_runner() as runner:
self.assertEqual(runner._spark_script_args(0),
['<step 0 output>', '<step 0 input>'])
def test_spark_jar(self):
job = MRSparkJar(['--jar-arg', 'foo', '--jar-arg', 'bar'])
job.sandbox()
with job.make_runner() as runner:
self.assertEqual(
runner._spark_script_args(0),
['foo', 'bar'])
def test_spark_jar_interpolation(self):
job = MRSparkJar(['--jar-arg', OUTPUT, '--jar-arg', INPUT])
job.sandbox()
with job.make_runner() as runner:
self.assertEqual(
runner._spark_script_args(0),
['<step 0 output>', '<step 0 input>'])
def test_spark_jar(self):
job = MRSparkJar(['--jar-arg', 'foo', '--jar-arg', 'bar'])
job.sandbox()
with job.make_runner() as runner:
self.assertEqual(
runner._spark_script_args(0),
['foo', 'bar'])
def test_spark_jar_step_without_mr_job_script(self):
spark_jar_path = self.makefile('fireflies.jar')
steps = MRSparkJar(['--jar', spark_jar_path])._steps_desc()
runner = EMRJobRunner(steps=steps, stdin=BytesIO())
runner.run()
runner.cleanup()