def test_spark_jar_interpolation(self): job = MRSparkJar(['--jar-arg', OUTPUT, '--jar-arg', INPUT]) job.sandbox() with job.make_runner() as runner: self.assertEqual(runner._spark_script_args(0), ['<step 0 output>', '<step 0 input>'])
def test_spark_jar(self): job = MRSparkJar(['--jar-arg', 'foo', '--jar-arg', 'bar']) job.sandbox() with job.make_runner() as runner: self.assertEqual( runner._spark_script_args(0), ['foo', 'bar'])
def test_spark_jar_interpolation(self): job = MRSparkJar(['--jar-arg', OUTPUT, '--jar-arg', INPUT]) job.sandbox() with job.make_runner() as runner: self.assertEqual( runner._spark_script_args(0), ['<step 0 output>', '<step 0 input>'])
def test_spark_jar(self): job = MRSparkJar(['--jar-arg', 'foo', '--jar-arg', 'bar']) job.sandbox() with job.make_runner() as runner: self.assertEqual( runner._spark_script_args(0), ['foo', 'bar'])
def test_spark_jar_step_without_mr_job_script(self): spark_jar_path = self.makefile('fireflies.jar') steps = MRSparkJar(['--jar', spark_jar_path])._steps_desc() runner = EMRJobRunner(steps=steps, stdin=BytesIO()) runner.run() runner.cleanup()