def test_spark_mr_job(self):
job = MRNullSpark()
job.sandbox()
with job.make_runner() as runner:
self.assertEqual(runner._spark_script_args(0), [
'--step-num=0', '--spark', '<step 0 input>', '<step 0 output>'
])
def test_spark_passthrough_arg(self):
job = MRNullSpark(['--extra-spark-arg=--verbose'])
job.sandbox()
with job.make_runner() as runner:
self.assertEqual(runner._spark_script_args(0), [
'--step-num=0', '--spark', '--extra-spark-arg=--verbose',
'<step 0 input>', '<step 0 output>'
])
def test_spark_mr_job(self):
job = MRNullSpark()
job.sandbox()
with job.make_runner() as runner:
self.assertEqual(
runner._spark_script_args(0),
['--step-num=0',
'--spark',
'<step 0 input>',
'<step 0 output>'])
def test_spark_passthrough_arg(self):
job = MRNullSpark(['--extra-spark-arg=--verbose'])
job.sandbox()
with job.make_runner() as runner:
self.assertEqual(
runner._spark_script_args(0),
['--step-num=0',
'--spark',
'--extra-spark-arg=--verbose',
'<step 0 input>',
'<step 0 output>'])
def test_dont_upload_mrjob_zip(self):
job = MRNullSpark(['-r', 'spark', '--spark-master', 'yarn'])
job.sandbox()
with job.make_runner() as runner:
runner.run()
self.assertTrue(exists(runner._mrjob_zip_path))
self.assertNotIn(runner._mrjob_zip_path,
runner._upload_mgr.path_to_uri())
self.assertIn(runner._mrjob_zip_path, runner._spark_submit_args(0))
def test_dont_upload_mrjob_zip(self):
job = MRNullSpark(['-r', 'spark', '--spark-master', 'yarn'])
job.sandbox()
with job.make_runner() as runner:
runner.run()
self.assertTrue(exists(runner._mrjob_zip_path))
self.assertNotIn(runner._mrjob_zip_path,
runner._upload_mgr.path_to_uri())
self.assertIn(runner._mrjob_zip_path, runner._spark_submit_args(0))
def test_eggs(self):
egg1_path = self.makefile('dragon.egg')
egg2_path = self.makefile('horton.egg')
job = MRNullSpark(
['-r', 'spark', '--py-files',
'%s,%s' % (egg1_path, egg2_path)])
job.sandbox()
with job.make_runner() as runner:
runner.run()
py_files_arg = '%s,%s,%s' % (egg1_path, egg2_path,
runner._mrjob_zip_path)
self.assertIn(py_files_arg, runner._spark_submit_args(0))
def test_eggs(self):
egg1_path = self.makefile('dragon.egg')
egg2_path = self.makefile('horton.egg')
job = MRNullSpark([
'-r', 'spark',
'--py-files', '%s,%s' % (egg1_path, egg2_path)])
job.sandbox()
with job.make_runner() as runner:
runner.run()
py_files_arg = '%s,%s,%s' % (
egg1_path, egg2_path, runner._mrjob_zip_path)
self.assertIn(py_files_arg, runner._spark_submit_args(0))
def test_spark_file_arg(self):
foo_path = self.makefile('foo')
job = MRNullSpark(['--extra-file', foo_path])
job.sandbox()
with job.make_runner() as runner:
self.assertEqual(runner._spark_script_args(0), [
'--step-num=0', '--spark', '--extra-file', 'foo',
'<step 0 input>', '<step 0 output>'
])
name_to_path = runner._working_dir_mgr.name_to_path('file')
self.assertIn('foo', name_to_path)
self.assertEqual(name_to_path['foo'], foo_path)
def test_spark_step_without_mr_job_script(self):
steps = MRNullSpark()._steps_desc()
# need to be able to call the script's spark() method
self.assertRaises(ValueError,
EMRJobRunner,
steps=steps,
stdin=BytesIO())
def test_spark_file_arg(self):
foo_path = self.makefile('foo')
job = MRNullSpark(['--extra-file', foo_path])
job.sandbox()
with job.make_runner() as runner:
self.assertEqual(
runner._spark_script_args(0),
['--step-num=0',
'--spark',
'--extra-file',
'foo',
'<step 0 input>',
'<step 0 output>'])
name_to_path = runner._working_dir_mgr.name_to_path('file')
self.assertIn('foo', name_to_path)
self.assertEqual(name_to_path['foo'], foo_path)
def test_no_spark_steps(self):
# just a sanity check; _STEP_TYPES is tested in a lot of ways
job = MRNullSpark(['-r', 'local'])
job.sandbox()
self.assertRaises(NotImplementedError, job.make_runner)