def test_spark_method(self): j = MRJob(["--no-conf"]) j.spark = MagicMock() self.assertEqual(j.steps(), [SparkStep(j.spark)]) self.assertEqual(j._steps_desc(), [dict(type="spark", spark_args=[])])
def test_spark_and_spark_args_methods(self): j = MRJob(["--no-conf"]) j.spark = MagicMock() j.spark_args = MagicMock(return_value=["argh", "ARRRRGH!"]) self.assertEqual(j.steps(), [SparkStep(j.spark, spark_args=["argh", "ARRRRGH!"])]) self.assertEqual(j._steps_desc(), [dict(type="spark", spark_args=["argh", "ARRRRGH!"])])
def test_spark_with_step_num(self): job = MRJob(["--step-num=1", "--spark", "input_dir", "output_dir"]) mapper = MagicMock() spark = MagicMock() job.steps = Mock(return_value=[MRStep(mapper=mapper), SparkStep(spark)]) job.execute() spark.assert_called_once_with("input_dir", "output_dir") self.assertFalse(mapper.called)
def test_spark_method(self): j = MRJob(['--no-conf']) j.spark = MagicMock() self.assertEqual( j.steps(), [SparkStep(j.spark)] ) self.assertEqual( j._steps_desc(), [dict(type='spark', jobconf={}, spark_args=[])] )
def test_spark_and_spark_args_methods(self): j = MRJob(['--no-conf']) j.spark = MagicMock() j.spark_args = MagicMock(return_value=['argh', 'ARRRRGH!']) self.assertEqual( j.steps(), [SparkStep(j.spark, spark_args=['argh', 'ARRRRGH!'])] ) self.assertEqual( j._steps_desc(), [dict(type='spark', jobconf={}, spark_args=['argh', 'ARRRRGH!'])] )
def test_spark_with_step_num(self): job = MRJob(['--step-num=1', '--spark', 'input_dir', 'output_dir']) mapper = MagicMock() spark = MagicMock() job.steps = Mock( return_value=[MRStep( mapper=mapper), SparkStep(spark)]) job.execute() spark.assert_called_once_with('input_dir', 'output_dir') self.assertFalse(mapper.called)
def test_spark_args_ignored_without_spark(self): j = MRJob(['--no-conf']) j.reducer = MagicMock() j.spark_args = MagicMock(spark_args=['argh', 'ARRRRGH!']) self.assertEqual(j.steps(), [MRStep(reducer=j.reducer)])
def test_spark_args_ignored_without_spark(self): j = MRJob(["--no-conf"]) j.reducer = MagicMock() j.spark_args = MagicMock(spark_args=["argh", "ARRRRGH!"]) self.assertEqual(j.steps(), [MRStep(reducer=j.reducer)])