def test_spark(self): job = MRJob(["--spark", "input_dir", "output_dir"]) job.spark = MagicMock() job.execute() job.spark.assert_called_once_with("input_dir", "output_dir")
def test_spark(self): job = MRJob(['--spark', 'input_dir', 'output_dir']) job.spark = MagicMock() job.execute() job.spark.assert_called_once_with('input_dir', 'output_dir')
def test_wrong_type_of_step(self): mr_job = MRJob() mr_job.spark = MagicMock() self.assertRaises(TypeError, mr_job.run_mapper) self.assertRaises(TypeError, mr_job.run_combiner) self.assertRaises(TypeError, mr_job.run_reducer)
def test_spark_method(self): j = MRJob(['--no-conf']) j.spark = MagicMock() self.assertEqual(j.steps(), [SparkStep(j.spark)]) self.assertEqual(j._steps_desc(), [dict(type='spark', spark_args=[])])
def test_spark_method(self): j = MRJob(["--no-conf"]) j.spark = MagicMock() self.assertEqual(j.steps(), [SparkStep(j.spark)]) self.assertEqual(j._steps_desc(), [dict(type="spark", spark_args=[])])
def test_spark_and_spark_args_methods(self): j = MRJob(["--no-conf"]) j.spark = MagicMock() j.spark_args = MagicMock(return_value=["argh", "ARRRRGH!"]) self.assertEqual(j.steps(), [SparkStep(j.spark, spark_args=["argh", "ARRRRGH!"])]) self.assertEqual(j._steps_desc(), [dict(type="spark", spark_args=["argh", "ARRRRGH!"])])
def test_spark_and_spark_args_methods(self): j = MRJob(['--no-conf']) j.spark = MagicMock() j.spark_args = MagicMock(return_value=['argh', 'ARRRRGH!']) self.assertEqual(j.steps(), [SparkStep(j.spark, spark_args=['argh', 'ARRRRGH!'])]) self.assertEqual(j._steps_desc(), [dict(type='spark', spark_args=['argh', 'ARRRRGH!'])])
def test_spark_method(self): j = MRJob(['--no-conf']) j.spark = MagicMock() self.assertEqual( j.steps(), [SparkStep(j.spark)] ) self.assertEqual( j._steps_desc(), [dict(type='spark', jobconf={}, spark_args=[])] )
def test_spark_and_spark_args_methods(self): j = MRJob(['--no-conf']) j.spark = MagicMock() j.spark_args = MagicMock(return_value=['argh', 'ARRRRGH!']) self.assertEqual( j.steps(), [SparkStep(j.spark, spark_args=['argh', 'ARRRRGH!'])] ) self.assertEqual( j._steps_desc(), [dict(type='spark', jobconf={}, spark_args=['argh', 'ARRRRGH!'])] )
def test_too_few_args(self): job = MRJob(['--spark']) job.spark = MagicMock() self.assertRaises(ValueError, job.execute)
def test_wrong_step_num(self): job = MRJob(['--step-num=1', '--spark', 'input_dir', 'output_dir']) job.spark = MagicMock() self.assertRaises(ValueError, job.execute)
def test_spark_and_streaming_dont_mix(self): j = MRJob(['--no-conf']) j.mapper = MagicMock() j.spark = MagicMock() self.assertRaises(ValueError, j.steps)
def test_too_many_args(self): job = MRJob(['--spark', 'input_dir', 'output_dir', 'error_dir']) job.spark = MagicMock() self.assertRaises(ValueError, job.execute)
def test_too_many_args(self): job = MRJob(["--spark", "input_dir", "output_dir", "error_dir"]) job.spark = MagicMock() self.assertRaises(ValueError, job.execute)
def test_wrong_step_num(self): job = MRJob(["--step-num=1", "--spark", "input_dir", "output_dir"]) job.spark = MagicMock() self.assertRaises(ValueError, job.execute)