def _test_spark_executor_memory(self, conf_value, megs): runner = LocalMRJobRunner( jobconf={'spark.executor.memory': conf_value}) self.assertEqual(runner._spark_master(), 'local-cluster[%d,1,%d]' % ( cpu_count(), megs))
def test_num_cores(self): runner = LocalMRJobRunner(num_cores=3) self.assertEqual(runner._spark_master(), 'local-cluster[3,1,1024]')
def test_default_spark_master(self): runner = LocalMRJobRunner() self.assertEqual(runner._spark_master(), 'local-cluster[%d,1,1024]' % cpu_count())