def test_parse_spark_conf_bad(self): dir_path = os.path.dirname(os.path.realpath(__file__)) try: _parse_spark_conf(os.path.join(dir_path, 'spark-fail.conf')) except RuntimeError as e: self.assertTrue("may be missing an '='" in str(e)) else: self.fail("Expected spark conf parser to fail on bad conf")
def test_parse_spark_conf_good(self): dir_path = os.path.dirname(os.path.realpath(__file__)) conf = _parse_spark_conf(os.path.join(dir_path, 'spark-pass.conf')) self.assertEqual(16, len(conf)) self.assertEqual('false', conf['spark.shuffle.io.preferDirectBufs']) self.assertEqual('2', conf['spark.executor.cores']) self.assertEqual('-Xmx1536m', conf['spark.executor.extrajavaoptions']) self.assertEqual('2g', conf['spark.driver.maxResultSize']) self.assertEqual('384', conf['spark.yarn.executor.memoryOverhead'])