def test_parse_spark_conf_bad(self):
dir_path = os.path.dirname(os.path.realpath(__file__))
try:
_parse_spark_conf(os.path.join(dir_path, 'spark-fail.conf'))
except RuntimeError as e:
self.assertTrue("may be missing an '='" in str(e))
else:
self.fail("Expected spark conf parser to fail on bad conf")
def test_parse_spark_conf_good(self):
dir_path = os.path.dirname(os.path.realpath(__file__))
conf = _parse_spark_conf(os.path.join(dir_path, 'spark-pass.conf'))
self.assertEqual(16, len(conf))
self.assertEqual('false', conf['spark.shuffle.io.preferDirectBufs'])
self.assertEqual('2', conf['spark.executor.cores'])
self.assertEqual('-Xmx1536m', conf['spark.executor.extrajavaoptions'])
self.assertEqual('2g', conf['spark.driver.maxResultSize'])
self.assertEqual('384', conf['spark.yarn.executor.memoryOverhead'])
