def test_spark_property():
c = sparklauncher.SparkConfiguration()
c.conf.spark.executor.cores = 5
assert c.conf._conf_dict.get('spark.executor.cores') == 5
c.conf.set_if_unset('spark.executor.cores', 10)
assert c.conf._conf_dict.get('spark.executor.cores') == 5
def test_spark_launcher_multiple_argument():
c = sparklauncher.SparkConfiguration()
archive = ["/path/to/some/archive.zip", "/path/to/someother/archive.zip"]
c.archives = archive
c._set_environment_variables()
assert ('--archives ' +
','.join(archive)) in os.environ['PYSPARK_SUBMIT_ARGS']
def test_prepare_interactive():
c = sparklauncher.SparkConfiguration()
new_conf = prepare_pyspark_yarn_interactive("conda",
"hdfs://some/env/conda.zip", c)
# Must be a new instance not a copy.
assert new_conf is not c
expected_python = os.path.join(".", "CONDA", "conda", "bin", "python")
assert new_conf._python_path == expected_python
# archive must be added tp the arguments that will be supplied.
assert "hdfs://some/env/conda.zip#CONDA" in new_conf.archives
assert os.environ["PYSPARK_PYTHON"] == expected_python
def test_sparkconf_hasattr():
c = sparklauncher.SparkConfiguration()
assert hasattr(c, "foo") is False
assert hasattr(c, "driver_memory") is True
assert hasattr(c, "driver-memory") is True
assert c.driver_memory is None
c.driver_memory = "4g"
assert c.driver_memory == "4g"
with pytest.raises(AttributeError):
# make sure that attempting to access unknown variables raises an attribute error
getattr(c, 'foo')
with pytest.raises(AttributeError):
# make sure that attempting to set unknown variables raises an attribute error
c.foo = "bar"
def spark_tuple_module(request):
conf = sparklauncher.SparkConfiguration()
conf.master = "local[1]"
sc, sqlContext = conf.sql_context("test")
request.addfinalizer(sc.stop)
return sc, sqlContext
def test_set_spark_property():
c = sparklauncher.SparkConfiguration()
c.driver_memory = "4g"
def test_config_priority():
c = sparklauncher.SparkConfiguration()
c.driver_memory = "4g"
c.conf.spark.driver.memory = "5g"
c._set_environment_variables()
assert '--driver-memory 5g' in os.environ['PYSPARK_SUBMIT_ARGS']
def test_spark_launcher_argument():
c = sparklauncher.SparkConfiguration()
archive = "/path/to/some/archive.zip"
c.archives = archive
c._set_environment_variables()
assert ('--archives ' + archive) in os.environ['PYSPARK_SUBMIT_ARGS']
