def test_spark_property(): c = sparklauncher.SparkConfiguration() c.conf.spark.executor.cores = 5 assert c.conf._conf_dict.get('spark.executor.cores') == 5 c.conf.set_if_unset('spark.executor.cores', 10) assert c.conf._conf_dict.get('spark.executor.cores') == 5
def test_spark_launcher_multiple_argument(): c = sparklauncher.SparkConfiguration() archive = ["/path/to/some/archive.zip", "/path/to/someother/archive.zip"] c.archives = archive c._set_environment_variables() assert ('--archives ' + ','.join(archive)) in os.environ['PYSPARK_SUBMIT_ARGS']
def test_prepare_interactive(): c = sparklauncher.SparkConfiguration() new_conf = prepare_pyspark_yarn_interactive("conda", "hdfs://some/env/conda.zip", c) # Must be a new instance not a copy. assert new_conf is not c expected_python = os.path.join(".", "CONDA", "conda", "bin", "python") assert new_conf._python_path == expected_python # archive must be added tp the arguments that will be supplied. assert "hdfs://some/env/conda.zip#CONDA" in new_conf.archives assert os.environ["PYSPARK_PYTHON"] == expected_python
def test_sparkconf_hasattr(): c = sparklauncher.SparkConfiguration() assert hasattr(c, "foo") is False assert hasattr(c, "driver_memory") is True assert hasattr(c, "driver-memory") is True assert c.driver_memory is None c.driver_memory = "4g" assert c.driver_memory == "4g" with pytest.raises(AttributeError): # make sure that attempting to access unknown variables raises an attribute error getattr(c, 'foo') with pytest.raises(AttributeError): # make sure that attempting to set unknown variables raises an attribute error c.foo = "bar"
def spark_tuple_module(request): conf = sparklauncher.SparkConfiguration() conf.master = "local[1]" sc, sqlContext = conf.sql_context("test") request.addfinalizer(sc.stop) return sc, sqlContext
def test_set_spark_property(): c = sparklauncher.SparkConfiguration() c.driver_memory = "4g"
def test_config_priority(): c = sparklauncher.SparkConfiguration() c.driver_memory = "4g" c.conf.spark.driver.memory = "5g" c._set_environment_variables() assert '--driver-memory 5g' in os.environ['PYSPARK_SUBMIT_ARGS']
def test_spark_launcher_argument(): c = sparklauncher.SparkConfiguration() archive = "/path/to/some/archive.zip" c.archives = archive c._set_environment_variables() assert ('--archives ' + archive) in os.environ['PYSPARK_SUBMIT_ARGS']