def test_create_spark_config_str(is_mrjob):
spark_opts = {
"spark.master": "mesos://some-host:5050",
"spark.executor.memory": "4g",
"spark.max.cores": "10",
}
output = spark_run.create_spark_config_str(spark_opts, is_mrjob)
if is_mrjob:
assert output == ("--spark-master=mesos://some-host:5050 "
"--jobconf spark.executor.memory=4g "
"--jobconf spark.max.cores=10")
else:
assert output == ("--conf spark.master=mesos://some-host:5050 "
"--conf spark.executor.memory=4g "
"--conf spark.max.cores=10")
def test_get_spark_config(
mock_find_mesos_leader,
):
mock_find_mesos_leader.return_value = 'fake_leader'
args = mock.MagicMock()
args.cluster = 'fake_cluster'
spark_conf = get_spark_config(
args=args,
container_name='fake_name',
spark_ui_port=123,
docker_img='fake-registry/fake-service',
system_paasta_config=SystemPaastaConfig(
{"cluster_fqdn_format": "paasta-{cluster:s}.something"},
'fake_dir',
),
volumes=['v1:v1:rw', 'v2:v2:rw'],
)
assert spark_conf['spark.master'] == 'mesos://fake_leader:5050'
assert 'spark.master=mesos://fake_leader:5050' in create_spark_config_str(spark_conf)
def test_find_master(self):
spark_conf = self.get_spark_config()
assert spark_conf["spark.master"] == "mesos://fake_leader:5050"
assert "spark.master=mesos://fake_leader:5050" in create_spark_config_str(
spark_conf, is_mrjob=False)