def test_create_spark_config_str(is_mrjob): spark_opts = { "spark.master": "mesos://some-host:5050", "spark.executor.memory": "4g", "spark.max.cores": "10", } output = spark_run.create_spark_config_str(spark_opts, is_mrjob) if is_mrjob: assert output == ("--spark-master=mesos://some-host:5050 " "--jobconf spark.executor.memory=4g " "--jobconf spark.max.cores=10") else: assert output == ("--conf spark.master=mesos://some-host:5050 " "--conf spark.executor.memory=4g " "--conf spark.max.cores=10")
def test_get_spark_config( mock_find_mesos_leader, ): mock_find_mesos_leader.return_value = 'fake_leader' args = mock.MagicMock() args.cluster = 'fake_cluster' spark_conf = get_spark_config( args=args, container_name='fake_name', spark_ui_port=123, docker_img='fake-registry/fake-service', system_paasta_config=SystemPaastaConfig( {"cluster_fqdn_format": "paasta-{cluster:s}.something"}, 'fake_dir', ), volumes=['v1:v1:rw', 'v2:v2:rw'], ) assert spark_conf['spark.master'] == 'mesos://fake_leader:5050' assert 'spark.master=mesos://fake_leader:5050' in create_spark_config_str(spark_conf)
def test_find_master(self): spark_conf = self.get_spark_config() assert spark_conf["spark.master"] == "mesos://fake_leader:5050" assert "spark.master=mesos://fake_leader:5050" in create_spark_config_str( spark_conf, is_mrjob=False)