def test_set_yarn_spark_resource_config_fallback( patched_virtual_memory, patched_cpu_count, patched_yarn_config, patched_spark_config, default_bootstrapper: Bootstrapper, ) -> None: mocked_virtual_memory_total = PropertyMock(return_value=123 * 1024 * 1024) type(patched_virtual_memory.return_value ).total = mocked_virtual_memory_total patched_cpu_count.return_value = 456 default_bootstrapper.load_processing_job_config = MagicMock( return_value=None) default_bootstrapper.load_instance_type_info = MagicMock(return_value=None) default_bootstrapper.get_yarn_spark_resource_config = MagicMock( return_value=(patched_yarn_config, patched_spark_config)) default_bootstrapper.set_yarn_spark_resource_config() patched_virtual_memory.assert_called_once() mocked_virtual_memory_total.assert_called_once() patched_cpu_count.assert_called_once() default_bootstrapper.load_processing_job_config.assert_called_once() default_bootstrapper.load_instance_type_info.assert_called_once() default_bootstrapper.get_yarn_spark_resource_config.assert_called_once_with( 1, 123, 456) patched_yarn_config.write_config.assert_called_once() patched_spark_config.write_config.assert_called_once()
def test_set_yarn_spark_resource_config( patched_yarn_config, patched_spark_config, default_bootstrapper: Bootstrapper ) -> None: processing_job_config = { "ProcessingResources": {"ClusterConfig": {"InstanceType": "foo.xbar", "InstanceCount": 123}} } instance_type_info = {"foo.xbar": {"MemoryInfo": {"SizeInMiB": 456}, "VCpuInfo": {"DefaultVCpus": 789}}} default_bootstrapper.load_processing_job_config = MagicMock(return_value=processing_job_config) default_bootstrapper.load_instance_type_info = MagicMock(return_value=instance_type_info) default_bootstrapper.get_yarn_spark_resource_config = MagicMock( return_value=(patched_yarn_config, patched_spark_config) ) default_bootstrapper.set_yarn_spark_resource_config() default_bootstrapper.load_processing_job_config.assert_called_once() default_bootstrapper.load_instance_type_info.assert_called_once() default_bootstrapper.get_yarn_spark_resource_config.assert_called_once_with(123, 456, 789) patched_yarn_config.write_config.assert_called_once() patched_spark_config.write_config.assert_called_once()
def test_get_yarn_spark_resource_config( default_bootstrapper: Bootstrapper) -> None: # Using a cluster with one single m5.xlarge instance, calculate Yarn and Spark configs, and double check the math instance_mem_mb = 16384 instance_cores = 4 yarn_config, spark_config = default_bootstrapper.get_yarn_spark_resource_config( 1, instance_mem_mb, instance_cores) exp_yarn_max_mem_mb = 15892 # = int(instance_mem_mb * .97) = int(16384 * .97) = int(15892.48) exp_yarn_config_props = { "yarn.scheduler.minimum-allocation-mb": "1", "yarn.scheduler.maximum-allocation-mb": str(exp_yarn_max_mem_mb), "yarn.scheduler.minimum-allocation-vcores": "1", "yarn.scheduler.maximum-allocation-vcores": str(instance_cores), "yarn.nodemanager.resource.memory-mb": str(exp_yarn_max_mem_mb), "yarn.nodemanager.resource.cpu-vcores": str(instance_cores), } assert yarn_config.Classification == "yarn-site" assert yarn_config.Properties == exp_yarn_config_props exp_executor_cores = 4 # = instance_cores = 4 exp_executor_count_total = 1 # = instance_count * executor_count_per_instance = 1 * 1 exp_default_parallelism = 8 # = instance_count * instance_cores * 2 = 1 * 4 * 2 exp_driver_mem_mb = 2048 # = 2 * 1024 exp_driver_mem_ovr_mb = 204 # = int(driver_mem_mb * driver_mem_ovr_pct) = int(2048 * 0.1) = int(204.8) # = int((instance_mem_mb - driver_mem_mb - driver_mem_ovr_mb) / # (executor_count_per_instance + executor_count_per_instance * executor_mem_ovr_pct)) # = int((15892 - 2048 - 204) / (1 + 1 * 0.1)) # = int(13640 / 1.1) exp_executor_mem_mb = 12399 exp_executor_mem_ovr_mb = 1239 # = int(executor_mem_mb * executor_mem_ovr_pct) = int(12399 * 0.1) = int(1239.9) exp_driver_gc_config = ( "-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=70 -XX:MaxHeapFreeRatio=70 " "-XX:+CMSClassUnloadingEnabled") exp_driver_java_opts = "-XX:OnOutOfMemoryError='kill -9 %p' " f"{exp_driver_gc_config}" # ConcGCThreads = max(int(executor_cores / 4), 1) = max(int(4 / 4), 1) = max(1, 1) = 1 # ParallelGCThreads = max(int(3 * executor_cores / 4), 1) = max(int(3 * 4 / 4), 1) = max(3, 1) = 3 exp_executor_gc_config = ( "-XX:+UseParallelGC -XX:InitiatingHeapOccupancyPercent=70 " "-XX:ConcGCThreads=1 " "-XX:ParallelGCThreads=3 ") exp_executor_java_opts = ( "-verbose:gc -XX:OnOutOfMemoryError='kill -9 %p' " "-XX:+PrintGCDetails -XX:+PrintGCDateStamps " f"{exp_executor_gc_config}") exp_spark_config_props = { "spark.driver.memory": f"{exp_driver_mem_mb}m", "spark.driver.memoryOverhead": f"{exp_driver_mem_ovr_mb}m", "spark.driver.defaultJavaOptions": f"{exp_driver_java_opts}", "spark.executor.memory": f"{exp_executor_mem_mb}m", "spark.executor.memoryOverhead": f"{exp_executor_mem_ovr_mb}m", "spark.executor.cores": f"{exp_executor_cores}", "spark.executor.defaultJavaOptions": f"{exp_executor_java_opts}", "spark.executor.instances": f"{exp_executor_count_total}", "spark.default.parallelism": f"{exp_default_parallelism}", } assert spark_config.Classification == "spark-defaults" assert spark_config.Properties == exp_spark_config_props # Using the same instance type, increase the instance count by 10x yarn_config, spark_config = default_bootstrapper.get_yarn_spark_resource_config( 10, instance_mem_mb, instance_cores) # Yarn config should be the same assert yarn_config.Properties == exp_yarn_config_props # Spark config should be the same with more 10x executors and parallelism exp_spark_config_props[ "spark.executor.instances"] = f"{exp_executor_count_total * 10}" exp_spark_config_props[ "spark.default.parallelism"] = f"{exp_default_parallelism * 10}" assert spark_config.Properties == exp_spark_config_props