def test_emit_resource_requirements(tmpdir): spark_config_dict = { 'spark.executor.cores': '2', 'spark.cores.max': '4', 'spark.executor.memory': '4g', 'spark.mesos.executor.memoryOverhead': '555', 'spark.app.name': 'paasta_spark_run_johndoe_2_3', 'spark.mesos.constraints': 'pool:cool-pool\\;other:value', } clusterman_yaml_contents = { 'mesos_clusters': { 'anywhere-prod': { 'aws_region': 'us-north-14', }, }, } clusterman_yaml_file_path = tmpdir.join('fake_clusterman.yaml') with open(clusterman_yaml_file_path, 'w') as f: YAML().dump(clusterman_yaml_contents, f) with mock.patch( 'paasta_tools.cli.cmds.spark_run.get_clusterman_metrics', autospec=True, ), mock.patch( 'paasta_tools.cli.cmds.spark_run.clusterman_metrics', autospec=True, ) as mock_clusterman_metrics, mock.patch( 'paasta_tools.cli.cmds.spark_run.CLUSTERMAN_YAML_FILE_PATH', clusterman_yaml_file_path, autospec=None, # we're replacing this name, so we can't autospec ), mock.patch( 'time.time', return_value=1234, autospec=True, ): mock_clusterman_metrics.generate_key_with_dimensions.side_effect = lambda name, dims: ( f'{name}|framework_name={dims["framework_name"]},webui_url={dims["webui_url"]}' ) emit_resource_requirements(spark_config_dict, 'anywhere-prod', 'http://spark.yelp') mock_clusterman_metrics.ClustermanMetricsBotoClient.assert_called_once_with( region_name='us-north-14', app_identifier='cool-pool', ) metrics_writer = mock_clusterman_metrics.ClustermanMetricsBotoClient.return_value.\ get_writer.return_value.__enter__.return_value metric_key_template = ( 'requested_{resource}|framework_name=paasta_spark_run_johndoe_2_3,webui_url=http://spark.yelp' ) expected_memory_request = (4 * 1024 + 555) * 2 metrics_writer.send.assert_has_calls( [ mock.call((metric_key_template.format(resource='cpus'), 1234, 4)), mock.call((metric_key_template.format(resource='mem'), 1234, expected_memory_request)), mock.call((metric_key_template.format(resource='disk'), 1234, expected_memory_request)), ], any_order=True, )
def test_emit_resource_requirements(tmpdir): spark_config_dict = { "spark.executor.cores": "2", "spark.cores.max": "4", "spark.executor.memory": "4g", "spark.mesos.executor.memoryOverhead": "555", "spark.app.name": "paasta_spark_run_johndoe_2_3", "spark.mesos.constraints": "pool:cool-pool\\;other:value", } clusterman_yaml_contents = { "clusters": { "anywhere-prod": { "aws_region": "us-north-14" } } } clusterman_yaml_file_path = tmpdir.join("fake_clusterman.yaml") with open(clusterman_yaml_file_path, "w") as f: YAML().dump(clusterman_yaml_contents, f) with mock.patch( "paasta_tools.cli.cmds.spark_run.get_clusterman_metrics", autospec=True ), mock.patch( "paasta_tools.cli.cmds.spark_run.clusterman_metrics", autospec=True ) as mock_clusterman_metrics, mock.patch( "paasta_tools.cli.cmds.spark_run.CLUSTERMAN_YAML_FILE_PATH", clusterman_yaml_file_path, autospec=None, # we're replacing this name, so we can't autospec ), mock.patch("time.time", return_value=1234, autospec=True), mock.patch( "paasta_tools.cli.cmds.spark_run.get_spark_resource_requirements", autospec=True, ) as mock_spark_requirements: metric_key_template = "requested_{resource}|framework_name=paasta_spark_run_johndoe_2_3,webui_url=http://spark.yelp" expected_memory_request = (4 * 1024 + 555) * 2 mock_spark_requirements.return_value = { "cpus": (metric_key_template.format(resource="cpus"), 4), "mem": ( metric_key_template.format(resource="mem"), expected_memory_request, ), "disk": ( metric_key_template.format(resource="disk"), expected_memory_request, ), } emit_resource_requirements(spark_config_dict, "anywhere-prod", "http://spark.yelp") mock_clusterman_metrics.ClustermanMetricsBotoClient.assert_called_once_with( region_name="us-north-14", app_identifier="cool-pool") metrics_writer = ( mock_clusterman_metrics.ClustermanMetricsBotoClient.return_value. get_writer.return_value.__enter__.return_value) metrics_writer.send.assert_has_calls( [ mock.call( (metric_key_template.format(resource="cpus"), 1234, 4)), mock.call(( metric_key_template.format(resource="mem"), 1234, expected_memory_request, )), mock.call(( metric_key_template.format(resource="disk"), 1234, expected_memory_request, )), ], any_order=True, )