def test_emit_resource_requirements(tmpdir):
spark_config_dict = {
'spark.executor.cores': '2',
'spark.cores.max': '4',
'spark.executor.memory': '4g',
'spark.mesos.executor.memoryOverhead': '555',
'spark.app.name': 'paasta_spark_run_johndoe_2_3',
'spark.mesos.constraints': 'pool:cool-pool\\;other:value',
}
clusterman_yaml_contents = {
'mesos_clusters': {
'anywhere-prod': {
'aws_region': 'us-north-14',
},
},
}
clusterman_yaml_file_path = tmpdir.join('fake_clusterman.yaml')
with open(clusterman_yaml_file_path, 'w') as f:
YAML().dump(clusterman_yaml_contents, f)
with mock.patch(
'paasta_tools.cli.cmds.spark_run.get_clusterman_metrics', autospec=True,
), mock.patch(
'paasta_tools.cli.cmds.spark_run.clusterman_metrics', autospec=True,
) as mock_clusterman_metrics, mock.patch(
'paasta_tools.cli.cmds.spark_run.CLUSTERMAN_YAML_FILE_PATH',
clusterman_yaml_file_path,
autospec=None, # we're replacing this name, so we can't autospec
), mock.patch(
'time.time', return_value=1234, autospec=True,
):
mock_clusterman_metrics.generate_key_with_dimensions.side_effect = lambda name, dims: (
f'{name}|framework_name={dims["framework_name"]},webui_url={dims["webui_url"]}'
)
emit_resource_requirements(spark_config_dict, 'anywhere-prod', 'http://spark.yelp')
mock_clusterman_metrics.ClustermanMetricsBotoClient.assert_called_once_with(
region_name='us-north-14',
app_identifier='cool-pool',
)
metrics_writer = mock_clusterman_metrics.ClustermanMetricsBotoClient.return_value.\
get_writer.return_value.__enter__.return_value
metric_key_template = (
'requested_{resource}|framework_name=paasta_spark_run_johndoe_2_3,webui_url=http://spark.yelp'
)
expected_memory_request = (4 * 1024 + 555) * 2
metrics_writer.send.assert_has_calls(
[
mock.call((metric_key_template.format(resource='cpus'), 1234, 4)),
mock.call((metric_key_template.format(resource='mem'), 1234, expected_memory_request)),
mock.call((metric_key_template.format(resource='disk'), 1234, expected_memory_request)),
],
any_order=True,
)
def test_emit_resource_requirements(tmpdir):
spark_config_dict = {
"spark.executor.cores": "2",
"spark.cores.max": "4",
"spark.executor.memory": "4g",
"spark.mesos.executor.memoryOverhead": "555",
"spark.app.name": "paasta_spark_run_johndoe_2_3",
"spark.mesos.constraints": "pool:cool-pool\\;other:value",
}
clusterman_yaml_contents = {
"clusters": {
"anywhere-prod": {
"aws_region": "us-north-14"
}
}
}
clusterman_yaml_file_path = tmpdir.join("fake_clusterman.yaml")
with open(clusterman_yaml_file_path, "w") as f:
YAML().dump(clusterman_yaml_contents, f)
with mock.patch(
"paasta_tools.cli.cmds.spark_run.get_clusterman_metrics",
autospec=True
), mock.patch(
"paasta_tools.cli.cmds.spark_run.clusterman_metrics", autospec=True
) as mock_clusterman_metrics, mock.patch(
"paasta_tools.cli.cmds.spark_run.CLUSTERMAN_YAML_FILE_PATH",
clusterman_yaml_file_path,
autospec=None, # we're replacing this name, so we can't autospec
), mock.patch("time.time", return_value=1234, autospec=True), mock.patch(
"paasta_tools.cli.cmds.spark_run.get_spark_resource_requirements",
autospec=True,
) as mock_spark_requirements:
metric_key_template = "requested_{resource}|framework_name=paasta_spark_run_johndoe_2_3,webui_url=http://spark.yelp"
expected_memory_request = (4 * 1024 + 555) * 2
mock_spark_requirements.return_value = {
"cpus": (metric_key_template.format(resource="cpus"), 4),
"mem": (
metric_key_template.format(resource="mem"),
expected_memory_request,
),
"disk": (
metric_key_template.format(resource="disk"),
expected_memory_request,
),
}
emit_resource_requirements(spark_config_dict, "anywhere-prod",
"http://spark.yelp")
mock_clusterman_metrics.ClustermanMetricsBotoClient.assert_called_once_with(
region_name="us-north-14", app_identifier="cool-pool")
metrics_writer = (
mock_clusterman_metrics.ClustermanMetricsBotoClient.return_value.
get_writer.return_value.__enter__.return_value)
metrics_writer.send.assert_has_calls(
[
mock.call(
(metric_key_template.format(resource="cpus"), 1234, 4)),
mock.call((
metric_key_template.format(resource="mem"),
1234,
expected_memory_request,
)),
mock.call((
metric_key_template.format(resource="disk"),
1234,
expected_memory_request,
)),
],
any_order=True,
)
