def test_cw_dashboard_builder(mocker, test_datadir, config_file_name):
mock_aws_api(mocker)
mocker.patch(
"pcluster.config.cluster_config.HeadNodeNetworking.availability_zone",
new_callable=PropertyMock(return_value="us-east-1a"),
)
# mock bucket initialization parameters
mock_bucket(mocker)
input_yaml = load_yaml_dict(test_datadir / config_file_name)
cluster_config = ClusterSchema(cluster_name="clustername").load(input_yaml)
print(cluster_config)
generated_template = CDKTemplateBuilder().build_cluster_template(
cluster_config=cluster_config,
bucket=dummy_cluster_bucket(),
stack_name="clustername")
output_yaml = yaml.dump(generated_template, width=float("inf"))
print(output_yaml)
if cluster_config.is_cw_dashboard_enabled:
if cluster_config.shared_storage:
_verify_ec2_metrics_conditions(cluster_config, output_yaml)
if cluster_config.is_cw_logging_enabled:
_verify_head_node_logs_conditions(cluster_config, output_yaml)
else:
assert_that(output_yaml).does_not_contain("Head Node Logs")
def test_imagebuilder_schema(mocker, test_datadir, config_file_name, response):
mock_aws_api(mocker)
mocker.patch("pcluster.imagebuilder_utils.get_ami_id",
return_value="ami-0185634c5a8a37250")
mocker.patch(
"pcluster.aws.ec2.Ec2Client.describe_image",
return_value=response,
)
# Load imagebuilder model from Yaml file
input_yaml = load_yaml_dict(test_datadir / config_file_name)
print(input_yaml)
imagebuilder_config = ImageBuilderSchema().load(input_yaml)
print(imagebuilder_config)
# Re-create Yaml file from model and compare content
image_builder_schema = ImageBuilderSchema()
image_builder_schema.context = {"delete_defaults_when_dump": True}
output_json = image_builder_schema.dump(imagebuilder_config)
# Assert imagebuilder config file can be convert to imagebuilder config
assert_that(json.dumps(input_yaml, sort_keys=True)).is_equal_to(
json.dumps(output_json, sort_keys=True))
# Print output yaml
output_yaml = yaml.dump(output_json)
print(output_yaml)
def _load_cluster_model_from_yaml(test_datadir, config_file_name):
# https://github.com/marshmallow-code/marshmallow/issues/1126
# TODO use yaml render_module: https://marshmallow.readthedocs.io/en/3.0/api_reference.html#marshmallow.Schema.Meta
input_yaml = load_yaml_dict(test_datadir / config_file_name)
print(input_yaml)
copy_input_yaml = deepcopy(input_yaml)
cluster = ClusterSchema(cluster_name="clustername").load(copy_input_yaml)
print(cluster)
return input_yaml, cluster
def load_cluster_model_from_yaml(config_file_name, test_datadir=None):
if test_datadir:
path = test_datadir / config_file_name
else:
# If test_datadir is not specified, find configs in example_configs directory
path = f"{os.path.dirname(__file__)}/example_configs/{config_file_name}"
input_yaml = load_yaml_dict(path)
print(input_yaml)
copy_input_yaml = deepcopy(input_yaml)
cluster = ClusterSchema(cluster_name="clustername").load(copy_input_yaml)
print(cluster)
return input_yaml, cluster
def build_imagebuilder_template(image_config: ImageBuilderConfig,
image_id: str, bucket: S3Bucket):
"""Build template for the given imagebuilder and return as output in Yaml format."""
from aws_cdk.core import App # pylint: disable=C0415
from pcluster.templates.imagebuilder_stack import ImageBuilderCdkStack # pylint: disable=C0415
with tempfile.TemporaryDirectory() as tempdir:
output_file = "imagebuilder"
app = App(outdir=str(tempdir))
ImageBuilderCdkStack(app, output_file, image_config, image_id,
bucket)
app.synth()
generated_template = load_yaml_dict(
os.path.join(tempdir, f"{output_file}.template.json"))
return generated_template
def test_head_node_dna_json(mocker, test_datadir, config_file_name, expected_head_node_dna_json_file_name):
mock_aws_api(mocker)
input_yaml = load_yaml_dict(test_datadir / config_file_name)
cluster_config = ClusterSchema(cluster_name="clustername").load(input_yaml)
generated_template = CDKTemplateBuilder().build_cluster_template(
cluster_config=cluster_config, bucket=dummy_cluster_bucket(), stack_name="clustername"
)
generated_head_node_dna_json = json.loads(
_get_cfn_init_file_content(template=generated_template, resource="HeadNodeLaunchTemplate", file="/tmp/dna.json")
)
expected_head_node_dna_json = load_json_dict(test_datadir / expected_head_node_dna_json_file_name)
assert_that(generated_head_node_dna_json).is_equal_to(expected_head_node_dna_json)
def build_cluster_template(cluster_config: BaseClusterConfig,
bucket: S3Bucket,
stack_name: str,
log_group_name: str = None):
"""Build template for the given cluster and return as output in Yaml format."""
from aws_cdk.core import App # pylint: disable=C0415
from pcluster.templates.cluster_stack import ClusterCdkStack # pylint: disable=C0415
with tempfile.TemporaryDirectory() as tempdir:
output_file = str(stack_name)
app = App(outdir=str(tempdir))
ClusterCdkStack(app, output_file, stack_name, cluster_config,
bucket, log_group_name)
app.synth()
generated_template = load_yaml_dict(
os.path.join(tempdir, f"{output_file}.template.json"))
return generated_template
def _load_yaml(source_dir, file_name):
"""Get string data from yaml file."""
return yaml.dump(utils.load_yaml_dict(os.path.join(source_dir, file_name)))
def _load_config(config_file):
return ClusterSchema(cluster_name="clustername").load(
load_yaml_dict(config_file))
def _load_and_validate(config_path):
input_yaml = load_yaml_dict(config_path)
cluster = ClusterSchema(cluster_name="clustername").load(input_yaml)
failures = cluster.validate()
assert_that(failures).is_empty()
