def test_example_config_consistency(mocker):
"""Validate example file and try to convert to CFN."""
mocker.patch("pcluster.config.param_types.get_avail_zone",
return_value="mocked_avail_zone")
pcluster_config = PclusterConfig(
config_file=utils.get_pcluster_config_example(),
fail_on_file_absence=True)
cfn_params = pcluster_config.to_cfn()
assert_that(len(cfn_params)).is_equal_to(CFN_CONFIG_NUM_OF_PARAMS)
def assert_section_params(mocker, pcluster_config_reader, settings_label,
expected_cfn_params):
mocker.patch(
"pcluster.config.cfn_param_types.get_supported_architectures_for_instance_type",
return_value=["x86_64"])
mocker.patch(
"pcluster.utils.InstanceTypeInfo.init_from_instance_type",
return_value=InstanceTypeInfo({
"InstanceType": "t2.micro",
"VCpuInfo": {
"DefaultVCpus": 1,
"DefaultCores": 1,
"DefaultThreadsPerCore": 1
},
"NetworkInfo": {
"EfaSupported": False
},
}),
)
if isinstance(expected_cfn_params, SystemExit):
with pytest.raises(SystemExit):
PclusterConfig(
cluster_label="default",
config_file=pcluster_config_reader(
settings_label=settings_label),
fail_on_file_absence=True,
fail_on_error=True,
)
else:
pcluster_config = PclusterConfig(
config_file=pcluster_config_reader(settings_label=settings_label),
fail_on_file_absence=True)
cfn_params = pcluster_config.to_cfn()
assert_that(len(cfn_params)).is_equal_to(
get_cfn_config_num_of_params(pcluster_config))
remove_ignored_params(cfn_params)
for param_key, _ in cfn_params.items():
assert_that(cfn_params.get(param_key),
description=param_key).is_equal_to(
expected_cfn_params.get(param_key))
def assert_section_params(mocker, pcluster_config_reader, settings_label, expected_cfn_params):
if isinstance(expected_cfn_params, SystemExit):
with pytest.raises(SystemExit):
PclusterConfig(
cluster_label="default",
config_file=pcluster_config_reader(settings_label=settings_label),
fail_on_file_absence=True,
fail_on_error=True,
)
else:
pcluster_config = PclusterConfig(
config_file=pcluster_config_reader(settings_label=settings_label), fail_on_file_absence=True
)
cfn_params = pcluster_config.to_cfn()
assert_that(len(cfn_params)).is_equal_to(CFN_CONFIG_NUM_OF_PARAMS)
for param_key, _ in cfn_params.items():
assert_that(cfn_params.get(param_key), description=param_key).is_equal_to(
expected_cfn_params.get(param_key)
)
def execute(args):
LOGGER.info("Retrieving configuration from CloudFormation for cluster {0}...".format(args.cluster_name))
base_config = PclusterConfig(config_file=args.config_file, cluster_name=args.cluster_name)
stack_status = base_config.cfn_stack.get("StackStatus")
if "IN_PROGRESS" in stack_status:
utils.error("Cannot execute update while stack is in {} status.".format(stack_status))
LOGGER.info("Validating configuration file {0}...".format(args.config_file if args.config_file else ""))
stack_name = utils.get_stack_name(args.cluster_name)
target_config = PclusterConfig(
config_file=args.config_file, cluster_label=args.cluster_template, fail_on_file_absence=True
)
target_config.validate()
if _check_cluster_models(base_config, target_config, args.cluster_template) and _check_changes(
args, base_config, target_config
):
# Update base config settings
base_config.update(target_config)
cfn_params = base_config.to_cfn()
cfn_client = boto3.client("cloudformation")
_restore_cfn_only_params(cfn_client, args, cfn_params, stack_name, target_config)
s3_bucket_name = cfn_params["ResourcesS3Bucket"]
tags = _get_target_config_tags_list(target_config)
artifact_directory = cfn_params["ArtifactS3RootDirectory"]
is_hit = utils.is_hit_enabled_cluster(base_config.cfn_stack)
template_url = None
if is_hit:
try:
upload_hit_resources(
s3_bucket_name, artifact_directory, target_config, target_config.to_storage().json_params, tags
)
except Exception:
utils.error("Failed when uploading resources to cluster S3 bucket {0}".format(s3_bucket_name))
template_url = evaluate_pcluster_template_url(target_config)
try:
upload_dashboard_resource(
s3_bucket_name,
artifact_directory,
target_config,
target_config.to_storage().json_params,
target_config.to_storage().cfn_params,
)
except Exception:
utils.error("Failed when uploading the dashboard resource to cluster S3 bucket {0}".format(s3_bucket_name))
_update_cluster(
args,
cfn_client,
cfn_params,
stack_name,
use_previous_template=not is_hit,
template_url=template_url,
tags=tags,
)
else:
LOGGER.info("Update aborted.")
sys.exit(1)
def create(args): # noqa: C901 FIXME!!!
LOGGER.info("Beginning cluster creation for cluster: %s",
args.cluster_name)
LOGGER.debug("Building cluster config based on args %s", str(args))
# Build the config based on args
pcluster_config = PclusterConfig(config_file=args.config_file,
cluster_label=args.cluster_template,
fail_on_file_absence=True)
pcluster_config.validate()
# get CFN parameters, template url and tags from config
cluster_section = pcluster_config.get_section("cluster")
cfn_params = pcluster_config.to_cfn()
_check_for_updates(pcluster_config)
batch_temporary_bucket = None
try:
cfn_client = boto3.client("cloudformation")
stack_name = utils.get_stack_name(args.cluster_name)
# If scheduler is awsbatch create bucket with resources
if cluster_section.get_param_value("scheduler") == "awsbatch":
batch_resources = pkg_resources.resource_filename(
__name__, "resources/batch")
batch_temporary_bucket = _create_bucket_with_batch_resources(
stack_name=stack_name,
resources_dir=batch_resources,
region=pcluster_config.region)
cfn_params["ResourcesS3Bucket"] = batch_temporary_bucket
LOGGER.info("Creating stack named: %s", stack_name)
LOGGER.debug(cfn_params)
# determine the CloudFormation Template URL to use
template_url = _evaluate_pcluster_template_url(
pcluster_config, preferred_template_url=args.template_url)
# merge tags from configuration, command-line and internal ones
tags = _evaluate_tags(pcluster_config, preferred_tags=args.tags)
# append extra parameters from command-line
if args.extra_parameters:
LOGGER.debug("Adding extra parameters to the CFN parameters")
cfn_params.update(dict(args.extra_parameters))
# prepare input parameters for stack creation and create the stack
LOGGER.debug(cfn_params)
params = [{
"ParameterKey": key,
"ParameterValue": value
} for key, value in cfn_params.items()]
stack = cfn_client.create_stack(
StackName=stack_name,
TemplateURL=template_url,
Parameters=params,
Capabilities=["CAPABILITY_IAM"],
DisableRollback=args.norollback,
Tags=tags,
)
LOGGER.debug("StackId: %s", stack.get("StackId"))
if not args.nowait:
utils.verify_stack_creation(stack_name, cfn_client)
LOGGER.info("")
result_stack = utils.get_stack(stack_name, cfn_client)
_print_stack_outputs(result_stack)
else:
stack_status = utils.get_stack(stack_name,
cfn_client).get("StackStatus")
LOGGER.info("Status: %s", stack_status)
except ClientError as e:
LOGGER.critical(e.response.get("Error").get("Message"))
sys.stdout.flush()
if batch_temporary_bucket:
utils.delete_s3_bucket(bucket_name=batch_temporary_bucket)
sys.exit(1)
except KeyboardInterrupt:
LOGGER.info("\nExiting...")
sys.exit(0)
except KeyError as e:
LOGGER.critical("ERROR: KeyError - reason:")
LOGGER.critical(e)
if batch_temporary_bucket:
utils.delete_s3_bucket(bucket_name=batch_temporary_bucket)
sys.exit(1)
except Exception as e:
LOGGER.critical(e)
if batch_temporary_bucket:
utils.delete_s3_bucket(bucket_name=batch_temporary_bucket)
sys.exit(1)
def update(args): # noqa: C901 FIXME!!!
LOGGER.info("Updating: %s", args.cluster_name)
stack_name = utils.get_stack_name(args.cluster_name)
pcluster_config = PclusterConfig(config_file=args.config_file,
cluster_label=args.cluster_template,
fail_on_file_absence=True)
pcluster_config.validate()
cfn_params = pcluster_config.to_cfn()
cluster_section = pcluster_config.get_section("cluster")
cfn = boto3.client("cloudformation")
if cluster_section.get_param_value("scheduler") != "awsbatch":
if not args.reset_desired:
asg_name = _get_asg_name(stack_name)
desired_capacity = (
boto3.client("autoscaling").describe_auto_scaling_groups(
AutoScalingGroupNames=[asg_name]).get(
"AutoScalingGroups")[0].get("DesiredCapacity"))
cfn_params["DesiredSize"] = str(desired_capacity)
else:
if args.reset_desired:
LOGGER.info(
"reset_desired flag does not work with awsbatch scheduler")
params = utils.get_stack(stack_name, cfn).get("Parameters")
for parameter in params:
if parameter.get("ParameterKey") == "ResourcesS3Bucket":
cfn_params["ResourcesS3Bucket"] = parameter.get(
"ParameterValue")
try:
LOGGER.debug(cfn_params)
if args.extra_parameters:
LOGGER.debug("Adding extra parameters to the CFN parameters")
cfn_params.update(dict(args.extra_parameters))
cfn_params = [{
"ParameterKey": key,
"ParameterValue": value
} for key, value in cfn_params.items()]
LOGGER.info("Calling update_stack")
cfn.update_stack(StackName=stack_name,
UsePreviousTemplate=True,
Parameters=cfn_params,
Capabilities=["CAPABILITY_IAM"])
stack_status = utils.get_stack(stack_name, cfn).get("StackStatus")
if not args.nowait:
while stack_status == "UPDATE_IN_PROGRESS":
stack_status = utils.get_stack(stack_name,
cfn).get("StackStatus")
events = cfn.describe_stack_events(
StackName=stack_name).get("StackEvents")[0]
resource_status = ("Status: %s - %s" %
(events.get("LogicalResourceId"),
events.get("ResourceStatus"))).ljust(80)
sys.stdout.write("\r%s" % resource_status)
sys.stdout.flush()
time.sleep(5)
else:
stack_status = utils.get_stack(stack_name, cfn).get("StackStatus")
LOGGER.info("Status: %s", stack_status)
except ClientError as e:
LOGGER.critical(e.response.get("Error").get("Message"))
sys.exit(1)
except KeyboardInterrupt:
LOGGER.info("\nExiting...")
sys.exit(0)