def bucket(self):
"""Return a bucket configuration."""
if self.__bucket:
return self.__bucket
if self.__source_config_text:
# get custom_s3_bucket in create command
custom_bucket_name = self.config.custom_s3_bucket
else:
# get custom_s3_bucket in delete, update commands
custom_bucket_name = self.stack.s3_bucket_name
if custom_bucket_name == S3Bucket.get_bucket_name(
AWSApi.instance().sts.get_account_id(), get_region()):
custom_bucket_name = None
try:
self.__bucket = S3BucketFactory.init_s3_bucket(
service_name=self.name,
stack_name=self.stack_name,
custom_s3_bucket=custom_bucket_name,
artifact_directory=self.s3_artifacts_dir,
)
except AWSClientError as e:
raise _cluster_error_mapper(
e, f"Unable to initialize s3 bucket. {e}")
return self.__bucket
def get_templates_bucket_path():
"""Return a string containing the path of bucket."""
region = get_region()
s3_suffix = ".cn" if region.startswith("cn") else ""
return (
f"https://{region}-aws-parallelcluster.s3.{region}.amazonaws.com{s3_suffix}/"
f"parallelcluster/{get_installed_version()}/templates/")
def _validate(self, auto_import_policy, import_path):
if auto_import_policy is not None:
bucket = get_bucket_name_from_s3_url(import_path)
if AWSApi.instance().s3.get_bucket_region(bucket) != get_region():
self._add_failure(
"FSx auto import is not supported for cross-region buckets.",
FailureLevel.ERROR)
def __init__(self,
resource_id,
log_group_name,
bucket,
output_dir,
bucket_prefix=None,
keep_s3_objects=False):
# check bucket
bucket_region = AWSApi.instance().s3.get_bucket_region(
bucket_name=bucket)
if bucket_region != get_region():
raise LogsExporterError(
f"The bucket used for exporting logs must be in the same region as the {resource_id}. "
f"The given resource is in {get_region()}, but the bucket's region is {bucket_region}."
)
self.bucket = bucket
self.log_group_name = log_group_name
self.output_dir = output_dir
self.keep_s3_objects = keep_s3_objects
if bucket_prefix:
self.bucket_prefix = bucket_prefix
self.delete_everything_under_prefix = False
else:
# If the default bucket prefix is being used and there's nothing underneath that prefix already
# then we can delete everything under that prefix after downloading the data
# (unless keep-s3-objects is specified)
self.bucket_prefix = f"{resource_id}-logs-{datetime.datetime.now().strftime('%Y%m%d%H%M')}"
self.delete_everything_under_prefix = AWSApi.instance(
).s3_resource.is_empty(bucket, self.bucket_prefix)
def _get_log_group_arn(self):
log_group_arn = self.format_arn(
service="logs",
resource="log-group",
region=get_region(),
sep=":",
resource_name=f"/aws/imagebuilder/{self._build_image_recipe_name()}",
)
return log_group_arn
def __init__(self, stack: ClusterStack):
# Cluster info
self.name = stack.cluster_name
self.region = get_region()
self.version = stack.version
self.scheduler = stack.scheduler
self.status = stack.status # FIXME cluster status should be different from stack status
# Stack info
self.stack_arn = stack.id
self.stack_name = stack.name
self.stack_status = stack.status
self.stack_outputs = stack.outputs
def describe_image_by_imagebuilder_arn_tag(self, image_id: str):
"""Return a dict of image info by searching imagebuilder arn tag."""
partition = get_partition()
region = get_region()
name = "{0}-{1}".format(IMAGEBUILDER_RESOURCE_NAME_PREFIX, image_id)[0:1024].lower()
filters = [
{
"Name": "tag:" + IMAGEBUILDER_ARN_TAG,
"Values": [f"arn:{partition}:imagebuilder:{region}:*:image/{name}*"],
}
]
owners = ["self"]
return self.describe_images(ami_ids=[], filters=filters, owners=owners)[0]
def _get_keys():
"""Return a list of keys."""
keypairs = boto3.client("ec2").describe_key_pairs()
key_options = []
for key in keypairs.get("KeyPairs"):
key_name = key.get("KeyName")
key_options.append(key_name)
if not key_options:
print(
"No KeyPair found in region {0}, please create one following the guide: "
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html"
.format(get_region()))
return key_options
def _ssh(args, extra_args):
# pylint: disable=import-outside-toplevel
"""
Execute an SSH command to the head node instance, according to the [aliases] section if there.
:param args: pcluster CLI args
:param extra_args: pcluster CLI extra_args
"""
try:
try:
from shlex import quote as cmd_quote
except ImportError:
from pipes import quote as cmd_quote
result = PclusterApi().describe_cluster_instances(
cluster_name=args.cluster_name,
region=get_region(),
node_type=NodeType.HEAD_NODE)
if isinstance(result, list) and len(result) == 1:
# build command
cmd = "ssh {CFN_USER}@{HEAD_NODE_IP} {ARGS}".format(
CFN_USER=result[0].user,
HEAD_NODE_IP=result[0].public_ip_address
or result[0].private_ip_address,
ARGS=" ".join(cmd_quote(str(arg)) for arg in extra_args),
)
# run command
log_message = "SSH command: {0}".format(cmd)
if not args.dryrun:
LOGGER.debug(log_message)
# A nosec comment is appended to the following line in order to disable the B605 check.
# This check is disabled for the following reasons:
# - The args passed to the remote command are sanitized.
# - The default command to which these args is known.
# - Users have full control over any customization of the command to which args are passed.
os.system(cmd) # nosec nosemgrep
else:
print(log_message)
else:
utils.error(
f"Unable to connect to the cluster {args.cluster_name}.\n{result.message}"
)
except KeyboardInterrupt:
print("\nExiting...")
sys.exit(0)
def _dcv_connect(args):
"""
Execute pcluster dcv connect command.
:param args: pcluster cli arguments.
"""
result = PclusterApi().describe_cluster_instances(
cluster_name=args.cluster_name,
region=get_region(),
node_type=NodeType.HEAD_NODE)
if isinstance(result, list) and len(result) == 1:
head_node_ip = result[0].public_ip_address or result[
0].private_ip_address
# Prepare ssh command to execute in the head node instance
cmd = 'ssh {CFN_USER}@{HEAD_NODE_IP} {KEY} "{REMOTE_COMMAND} /home/{CFN_USER}"'.format(
CFN_USER=result[0].user,
HEAD_NODE_IP=head_node_ip,
KEY="-i {0}".format(args.key_path) if args.key_path else "",
REMOTE_COMMAND=DCV_CONNECT_SCRIPT,
)
try:
url = _retry(_retrieve_dcv_session_url,
func_args=[cmd, args.cluster_name, head_node_ip],
attempts=4)
url_message = "Please use the following one-time URL in your browser within 30 seconds:\n{0}".format(
url)
if args.show_url:
print(url_message)
return
try:
if not webbrowser.open_new(url):
raise webbrowser.Error("Unable to open the Web browser.")
except webbrowser.Error as e:
print("%s\n%s", e, url_message)
except DCVConnectionError as e:
error(
"Something went wrong during DCV connection.\n{0}"
"Please check the logs in the /var/log/parallelcluster/ folder "
"of the head node and submit an issue {1}\n".format(
e, PCLUSTER_ISSUES_LINK))
else:
error(f"Unable to connect to the cluster.\n{result.message}")
def automate_vpc_with_subnet_creation(network_configuration,
compute_subnet_size):
print(
"Beginning VPC creation. Please do not leave the terminal until the creation is finalized"
)
vpc_creator = VpcFactory(get_region())
vpc_id = vpc_creator.create()
vpc_creator.setup(vpc_id, name="ParallelClusterVPC" + TIMESTAMP)
if not vpc_creator.check(vpc_id):
logging.critical(
"Something went wrong in VPC creation. Please delete it and start the process again"
)
sys.exit(1)
vpc_parameters = {"vpc_id": vpc_id}
vpc_parameters.update(
automate_subnet_creation(vpc_id, network_configuration,
compute_subnet_size))
return vpc_parameters
def _register_validators(self):
# Volume size validator only validates specified volume size
if self.image and self.image.root_volume and self.image.root_volume.size:
self._register_validator(
EbsVolumeTypeSizeValidator,
volume_type=ROOT_VOLUME_TYPE,
volume_size=self.image.root_volume.size,
)
self._register_validator(
AMIVolumeSizeValidator,
volume_size=self.image.root_volume.size,
image=self.build.parent_image,
)
if self.custom_s3_bucket:
self._register_validator(S3BucketValidator,
bucket=self.custom_s3_bucket)
self._register_validator(S3BucketRegionValidator,
bucket=self.custom_s3_bucket,
region=get_region())
def get_templates_bucket_path():
"""Return a string containing the path of bucket."""
region = get_region()
s3_suffix = ".cn" if region.startswith("cn") else ""
return "https://{REGION}-aws-parallelcluster.s3.{REGION}.amazonaws.com{S3_SUFFIX}/templates/".format(
REGION=region, S3_SUFFIX=s3_suffix)
def _add_lambda_cleanup(self, policy_statements, build_tags):
lambda_cleanup_execution_role = None
if self.custom_cleanup_lambda_role:
execution_role = self.custom_cleanup_lambda_role
else:
# LambdaCleanupPolicies
self._add_resource_delete_policy(
policy_statements,
["cloudformation:DeleteStack"],
[
self.format_arn(
service="cloudformation",
resource="stack",
resource_name="{0}/{1}".format(
self.image_id, self._stack_unique_id()),
)
],
)
self._add_resource_delete_policy(
policy_statements,
["ec2:CreateTags"],
[
self.format_arn(
service="ec2",
account="",
resource="image",
region=region,
resource_name="*",
) for region in self._get_distribution_regions()
],
)
self._add_resource_delete_policy(
policy_statements,
["tag:TagResources"],
["*"],
)
self._add_resource_delete_policy(
policy_statements,
[
"iam:DetachRolePolicy", "iam:DeleteRole",
"iam:DeleteRolePolicy"
],
[
self.format_arn(
service="iam",
resource="role",
region="",
resource_name="{0}/{1}".format(
IAM_ROLE_PATH.strip("/"),
self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX + "Cleanup"),
),
)
],
)
self._add_resource_delete_policy(
policy_statements,
["lambda:DeleteFunction", "lambda:RemovePermission"],
[
self.format_arn(
service="lambda",
resource="function",
sep=":",
resource_name=self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX),
)
],
)
self._add_resource_delete_policy(
policy_statements,
["logs:DeleteLogGroup"],
[
self.format_arn(
service="logs",
resource="log-group",
sep=":",
resource_name="/aws/lambda/{0}:*".format(
self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX)),
)
],
)
self._add_resource_delete_policy(
policy_statements,
["iam:RemoveRoleFromInstanceProfile"],
[
self.format_arn(
service="iam",
resource="instance-profile",
region="",
resource_name="{0}/{1}".format(
IAM_ROLE_PATH.strip("/"),
self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX),
),
)
],
)
self._add_resource_delete_policy(
policy_statements,
["iam:DetachRolePolicy", "iam:DeleteRolePolicy"],
[
self.format_arn(
service="iam",
resource="role",
region="",
resource_name="{0}/{1}".format(
IAM_ROLE_PATH.strip("/"),
self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX),
),
)
],
)
self._add_resource_delete_policy(
policy_statements,
[
"SNS:GetTopicAttributes", "SNS:DeleteTopic",
"SNS:Unsubscribe"
],
[
self.format_arn(
service="sns",
resource="{0}".format(
self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX)),
)
],
)
policy_document = iam.PolicyDocument(statements=policy_statements)
managed_lambda_policy = [
Fn.sub(
"arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
),
]
# LambdaCleanupExecutionRole
lambda_cleanup_execution_role = iam.CfnRole(
self,
"DeleteStackFunctionExecutionRole",
managed_policy_arns=managed_lambda_policy,
assume_role_policy_document=get_assume_role_policy_document(
"lambda.amazonaws.com"),
path=IAM_ROLE_PATH,
policies=[
iam.CfnRole.PolicyProperty(
policy_document=policy_document,
policy_name="LambdaCleanupPolicy",
),
],
tags=build_tags,
role_name=self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX + "Cleanup"),
)
execution_role = lambda_cleanup_execution_role.attr_arn
# LambdaCleanupEnv
lambda_env = awslambda.CfnFunction.EnvironmentProperty(
variables={"IMAGE_STACK_ARN": self.stack_id})
# LambdaCWLogGroup
lambda_log = logs.CfnLogGroup(
self,
"DeleteStackFunctionLog",
log_group_name="/aws/lambda/{0}".format(
self._build_resource_name(IMAGEBUILDER_RESOURCE_NAME_PREFIX)),
)
# LambdaCleanupFunction
lambda_cleanup = awslambda.CfnFunction(
self,
"DeleteStackFunction",
function_name=self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX),
code=awslambda.CfnFunction.CodeProperty(
s3_bucket=self.config.custom_s3_bucket
or S3Bucket.get_bucket_name(
AWSApi.instance().sts.get_account_id(), get_region()),
s3_key=self.bucket.get_object_key(S3FileType.CUSTOM_RESOURCES,
"artifacts.zip"),
),
handler="delete_image_stack.handler",
memory_size=128,
role=execution_role,
runtime="python3.8",
timeout=900,
environment=lambda_env,
tags=build_tags,
)
permission = awslambda.CfnPermission(
self,
"DeleteStackFunctionPermission",
action="lambda:InvokeFunction",
principal="sns.amazonaws.com",
function_name=lambda_cleanup.attr_arn,
source_arn=Fn.ref("BuildNotificationTopic"),
)
lambda_cleanup.add_depends_on(lambda_log)
return lambda_cleanup, permission, lambda_cleanup_execution_role, lambda_log
def region(self):
"""Return bucket region."""
if self.__region is None:
self.__region = get_region()
return self.__region
def _validate_vpc(vpc_id):
# This function should be further expandend once we decide to allow the user to use his vpcs. For example, we should
# also check for the presence of a NAT gateway
if not VpcFactory(get_region()).check(vpc_id):
logging.error(
"WARNING: The VPC does not have the correct parameters set.")
def replace_url_parameters(url):
"""Replace ${Region} and ${URLSuffix} in url."""
return url.replace("${Region}",
get_region()).replace("${URLSuffix}",
get_url_domain_suffix())
def get_partition():
"""Get partition for the region set in the environment."""
return next(("aws-" + partition for partition in ["us-gov", "cn"]
if get_region().startswith(partition)), "aws")
def __init__(self, imagebuilder: ImageBuilder):
self.stack_exist = False
self.image_exist = False
self.region = get_region()
# image config file url
self.image_configuration = imagebuilder.config_url
def _get_log_group_arn(self):
"""Get log group arn."""
return "arn:{0}:logs:{1}:{2}:log-group:{3}".format(
get_partition(), get_region(), AWSApi.instance().sts.get_account_id(), self._log_group_name
)
def _get_custom_bucket(self):
"""Try to get custom bucket name from image tag or stack tag."""
custom_bucket_name = None
try:
custom_bucket_name = self.image.s3_bucket_name
except ImageError as e:
if not isinstance(e, NonExistingImageError):
raise _imagebuilder_error_mapper(e, f"Unable to get S3 bucket name from image {self.image_id}. {e}")
if custom_bucket_name is None:
try:
custom_bucket_name = self.stack.s3_bucket_name
except StackError as e:
raise _imagebuilder_error_mapper(e, f"Unable to get S3 bucket name from image {self.image_id}. {e}")
return (
custom_bucket_name
if custom_bucket_name != S3Bucket.get_bucket_name(AWSApi.instance().sts.get_account_id(), get_region())
else None
)
