def delete_image(image_id, region=None, force=None):
"""
Initiate the deletion of the custom ParallelCluster image.
:param image_id: Id of the image
:type image_id: str
:param region: AWS Region that the operation corresponds to.
:type region: str
:param force: Force deletion in case there are instances using the AMI or in case the AMI is shared
(Defaults to 'false'.)
:type force: bool
:rtype: DeleteImageResponseContent
"""
force = force or False
imagebuilder = ImageBuilder(image_id=image_id)
image, stack = _get_underlying_image_or_stack(imagebuilder)
imagebuilder.delete(force=force)
return DeleteImageResponseContent(
image=ImageInfoSummary(
image_id=image_id,
image_build_status=ImageBuildStatus.DELETE_IN_PROGRESS,
cloudformation_stack_status=CloudFormationStackStatus.DELETE_IN_PROGRESS if stack else None,
cloudformation_stack_arn=stack.id if stack else None,
region=os_lib.environ.get("AWS_DEFAULT_REGION"),
version=stack.version if stack else image.version,
)
)
def list_image_log_streams(image_id, region=None, next_token=None):
"""
Retrieve the list of log streams associated with an image.
:param image_id: Id of the image.
:type image_id: str
:param region: Region that the given image belongs to.
:type region: str
:param next_token: Token to use for paginated requests.
:type next_token: str
:rtype: ListImageLogStreamsResponseContent
"""
def convert_log(log):
log["logStreamArn"] = log.pop("arn")
if "storedBytes" in log:
del log["storedBytes"]
for ts_name in ["creationTime", "firstEventTimestamp", "lastEventTimestamp", "lastIngestionTime"]:
log[ts_name] = to_iso_timestr(to_utc_datetime(log[ts_name]))
return LogStream.from_dict(log)
imagebuilder = ImageBuilder(image_id=image_id)
logs = imagebuilder.list_log_streams(next_token=next_token)
log_streams = [convert_log(log) for log in logs.log_streams]
next_token = logs.next_token
return ListImageLogStreamsResponseContent(log_streams=log_streams, next_token=next_token)
def test_config_object_initialization(config, expected_result,
expected_error_message):
if expected_error_message:
with pytest.raises(BadRequest, match=expected_error_message):
_ = ImageBuilder(config=config).config
elif config is None:
result = ImageBuilder(config=config).config
assert_that(result).is_none()
else:
result = ImageBuilder(config=config).config
with soft_assertions():
assert_that(result.build.instance_type).is_equal_to(
expected_result.build.instance_type)
assert_that(result.build.parent_image).is_equal_to(
expected_result.build.parent_image)
def _export_image_logs(args: Namespace, output_file: str = None):
"""Export the logs associated to the image."""
LOGGER.debug("Beginning export of logs for the image: %s",
args.image_id)
# retrieve imagebuilder config and generate model
imagebuilder = ImageBuilder(image_id=args.image_id)
url = imagebuilder.export_logs(
bucket=args.bucket,
bucket_prefix=args.bucket_prefix,
keep_s3_objects=args.keep_s3_objects,
start_time=args.start_time,
end_time=args.end_time,
output_file=output_file,
)
LOGGER.debug("Image's logs exported correctly to %s", url)
return {"path": output_file} if output_file else {"url": url}
def test_delete_with_ec2_error(mocker, error, returned_error):
with pytest.raises(returned_error):
mock_aws_api(mocker)
mocker.patch("pcluster.aws.cfn.CfnClient.stack_exists",
return_value=False)
mocker.patch("pcluster.aws.ec2.Ec2Client.image_exists",
side_effect=(error))
ImageBuilder("imageId").delete(force=True)
def image_builder(self, set_env):
set_env("AWS_DEFAULT_REGION", "us-east-2")
return ImageBuilder(
image_id=FAKE_ID,
stack=ImageBuilderStack({
"StackName":
FAKE_NAME,
"CreationTime":
"2021-06-04 10:23:20.199000+00:00"
}),
)
def delete_image(image_id: str, region: str, force: bool = False):
"""
Delete image and imagebuilder stack.
:param image_id: Id for pcluster Image, the same as imagebuilder cfn stack name
:param region: AWS region
:param force: Delete image even if the image is shared or instance is using it
"""
try:
if region:
os.environ["AWS_DEFAULT_REGION"] = region
# retrieve imagebuilder config and generate model
imagebuilder = ImageBuilder(image_id=image_id)
image, _ = PclusterApi._get_underlying_image_or_stack(imagebuilder)
imagebuilder.delete(force=force)
if image:
return ImageBuilderImageInfo(imagebuilder=imagebuilder)
return ImageBuilderStackInfo(imagebuilder=imagebuilder)
except Exception as e:
return ApiFailure(str(e))
def get_image_stack_events(image_id, region=None, next_token=None):
"""
Retrieve the events associated with the stack for a given image build.
:param image_id: Id of the image.
:type image_id: str
:param region: AWS Region that the operation corresponds to.
:type region: str
:param next_token: Token to use for paginated requests.
:type next_token: str
:rtype: GetImageStackEventsResponseContent
"""
imagebuilder = ImageBuilder(image_id=image_id)
stack_events = imagebuilder.get_stack_events(next_token=next_token)
def convert_event(event):
event = {k[0].lower() + k[1:]: v for k, v in event.items()}
event["timestamp"] = to_iso_timestr(to_utc_datetime(event["timestamp"]))
return StackEvent.from_dict(event)
events = [convert_event(event) for event in stack_events["StackEvents"]]
return GetImageStackEventsResponseContent(next_token=stack_events.get("NextToken", None), events=events)
def list_images(region: str):
"""
List existing images.
:param region: AWS region
:return list
"""
try:
if region:
os.environ["AWS_DEFAULT_REGION"] = region
# get built images by image name tag
images = AWSApi.instance().ec2.get_images()
imagebuilders = [
ImageBuilder(image=image, image_id=image.pcluster_image_id)
for image in images
]
images_response = [
ImageBuilderImageInfo(imagebuilder=imagebuilder)
for imagebuilder in imagebuilders
]
# get building image stacks by image name tag
stacks, _ = AWSApi.instance().cfn.get_imagebuilder_stacks()
imagebuilder_stacks = [
ImageBuilder(image_id=stack.get("StackName"),
stack=ImageBuilderStack(stack))
for stack in stacks
]
imagebuilder_stacks_response = [
ImageBuilderStackInfo(imagebuilder=imagebuilder)
for imagebuilder in imagebuilder_stacks
]
return images_response + imagebuilder_stacks_response
except Exception as e:
return ApiFailure(str(e))
def build_image(imagebuilder_config: str,
image_id: str,
region: str,
disable_rollback: bool = True):
"""
Load imagebuilder model from imagebuilder_config and create stack.
:param imagebuilder_config: imagebuilder configuration (str)
:param image_id: Id for pcluster Image, the same as imagebuilder cfn stack name
:param region: AWS region
:param disable_rollback: Disable rollback in case of failures
"""
try:
# Generate model from imagebuilder config dict
if region:
os.environ["AWS_DEFAULT_REGION"] = region
imagebuilder = ImageBuilder(image_id=image_id,
config=imagebuilder_config)
imagebuilder.create(disable_rollback)
return ImageBuilderStackInfo(imagebuilder=imagebuilder)
except ImageBuilderActionError as e:
return ApiFailure(str(e), e.validation_failures)
except Exception as e:
return ApiFailure(str(e))
def describe_image(image_id: str, region: str):
"""
Get image information.
:param image_id: Id for pcluster Image, the same as imagebuilder cfn stack name
:param region: AWS region
"""
try:
if region:
os.environ["AWS_DEFAULT_REGION"] = region
imagebuilder = ImageBuilder(image_id=image_id)
image, _ = PclusterApi._get_underlying_image_or_stack(imagebuilder)
if image:
return ImageBuilderImageInfo(imagebuilder=imagebuilder)
return ImageBuilderStackInfo(imagebuilder=imagebuilder)
except Exception as e:
return ApiFailure(str(e))
def describe_image(image_id, region=None):
"""
Get detailed information about an existing image.
:param image_id: Id of the image
:type image_id: str
:param region: AWS Region that the operation corresponds to.
:type region: str
:rtype: DescribeImageResponseContent
"""
imagebuilder = ImageBuilder(image_id=image_id)
try:
return _image_to_describe_image_response(imagebuilder)
except NonExistingImageError:
try:
return _stack_to_describe_image_response(imagebuilder)
except NonExistingStackError:
raise NotFoundException("No image or stack associated with ParallelCluster image id: {}.".format(image_id))
def build_image(
build_image_request_content,
suppress_validators=None,
validation_failure_level=None,
dryrun=None,
rollback_on_failure=None,
region=None,
):
"""
Create a custom ParallelCluster image in a given region.
:param build_image_request_content:
:param suppress_validators: Identifies one or more config validators to suppress.
Format: (ALL|type:[A-Za-z0-9]+)
:type suppress_validators: List[str]
:param validation_failure_level: Min validation level that will cause the image creation to fail.
Defaults to 'error'.
:type validation_failure_level: dict | bytes
:param dryrun: Only perform request validation without creating any resource.
It can be used to validate the image configuration. Response code: 200
(Defaults to 'false'.)
:type dryrun: bool
:param rollback_on_failure: When set, will automatically initiate an image stack rollback on failure.
(Defaults to 'false'.)
:type rollback_on_failure: bool
:param region: AWS Region that the operation corresponds to.
:type region: str
:rtype: BuildImageResponseContent
"""
assert_node_executable()
configure_aws_region_from_config(region, build_image_request_content["imageConfiguration"])
rollback_on_failure = rollback_on_failure if rollback_on_failure is not None else False
disable_rollback = not rollback_on_failure
validation_failure_level = validation_failure_level or ValidationLevel.ERROR
dryrun = dryrun or False
build_image_request_content = BuildImageRequestContent.from_dict(build_image_request_content)
try:
image_id = build_image_request_content.image_id
config = build_image_request_content.image_configuration
if not config:
LOGGER.error("Failed: configuration is required and cannot be empty")
raise BadRequestException("configuration is required and cannot be empty")
imagebuilder = ImageBuilder(image_id=image_id, config=config)
if dryrun:
imagebuilder.validate_create_request(
validator_suppressors=get_validator_suppressors(suppress_validators),
validation_failure_level=FailureLevel[validation_failure_level],
)
raise DryrunOperationException()
suppressed_validation_failures = imagebuilder.create(
disable_rollback=disable_rollback,
validator_suppressors=get_validator_suppressors(suppress_validators),
validation_failure_level=FailureLevel[validation_failure_level],
)
return BuildImageResponseContent(
image=_imagebuilder_stack_to_image_info_summary(imagebuilder.stack),
validation_messages=validation_results_to_config_validation_errors(suppressed_validation_failures) or None,
)
except ConfigValidationError as e:
raise _handle_config_validation_error(e)
except BadRequestImageBuilderActionError as e:
errors = validation_results_to_config_validation_errors(e.validation_failures)
raise BuildImageBadRequestException(
BuildImageBadRequestExceptionResponseContent(message=str(e), configuration_validation_errors=errors or None)
)
def get_image_log_events(
image_id,
log_stream_name,
region=None,
next_token=None,
start_from_head=None,
limit=None,
start_time: str = None,
end_time: str = None,
):
"""
Retrieve the events associated with an image build.
:param image_id: Id of the image.
:type image_id: str
:param log_stream_name: Name of the log stream.
:type log_stream_name: str
:param region: AWS Region that the operation corresponds to.
:type region: str
:param next_token: Token to use for paginated requests.
:type next_token: str
:param start_from_head: If the value is true, the earliest log events are returned first. If the value is false,
the latest log events are returned first. (Defaults to 'false'.)
:type start_from_head: bool
:param limit: The maximum number of log events returned. If you don't specify a value, the maximum is as many
log events as can fit in a response size of 1 MB, up to 10,000 log events.
:type limit:
:param start_time: The start of the time range, expressed in ISO 8601 format
(e.g. '2021-01-01T20:00:00Z'). Events with a timestamp equal to this time or later
than this time are included.
:type start_time: str
:param end_time: The end of the time range, expressed in ISO 8601 format (e.g. '2021-01-01T20:00:00Z').
Events with a timestamp equal to or later than this time are not included.
:type end_time: str
:rtype: GetImageLogEventsResponseContent
"""
start_dt = start_time and validate_timestamp(start_time, "start_time")
end_dt = end_time and validate_timestamp(end_time, "end_time")
if start_time and end_time and start_dt >= end_dt:
raise BadRequestException("start_time filter must be earlier than end_time filter.")
if limit and limit <= 0:
raise BadRequestException("'limit' must be a positive integer.")
imagebuilder = ImageBuilder(image_id=image_id)
log_events = imagebuilder.get_log_events(
log_stream_name,
start_time=start_dt,
end_time=end_dt,
start_from_head=start_from_head,
limit=limit,
next_token=next_token,
)
def convert_log_event(event):
del event["ingestionTime"]
event["timestamp"] = to_iso_timestr(to_utc_datetime(event["timestamp"]))
return LogEvent.from_dict(event)
events = [convert_log_event(e) for e in log_events.events]
return GetImageLogEventsResponseContent(
events=events, next_token=log_events.next_ftoken, prev_token=log_events.next_btoken
)
