def describe_cluster_instances_mock_response(instances):
result = []
for instance in instances:
node_type = instance.get("node_type") or "HeadNode"
response = {
"instanceId":
"i-0a9342a0000000000",
"instanceType":
"t2.micro",
"launchTime":
to_iso_timestr(to_utc_datetime("2021-04-30T00:00:00+00:00")),
"nodeType":
node_type,
"privateIpAddress":
"10.0.0.79",
"publicIpAddress":
"1.2.3.4",
"state":
"running",
}
if node_type == "Compute":
response["nodeType"] = "ComputeNode"
response["queueName"] = instance.get("queue_name")
result.append(response)
return {"instances": result}
def test_successful_request(self, mocker, client, scheduler, status,
last_status_updated_time):
mocker.patch("pcluster.aws.cfn.CfnClient.describe_stack",
return_value=cfn_describe_stack_mock_response(scheduler))
if scheduler == "slurm":
if status == "UNKNOWN":
dynamodb_item = {} # Test dynamodb item not exist
else:
dynamodb_item = {"Item": {"Status": status}}
if last_status_updated_time:
last_status_updated_time = str(last_status_updated_time)
dynamodb_item["Item"][
"LastUpdatedTime"] = last_status_updated_time
mocker.patch("pcluster.aws.dynamo.DynamoResource.get_item",
return_value=dynamodb_item)
elif scheduler == "awsbatch":
mocker.patch(
"pcluster.aws.batch.BatchClient.get_compute_environment_state",
return_value=status)
response = self._send_test_request(client)
with soft_assertions():
assert_that(response.status_code).is_equal_to(200)
expected_response = {"status": status}
if last_status_updated_time:
expected_response["lastStatusUpdatedTime"] = to_iso_timestr(
to_utc_datetime(last_status_updated_time))
assert_that(response.get_json()).is_equal_to(expected_response)
def convert_log(log):
log["logStreamArn"] = log.pop("arn")
if "storedBytes" in log:
del log["storedBytes"]
for ts_name in ["creationTime", "firstEventTimestamp", "lastEventTimestamp", "lastIngestionTime"]:
log[ts_name] = to_iso_timestr(to_utc_datetime(log[ts_name]))
return LogStream.from_dict(log)
def test_describe_of_image_already_available(self, client, mocker):
mocker.patch(
"pcluster.aws.ec2.Ec2Client.describe_image_by_id_tag",
return_value=_create_image_info("image1"),
)
mocker.patch("pcluster.aws.cfn.CfnClient.describe_stack_resource",
return_value=None)
mocker.patch(
"pcluster.api.controllers.image_operations_controller._presigned_config_url",
return_value="https://parallelcluster.aws.com/bucket/key",
)
expected_response = {
"creationTime": to_iso_timestr(datetime(2021, 4, 12)),
"ec2AmiInfo": {
"amiId":
"image1",
"amiName":
"image1",
"architecture":
"x86_64",
"state":
Ec2AmiState.AVAILABLE,
"description":
"description",
"tags": [
{
"key": "parallelcluster:image_id",
"value": "image1"
},
{
"key": "parallelcluster:version",
"value": "3.0.0"
},
{
"key": "parallelcluster:build_config",
"value": "s3://bucket/key"
},
],
},
"imageBuildStatus": ImageBuildStatus.BUILD_COMPLETE,
"imageConfiguration": {
"url": "https://parallelcluster.aws.com/bucket/key"
},
"imageId": "image1",
"region": "us-east-1",
"version": "3.0.0",
}
response = self._send_test_request(client, "image1")
with soft_assertions():
assert_that(response.status_code).is_equal_to(200)
assert_that(response.get_json()).is_equal_to(expected_response)
def default(self, obj):
"""Override the base method to add support for model objects serialization."""
if isinstance(obj, Model):
dikt = {}
for attr, _ in six.iteritems(obj.openapi_types):
value = getattr(obj, attr)
if value is None and not self.include_nulls:
continue
attr = obj.attribute_map[attr]
dikt[attr] = value
return dikt
elif isinstance(obj, datetime.date):
return to_iso_timestr(obj)
return FlaskJSONEncoder.default(self, obj)
def test_successful_request(self, mocker, client, scheduler, status,
last_status_updated_time):
mocker.patch("pcluster.aws.cfn.CfnClient.describe_stack",
return_value=cfn_describe_stack_mock_response(scheduler))
config_mock = mocker.patch("pcluster.models.cluster.Cluster.config")
config_mock.scheduling.scheduler = scheduler
if scheduler == "slurm":
if status == "UNKNOWN":
dynamodb_item = {} # Test dynamodb item not exist
else:
dynamodb_item = {"Item": {"Status": status}}
if last_status_updated_time:
last_status_updated_time = str(last_status_updated_time)
dynamodb_item["Item"][
"LastUpdatedTime"] = last_status_updated_time
# mock the method to check the status before update
mocker.patch("pcluster.aws.dynamo.DynamoResource.get_item",
return_value=dynamodb_item)
# mock the method to update the item in dynamodb
mocker.patch("pcluster.aws.dynamo.DynamoResource.put_item")
elif scheduler == "awsbatch":
mocker.patch(
"pcluster.aws.batch.BatchClient.get_compute_environment_state",
return_value=status)
if status == "ENABLED":
mocker.patch(
"pcluster.aws.batch.BatchClient.enable_compute_environment"
)
elif status == "DISABLED":
mocker.patch(
"pcluster.aws.batch.BatchClient.disable_compute_environment"
)
response = self._send_test_request(client,
request_body={"status": status})
with soft_assertions():
assert_that(response.status_code).is_equal_to(200)
expected_response = {"status": status}
if last_status_updated_time:
expected_response["lastStatusUpdatedTime"] = to_iso_timestr(
to_utc_datetime(last_status_updated_time))
assert_that(response.get_json()).is_equal_to(expected_response)
def test_describe_image_in_failed_state_with_reasons_and_associated_imagebuilder_image(
self, client, mocker):
mocker.patch(
"pcluster.aws.ec2.Ec2Client.describe_image_by_id_tag",
side_effect=ImageNotFoundError("describe_image_by_id_tag"),
)
mocker.patch(
"pcluster.aws.cfn.CfnClient.describe_stack",
return_value=_create_stack("image1",
CloudFormationStackStatus.CREATE_FAILED,
"cfn test reason"),
)
mocker.patch("pcluster.aws.cfn.CfnClient.describe_stack_resource",
return_value=None)
mocker.patch(
"pcluster.aws.cfn.CfnClient.describe_stack_resource",
return_value={
"StackResourceDetail": {
"PhysicalResourceId": "test_id"
}
},
)
mocker.patch(
"pcluster.aws.imagebuilder.ImageBuilderClient.get_image_state",
return_value={
"status": ImageBuilderImageStatus.FAILED,
"reason": "img test reason"
},
)
mocker.patch(
"pcluster.api.controllers.image_operations_controller._presigned_config_url",
return_value="https://parallelcluster.aws.com/bucket/key",
)
expected_response = {
"cloudformationStackArn":
"arn:image1",
"imageBuildLogsArn":
"arn:image1:build_log",
"cloudformationStackCreationTime":
to_iso_timestr(to_utc_datetime("2021-04-12 00:00:00")),
"cloudformationStackTags": [
{
"key": "parallelcluster:image_id",
"value": "image1"
},
{
"key": "parallelcluster:version",
"value": "3.0.0"
},
{
"key": "parallelcluster:build_config",
"value": "s3://bucket/key"
},
{
"key": "parallelcluster:build_log",
"value": "arn:image1:build_log"
},
],
"cloudformationStackStatus":
CloudFormationStackStatus.CREATE_FAILED,
"cloudformationStackStatusReason":
"cfn test reason",
"imageBuildStatus":
ImageBuildStatus.BUILD_FAILED,
"imageConfiguration": {
"url": "https://parallelcluster.aws.com/bucket/key"
},
"imageId":
"image1",
"imagebuilderImageStatus":
ImageBuilderImageStatus.FAILED,
"imagebuilderImageStatusReason":
"img test reason",
"region":
"us-east-1",
"version":
"3.0.0",
}
response = self._send_test_request(client, "image1")
with soft_assertions():
assert_that(response.status_code).is_equal_to(200)
assert_that(response.get_json()).is_equal_to(expected_response)
def test_describe_of_image_not_yet_available_with_no_associated_imagebuilder_image(
self, client, mocker):
mocker.patch(
"pcluster.aws.ec2.Ec2Client.describe_image_by_id_tag",
side_effect=ImageNotFoundError("describe_image_by_id_tag"),
)
mocker.patch(
"pcluster.aws.cfn.CfnClient.describe_stack",
return_value=_create_stack(
"image1", CloudFormationStackStatus.CREATE_IN_PROGRESS),
)
mocker.patch("pcluster.aws.cfn.CfnClient.describe_stack_resource",
return_value=None)
mocker.patch(
"pcluster.api.controllers.image_operations_controller._presigned_config_url",
return_value="https://parallelcluster.aws.com/bucket/key",
)
expected_response = {
"imageConfiguration": {
"url": "https://parallelcluster.aws.com/bucket/key"
},
"imageId":
"image1",
"imageBuildStatus":
ImageBuildStatus.BUILD_IN_PROGRESS,
"cloudformationStackStatus":
CloudFormationStackStatus.CREATE_IN_PROGRESS,
"cloudformationStackArn":
"arn:image1",
"imageBuildLogsArn":
"arn:image1:build_log",
"cloudformationStackCreationTime":
to_iso_timestr(datetime(2021, 4, 12)),
"cloudformationStackTags": [
{
"key": "parallelcluster:image_id",
"value": "image1"
},
{
"key": "parallelcluster:version",
"value": "3.0.0"
},
{
"key": "parallelcluster:build_config",
"value": "s3://bucket/key"
},
{
"key": "parallelcluster:build_log",
"value": "arn:image1:build_log"
},
],
"region":
"us-east-1",
"version":
"3.0.0",
}
response = self._send_test_request(client, "image1")
with soft_assertions():
assert_that(response.status_code).is_equal_to(200)
assert_that(response.get_json()).is_equal_to(expected_response)
def test_execute(self, mocker, set_env, args):
logs = LogStreams()
logs.log_streams = [
{
"logStreamName":
"ip-10-0-0-102.i-0717e670ad2549e72.cfn-init",
"creationTime":
1622802842228,
"firstEventTimestamp":
1622802790248,
"lastEventTimestamp":
1622802893126,
"lastIngestionTime":
1622802903119,
"uploadSequenceToken":
"4961...",
"arn":
("arn:aws:logs:eu-west-1:111:log-group:/aws/parallelcluster/"
"test22-202106041223:log-stream:ip-10-0-0-102.i-0717e670ad2549e72.cfn-init"
),
"storedBytes":
0,
},
{
"logStreamName":
"ip-10-0-0-102.i-0717e670ad2549e72.chef-client",
"creationTime":
1622802842207,
"firstEventTimestamp":
1622802837114,
"lastEventTimestamp":
1622802861226,
"lastIngestionTime":
1622802897558,
"uploadSequenceToken":
"4962...",
"arn":
("arn:aws:logs:eu-west-1:111:log-group:/aws/parallelcluster/"
"test22-202106041223:log-stream:ip-10-0-0-102.i-0717e670ad2549e72.chef-client"
),
"storedBytes":
0,
},
]
logs.next_token = "123-456"
list_log_streams_mock = mocker.patch(
"pcluster.api.controllers.image_logs_controller.ImageBuilder.list_log_streams",
return_value=logs)
set_env("AWS_DEFAULT_REGION", "us-east-1")
base_args = ["list-image-log-streams"]
command = base_args + self._build_cli_args({**REQUIRED_ARGS, **args})
out = run(command)
# cfn stack events are not displayed if next-token is passed
expected_out = [
{
"logStreamName":
"ip-10-0-0-102.i-0717e670ad2549e72.cfn-init",
"firstEventTimestamp":
to_iso_timestr(to_utc_datetime(1622802790248)),
"lastEventTimestamp":
to_iso_timestr(to_utc_datetime(1622802893126)),
},
{
"logStreamName":
"ip-10-0-0-102.i-0717e670ad2549e72.chef-client",
"firstEventTimestamp":
to_iso_timestr(to_utc_datetime(1622802837114)),
"lastEventTimestamp":
to_iso_timestr(to_utc_datetime(1622802861226)),
},
]
assert_that(out["nextToken"]).is_equal_to(logs.next_token)
for i in range(len(logs.log_streams)):
select_keys = {
"logStreamName", "firstEventTimestamp", "lastEventTimestamp"
}
out_select = {
k: v
for k, v in out["logStreams"][i].items() if k in select_keys
}
assert_that(out_select).is_equal_to(expected_out[i])
assert_that(list_log_streams_mock.call_args).is_length(2)
# verify arguments
kwargs = {"next_token": None}
kwargs.update(args)
list_log_streams_mock.assert_called_with(**kwargs)
def convert_event(event):
event = {k[0].lower() + k[1:]: v for k, v in event.items()}
event["timestamp"] = to_iso_timestr(to_utc_datetime(
event["timestamp"]))
return StackEvent.from_dict(event)
def convert_log_event(event):
del event["ingestionTime"]
event["timestamp"] = to_iso_timestr(to_utc_datetime(
event["timestamp"]))
return LogEvent.from_dict(event)