def test_inspect_pending_images(self):
"""
Test that only old "pending" images are found and reinspected.
Note that we effectively time-travel here to points in the past to
create the account, images, and instances. This is necessary because
updated_at is automatically set by Django and cannot be manually set,
but we need things with specific older updated_at times.
"""
real_now = get_now()
yesterday = real_now - datetime.timedelta(days=1)
with clouditardis(yesterday):
account = account_helper.generate_cloud_account()
image_old_inspected = account_helper.generate_image()
image_old_pending = account_helper.generate_image(
status=MachineImage.PENDING)
# an instance exists using old inspected image.
account_helper.generate_instance(cloud_account=account,
image=image_old_inspected)
# an instance exists using old pending image.
instance_old_pending = account_helper.generate_instance(
cloud_account=account, image=image_old_pending)
# another instance exists using the same old pending image, but the
# image should still only be reinspected once regardless of how
# many instances used it.
account_helper.generate_instance(cloud_account=account,
image=image_old_pending)
one_hour_ago = real_now - datetime.timedelta(seconds=60 * 60)
with clouditardis(one_hour_ago):
image_new_inspected = account_helper.generate_image()
image_new_pending = account_helper.generate_image(
status=MachineImage.PENDING)
# an instance exists using new inspected image.
account_helper.generate_instance(cloud_account=account,
image=image_new_inspected)
# an instance exists using new pending image, but it should not
# trigger inspection because the image is not old enough.
account_helper.generate_instance(cloud_account=account,
image=image_new_pending)
expected_calls = [
call(
account.content_object.account_arn,
image_old_pending.content_object.ec2_ami_id,
instance_old_pending.content_object.region,
)
]
with patch.object(tasks, "start_image_inspection") as mock_start:
tasks.inspect_pending_images()
mock_start.assert_has_calls(expected_calls, any_order=True)
def test_enable_verify_task_existing(
self,
mock_describe,
mock_verify,
mock_notify_sources,
):
"""Verify Task is assigned and re-enabled since it already existed."""
aws_clount = self.account.content_object
self.assertIsNone(aws_clount.verify_task)
schedule, _ = IntervalSchedule.objects.get_or_create(
every=settings.SCHEDULE_VERIFY_VERIFY_TASKS_INTERVAL,
period=IntervalSchedule.SECONDS,
)
verify_task = PeriodicTask.objects.create(
interval=schedule,
name=f"Verify {aws_clount.account_arn}.",
task="api.clouds.aws.tasks.verify_account_permissions",
kwargs=json.dumps({
"account_arn": aws_clount.account_arn,
}),
start_time=self.created_at,
)
verify_task.enabled = False
verify_task.save()
enable_date = util_helper.utc_dt(2019, 1, 4, 0, 0, 0)
with util_helper.clouditardis(enable_date):
self.account.enable()
self.assertIsNotNone(aws_clount.verify_task)
def test_enable_failure_too_many_trails(self, mock_notify_sources):
"""Test that enabling an account rolls back if too many trails present."""
# Normally you shouldn't directly manipulate the is_enabled value,
# but here we need to force it down to check that it gets set back.
self.account.is_enabled = False
self.account.save()
self.account.refresh_from_db()
self.assertFalse(self.account.is_enabled)
enable_date = util_helper.utc_dt(2019, 1, 4, 0, 0, 0)
with patch.object(
aws, "configure_cloudtrail"
) as mock_cloudtrail, patch.object(
aws, "verify_account_access"
) as mock_verify, patch.object(aws, "get_session"), patch(
"api.clouds.aws.tasks.initial_aws_describe_instances"
) as mock_initial_aws_describe_instances, util_helper.clouditardis(
enable_date):
mock_cloudtrail.side_effect = MaximumNumberOfTrailsExceededException(
"MaximumNumberOfTrailsExceededException", )
mock_verify.return_value = (True, "")
with self.assertLogs("api.clouds.aws.util", level="WARNING") as cm:
self.account.enable()
self.assertIn(
str(mock_cloudtrail.side_effect.detail),
cm.records[1].message,
)
mock_cloudtrail.assert_called()
mock_initial_aws_describe_instances.delay.assert_not_called()
self.account.refresh_from_db()
self.assertFalse(self.account.is_enabled)
self.assertEqual(self.account.enabled_at, self.account.created_at)
def test_instance_already_power_off(self, mock_aws,
mock_calculate_max_concurrent):
"""Don't create power_off for instance that already has recent power_off."""
event = account_helper.generate_single_instance_event(
self.instance,
self.power_off_time,
event_type=InstanceEvent.TYPE.power_off)
self.process_event(event)
mock_calculate_max_concurrent.reset_mock()
described_instances = {
self.region: [
util_helper.generate_dummy_describe_instance(
instance_id=self.aws_instance.ec2_instance_id,
state=aws.InstanceState.stopped,
)
]
}
mock_aws.describe_instances_everywhere.return_value = described_instances
describe_time = self.power_off_time + datetime.timedelta(days=1)
with util_helper.clouditardis(describe_time):
tasks.initial_aws_describe_instances(self.account.id)
mock_calculate_max_concurrent.assert_not_called()
self.assertExpectedEventsAndRun()
def test_power_off_not_present(self, mock_aws,
mock_calculate_max_concurrent):
"""Create power_off event for running instance not present in the describe."""
described_instances = {} # empty means no instances found.
mock_aws.describe_instances_everywhere.return_value = described_instances
with util_helper.clouditardis(self.power_off_time):
tasks.initial_aws_describe_instances(self.account.id)
mock_calculate_max_concurrent.assert_called()
self.assertExpectedEventsAndRun()
def test_disable_verify_task_existing(self, mock_notify_sources):
"""Verify Task is disabled."""
aws_clount = self.account.content_object
enable_date = util_helper.utc_dt(2019, 1, 4, 0, 0, 0)
with patch("api.clouds.aws.util.verify_permissions"), patch(
"api.clouds.aws.tasks.initial_aws_describe_instances"
), util_helper.clouditardis(enable_date):
self.account.enable()
self.assertTrue(aws_clount.verify_task.enabled)
self.account.content_object._disable_verify_task()
self.assertFalse(aws_clount.verify_task.enabled)
def test_generate_azure_account_default(self):
"""Assert generation of an AwsAccount with default/no args."""
created_at = util_helper.utc_dt(2017, 1, 1, 0, 0, 0)
with util_helper.clouditardis(created_at):
account = helper.generate_cloud_account(cloud_type="azure")
self.assertIsInstance(account, CloudAccount)
self.assertIsNotNone(account.content_object.subscription_id)
self.assertIsNotNone(account.content_object.tenant_id)
self.assertEqual(account.created_at, created_at)
self.assertTrue(account.is_enabled)
self.assertEqual(account.enabled_at, created_at)
self.assertIsNotNone(account.platform_authentication_id)
self.assertIsNotNone(account.platform_application_id)
self.assertIsNotNone(account.platform_source_id)
def test_generate_aws_account_default(self):
"""Assert generation of an AwsAccount with default/no args."""
created_at = util_helper.utc_dt(2017, 1, 1, 0, 0, 0)
with util_helper.clouditardis(created_at):
account = helper.generate_cloud_account()
self.assertIsInstance(account, CloudAccount)
self.assertIsNotNone(
re.match(r"\d{1,12}", str(account.content_object.aws_account_id)))
self.assertEqual(account.created_at, created_at)
self.assertTrue(account.is_enabled)
self.assertEqual(account.enabled_at, created_at)
self.assertIsNotNone(account.platform_authentication_id)
self.assertIsNotNone(account.platform_application_id)
self.assertIsNotNone(account.platform_source_id)
self.assertIsNotNone(account.content_object.verify_task)
def test_clouditardis(self):
"""Assert that clouditardis breaks the space-time continuum."""
the_present_time = datetime.datetime.now(datetime.timezone.utc)
the_present_date = the_present_time.date()
the_day_i_invented_time_travel = helper.utc_dt(1955, 11, 5, 4, 29)
with helper.clouditardis(the_day_i_invented_time_travel):
# local imports to simulate tests
from util.misc import get_now as _get_now, get_today as _get_today
the_past_time = _get_now()
the_past_date = _get_today()
self.assertLess(the_past_time, the_present_time)
self.assertLess(the_past_date, the_present_date)
def test_instance_found_not_running(self, mock_aws,
mock_calculate_max_concurrent):
"""Create power_off event for instance found stopped in the describe."""
described_instances = {
self.region: [
util_helper.generate_dummy_describe_instance(
instance_id=self.aws_instance.ec2_instance_id,
state=aws.InstanceState.stopped,
)
]
}
mock_aws.describe_instances_everywhere.return_value = described_instances
with util_helper.clouditardis(self.power_off_time):
tasks.initial_aws_describe_instances(self.account.id)
mock_calculate_max_concurrent.assert_called()
self.assertExpectedEventsAndRun()
def test_disable_succeeds(self, mock_notify_sources):
"""
Test that disabling an account does all the relevant cleanup.
Disabling a CloudAccount having an AwsCloudAccount should:
- set the CloudAccount.is_enabled to False
- create a power_off InstanceEvent for any powered-on instances
- disable the CloudTrail (via AwsCloudAccount.disable)
"""
self.assertTrue(self.account.is_enabled)
self.assertEqual(self.account.enabled_at, self.account.created_at)
instance = helper.generate_instance(cloud_account=self.account)
runtimes = [
(
util_helper.utc_dt(2019, 1, 1, 0, 0, 0),
util_helper.utc_dt(2019, 1, 2, 0, 0, 0),
),
(util_helper.utc_dt(2019, 1, 3, 0, 0, 0), None),
]
disable_date = util_helper.utc_dt(2019, 1, 4, 0, 0, 0)
with util_helper.clouditardis(disable_date):
# We need to generate the runs while inside the clouditardis because we
# don't need an ever-growing number of daily ConcurrentUsage objects being
# created as the real-world clock ticks forward.
for runtime in runtimes:
helper.generate_single_run(instance=instance, runtime=runtime)
self.assertEqual(3, aws_models.AwsInstanceEvent.objects.count())
with patch("api.clouds.aws.util.delete_cloudtrail"
) as mock_delete_cloudtrail:
mock_delete_cloudtrail.return_value = True
self.account.disable()
mock_delete_cloudtrail.assert_called()
self.account.refresh_from_db()
self.assertFalse(self.account.is_enabled)
self.assertEqual(self.account.enabled_at, self.created_at)
self.assertEqual(4, aws_models.AwsInstanceEvent.objects.count())
last_event = (models.InstanceEvent.objects.filter(
instance=instance).order_by("-occurred_at").first())
self.assertEqual(last_event.event_type,
models.InstanceEvent.TYPE.power_off)
def setUp(self):
"""Set up a bunch of test data."""
created_at = util_helper.utc_dt(2019, 4, 1, 1, 0, 0)
with util_helper.clouditardis(created_at):
self.user = util_helper.generate_test_user()
self.account = api_helper.generate_cloud_account(user=self.user)
self.image = api_helper.generate_image(rhel_detected=True)
self.instance = api_helper.generate_instance(self.account,
image=self.image)
api_helper.generate_single_run(
self.instance,
(
util_helper.utc_dt(2019, 5, 1, 1, 0, 0),
util_helper.utc_dt(2019, 5, 3, 1, 0, 0),
),
image=self.instance.machine_image,
calculate_concurrent_usage=False,
)
self.request_date = datetime.date(2019, 5, 2)
calculate_max_concurrent_usage(self.request_date, self.user.id)
self.factory = APIRequestFactory()
def test_enable_succeeds(self, mock_notify_sources):
"""
Test that enabling an account does all the relevant verification and setup.
Disabling a CloudAccount having an AwsCloudAccount should:
- set the CloudAccount.is_enabled to True
- verify AWS IAM access and permissions
- delay task to describe instances
"""
# Normally you shouldn't directly manipulate the is_enabled value,
# but here we need to force it down to check that it gets set back.
self.account.is_enabled = False
self.account.save()
self.account.refresh_from_db()
self.assertFalse(self.account.is_enabled)
enable_date = util_helper.utc_dt(2019, 1, 4, 0, 0, 0)
with patch(
"api.clouds.aws.util.verify_permissions"
) as mock_verify_permissions, patch(
"api.clouds.aws.tasks.initial_aws_describe_instances"
) as mock_initial_aws_describe_instances, util_helper.clouditardis(
enable_date):
self.account.enable()
mock_verify_permissions.assert_called()
mock_initial_aws_describe_instances.delay.assert_called()
self.account.refresh_from_db()
self.assertTrue(self.account.is_enabled)
self.assertEqual(self.account.enabled_at, enable_date)
self.assertTrue(self.account.content_object.verify_task.enabled)
self.assertEqual(
self.account.content_object.verify_task.start_time,
self.account.content_object.created_at,
)
def test_initial_aws_describe_instances_after_disable_enable(
self,
mock_util_aws,
mock_aws,
mock_start,
mock_sources_notify,
mock_schedule_concurrent_calculation_task,
):
"""
Test calling initial_aws_describe_instances multiple times.
Historically (and in the simplified ideal happy-path), we would only ever call
initial_aws_describe_instances exactly once when an AwsCloudAccount was first
created. However, since we added support to disable accounts, it's now possible
(and likely common) for an account to be created (and enabled), call describe,
be disabled, be enabled again, and call describe again during the enable.
We need to ensure that running described instances that generated "power_off"
InstanceEvents as a result of AwsCloudAccount.disable would later correctly get
new "power_on" InstanceEvents as a result of AwsCloudAccount.enable.
"""
account = account_helper.generate_cloud_account()
# Set up mocked data in AWS API responses.
region = util_helper.get_random_region()
described_ami = util_helper.generate_dummy_describe_image()
ami_id = described_ami["ImageId"]
all_instances = [
util_helper.generate_dummy_describe_instance(
image_id=ami_id, state=aws.InstanceState.running),
]
described_instances = {region: all_instances}
mock_aws.describe_instances_everywhere.return_value = described_instances
mock_util_aws.describe_images.return_value = [described_ami]
mock_util_aws.is_windows.side_effect = aws.is_windows
mock_util_aws.InstanceState.is_running = aws.InstanceState.is_running
mock_util_aws.AwsArn = aws.AwsArn
mock_util_aws.verify_account_access.return_value = True, []
inspection_call = call(
account.content_object.account_arn,
described_ami["ImageId"],
region,
)
date_of_initial_describe = util_helper.utc_dt(2020, 3, 1, 0, 0, 0)
date_of_disable = util_helper.utc_dt(2020, 3, 2, 0, 0, 0)
date_of_reenable = util_helper.utc_dt(2020, 3, 3, 0, 0, 0)
with util_helper.clouditardis(date_of_initial_describe):
tasks.initial_aws_describe_instances(account.id)
mock_start.assert_has_calls([inspection_call])
with util_helper.clouditardis(date_of_disable):
account.disable()
mock_util_aws.delete_cloudtrail.assert_called()
# Before calling "start_image_inspection" again, let's change the mocked return
# values from AWS to include another instance and image. We should ultimately
# expect the original instance + ami to be found but *not* describe its image.
# Only the new instance + ami should be fully described.
described_ami_2 = util_helper.generate_dummy_describe_image()
ami_id_2 = described_ami_2["ImageId"]
all_instances.append(
util_helper.generate_dummy_describe_instance(
image_id=ami_id_2, state=aws.InstanceState.running))
described_instances = {region: all_instances}
mock_aws.describe_instances_everywhere.return_value = described_instances
mock_util_aws.describe_images.return_value = [
described_ami, described_ami_2
]
inspection_call_2 = call(
account.content_object.account_arn,
described_ami_2["ImageId"],
region,
)
with patch.object(
tasks, "initial_aws_describe_instances"
) as mock_initial, util_helper.clouditardis(date_of_reenable):
# Even though we want to test initial_aws_describe_instances, we need to
# mock this particular call because we need to short-circuit Celery.
account.enable()
mock_initial.delay.assert_called()
with util_helper.clouditardis(date_of_reenable):
tasks.initial_aws_describe_instances(account.id)
mock_start.assert_has_calls([inspection_call_2])
# Now that the dust has settled, let's check that the two instances have events
# in the right configuration. The first instance is on, off, and on; the second
# instance is on.
aws_instance_1_events = (AwsInstance.objects.get(
ec2_instance_id=all_instances[0]["InstanceId"]).instance.get(
).instanceevent_set.order_by("occurred_at").all())
self.assertEqual(len(aws_instance_1_events), 3)
instance_1_event_1 = aws_instance_1_events[0]
self.assertEqual(instance_1_event_1.occurred_at,
date_of_initial_describe)
self.assertEqual(instance_1_event_1.event_type,
InstanceEvent.TYPE.power_on)
instance_1_event_2 = aws_instance_1_events[1]
self.assertEqual(instance_1_event_2.occurred_at, date_of_disable)
self.assertEqual(instance_1_event_2.event_type,
InstanceEvent.TYPE.power_off)
instance_1_event_3 = aws_instance_1_events[2]
self.assertEqual(instance_1_event_3.occurred_at, date_of_reenable)
self.assertEqual(instance_1_event_3.event_type,
InstanceEvent.TYPE.power_on)
aws_instance_2_events = (AwsInstance.objects.get(
ec2_instance_id=all_instances[1]["InstanceId"]).instance.get(
).instanceevent_set.order_by("occurred_at").all())
self.assertEqual(len(aws_instance_2_events), 1)
instance_2_event_1 = aws_instance_2_events[0]
self.assertEqual(instance_2_event_1.occurred_at, date_of_reenable)
self.assertEqual(instance_2_event_1.event_type,
InstanceEvent.TYPE.power_on)
def test_initial_aws_describe_instances_twice(
self,
mock_util_aws,
mock_aws,
mock_start,
mock_notify_sources,
mock_schedule_concurrent_calculation_task,
):
"""
Test calling initial_aws_describe_instances twice with no data changes.
This test asserts appropriate behavior if we call account.enable after
performing the initial AWS account describe once but without generating any new
activity after the initial describe. This is likely to happen for AWS accounts
with any long-running instances that have not changed state since the last time
account.enable was called. Since our account availability checks routinely call
account.enable to verify permissions, we need to test handling this use case.
Why is this a concern? We used to naively *always* insert a new InstanceEvent
for a running instance seen within initial_aws_describe_instances. That's not a
problem in isolation; the new event would result in the runs being recreated and
concurrent usage being recalculated. However, this becomes a problem if the
instance's run spans many days (meaning many more recalculations) or if the
account.enable function (which also calls initial_aws_describe_instances) is
being called frequently, which may be the case since we cannot control when
external callers hit an account's availability_check endpoint.
A resolution to this concern is to perform an existence check before allowing
initial_aws_describe_instances to insert a new InstanceEvent. If the most recent
event is power_on, then we don't need to store another power_on, and we don't
need to rebuild the run and recalculate concurrent usages. The updated expected
behavior of calling account.enable shortly after initial_aws_describe_instances
is that no new InstantEvent objects will be created by account.enable if a
running described instance's most recent event is power_on.
"""
account = account_helper.generate_cloud_account()
# Set up mocked data in AWS API responses.
region = util_helper.get_random_region()
described_ami = util_helper.generate_dummy_describe_image()
ec2_ami_id = described_ami["ImageId"]
described_instance = util_helper.generate_dummy_describe_instance(
image_id=ec2_ami_id, state=aws.InstanceState.running)
ec2_instance_id = described_instance["InstanceId"]
described_instances = {region: [described_instance]}
mock_aws.describe_instances_everywhere.return_value = described_instances
mock_util_aws.describe_images.return_value = [described_ami]
mock_util_aws.is_windows.side_effect = aws.is_windows
mock_util_aws.InstanceState.is_running = aws.InstanceState.is_running
mock_util_aws.AwsArn = aws.AwsArn
mock_util_aws.verify_account_access.return_value = True, []
date_of_initial_describe = util_helper.utc_dt(2020, 3, 1, 0, 0, 0)
date_of_redundant_enable = util_helper.utc_dt(2020, 3, 2, 0, 0, 0)
with util_helper.clouditardis(date_of_initial_describe):
tasks.initial_aws_describe_instances(account.id)
mock_start.assert_called_with(account.content_object.account_arn,
ec2_ami_id, region)
mock_schedule_concurrent_calculation_task.assert_called()
# Reset because we need to check these mocks again later.
mock_schedule_concurrent_calculation_task.reset_mock()
mock_start.reset_mock()
with patch.object(
tasks, "initial_aws_describe_instances"
) as mock_initial, util_helper.clouditardis(date_of_redundant_enable):
# Even though we want to test initial_aws_describe_instances, we need to
# mock this particular call because we need to short-circuit Celery.
account.enable()
mock_initial.delay.assert_called()
mock_notify_sources.delay.assert_called()
with util_helper.clouditardis(date_of_redundant_enable):
tasks.initial_aws_describe_instances(account.id)
# start_image_inspection should not be called because we already know
# about the image from the earlier initial_aws_describe_instances call.
mock_start.assert_not_called()
# schedule_concurrent_calculation_task should not be called because we
# should not have generated any new events here that would require it.
mock_schedule_concurrent_calculation_task.assert_not_called()
# The relevant describe and account.enable processing is now done.
# Now we just need to assert that we did not create redundant power_on events.
instance_events = (AwsInstance.objects.get(
ec2_instance_id=ec2_instance_id).instance.get().instanceevent_set.
order_by("occurred_at"))
self.assertEqual(instance_events.count(), 1)
instance_event = instance_events.first()
self.assertEqual(instance_event.occurred_at, date_of_initial_describe)
self.assertEqual(instance_event.event_type,
InstanceEvent.TYPE.power_on)
return template.render(**data)
def seeded_uuid4():
"""Generate uuid4 using insecure random so we can control its seed."""
return uuid.UUID(bytes=bytes([random.getrandbits(8) for _ in range(16)]),
version=4)
if __name__ == "__main__":
empty_check()
# Reset random seeds for more consistent output.
random.seed(0)
faker.Faker.seed(0)
docs_date = util_helper.utc_dt(2020, 5, 18, 13, 51, 59, 722367)
with transaction.atomic(), override_settings(
SOURCES_ENABLE_DATA_MANAGEMENT_FROM_KAFKA=False), patch.object(
uuid,
"uuid4") as mock_uuid4, util_helper.clouditardis(docs_date):
mock_uuid4.side_effect = seeded_uuid4
api_hander = DocsApiHandler()
public_responses = api_hander.gather_api_responses()
internal_responses = api_hander.gather_internal_api_responses()
transaction.set_rollback(True)
responses = {}
responses.update(public_responses)
responses.update(internal_responses)
output = render(responses)
output = "\n".join((line.rstrip() for line in output.split("\n")))
print(output)
def seeded_uuid4():
"""Generate uuid4 using insecure random so we can control its seed."""
return uuid.UUID(bytes=bytes([random.getrandbits(8) for _ in range(16)]),
version=4)
if __name__ == "__main__":
empty_check()
# Reset random seeds for more consistent output.
random.seed(0)
faker.Faker.seed(0)
docs_date = util_helper.utc_dt(2020, 5, 18, 13, 51, 59, 722367)
with transaction.atomic(), override_settings(
SOURCES_ENABLE_DATA_MANAGEMENT_FROM_KAFKA=False), patch.object(
uuid, "uuid4") as mock_uuid4, patch(
"api.tasks.notify_application_availability_task"
) as mock_notify_sources, util_helper.clouditardis(docs_date):
mock_uuid4.side_effect = seeded_uuid4
api_hander = DocsApiHandler()
public_responses = api_hander.gather_api_responses()
internal_responses = api_hander.gather_internal_api_responses()
transaction.set_rollback(True)
responses = {}
responses.update(public_responses)
responses.update(internal_responses)
output = render(responses)
output = "\n".join((line.rstrip() for line in output.split("\n")))
print(output)