def test_normalize_instance_on_off(self):
"""Test normalize_runs for one plain instance."""
powered_times = ((
util_helper.utc_dt(2019, 1, 9, 0, 0, 0),
util_helper.utc_dt(2019, 1, 10, 0, 0, 0),
), )
events = api_helper.generate_instance_events(self.instance_plain,
powered_times)
runs = util.normalize_runs(events)
self.assertEquals(len(runs), 1)
run = runs[0]
self.assertEquals(run.start_time, powered_times[0][0])
self.assertEquals(run.end_time, powered_times[0][1])
self.assertEquals(run.instance_id, self.instance_plain.id)
self.assertEquals(run.image_id, self.image_plain.id)
# special flags
self.assertFalse(run.is_cloud_access)
self.assertFalse(run.is_encrypted)
self.assertFalse(run.is_marketplace)
# rhel detection
self.assertFalse(run.rhel)
self.assertFalse(run.rhel_detected)
# openshift detection
self.assertFalse(run.openshift)
self.assertFalse(run.openshift_detected)
def test_generate_aws_events_with_args_and_some_times(self):
"""Assert generation of InstanceEvents with all specified args."""
account = helper.generate_cloud_account()
ec2_ami_id = util_helper.generate_dummy_image_id()
image = helper.generate_image(ec2_ami_id=ec2_ami_id)
instance = helper.generate_instance(account, image=image)
powered_times = (
(None, util_helper.utc_dt(2017, 1, 1)),
(util_helper.utc_dt(2017, 1, 2), util_helper.utc_dt(2017, 1, 3)),
(util_helper.utc_dt(2017, 1, 4), None),
)
instance_type = util_helper.get_random_instance_type()
subnet = str(uuid.uuid4())
events = helper.generate_instance_events(
instance,
powered_times,
instance_type=instance_type,
subnet=subnet,
)
self.assertEqual(len(events), 4)
# We don't care to check *everything* in the events since that should
# already be covered by ``test_generate_events_with_some_times``.
# Here we only care that argument values were set correctly.
for event in events:
if event.event_type != event.TYPE.power_off:
self.assertEqual(
event.instance.machine_image.content_object.ec2_ami_id, ec2_ami_id
)
self.assertEqual(event.content_object.instance_type, instance_type)
self.assertEqual(event.content_object.subnet, subnet)
def test_normalize_one_instance_on_on_off_off(self):
"""
Test normalize_runs one instance with "overlapping" ons and offs.
In this special case, the mock data simulates receiving information
that would indicate two potentially overlapping runs. However, since
we discard subsequent on and off events, we only generate one run.
"""
powered_times = (
(
util_helper.utc_dt(2019, 1, 9, 0, 0, 0),
util_helper.utc_dt(2019, 1, 15, 0, 0, 0),
),
(
util_helper.utc_dt(2019, 1, 10, 0, 0, 0),
util_helper.utc_dt(2019, 1, 19, 0, 0, 0),
),
)
events = api_helper.generate_instance_events(self.instance_plain,
powered_times)
random.shuffle(events)
runs = util.normalize_runs(events)
self.assertEquals(len(runs), 1)
run = runs[0]
self.assertEquals(run.start_time, powered_times[0][0])
self.assertEquals(run.end_time, powered_times[0][1])
self.assertEquals(run.instance_id, self.instance_plain.id)
self.assertEquals(run.image_id, self.image_plain.id)
def test_normalize_one_instance_on_off_off_off(self):
"""
Test normalize_runs one instance with multiple on and one off.
In this special case, only one run should be created. The first off
event is the only one relevant to that run.
"""
powered_times = (
(
util_helper.utc_dt(2019, 1, 9, 0, 0, 0),
util_helper.utc_dt(2019, 1, 10, 0, 0, 0),
),
(None, util_helper.utc_dt(2019, 1, 12, 0, 0, 0)),
(None, util_helper.utc_dt(2019, 1, 14, 0, 0, 0)),
)
events = api_helper.generate_instance_events(self.instance_plain,
powered_times)
random.shuffle(events)
runs = util.normalize_runs(events)
self.assertEquals(len(runs), 1)
run = runs[0]
self.assertEquals(run.start_time, powered_times[0][0])
self.assertEquals(run.end_time, powered_times[0][1])
self.assertEquals(run.instance_id, self.instance_plain.id)
self.assertEquals(run.image_id, self.image_plain.id)
def test_normalize_one_instance_multiple_on_off(self):
"""Test normalize_runs one instance having multiple on-off cycles."""
powered_times = (
(
util_helper.utc_dt(2019, 1, 9, 0, 0, 0),
util_helper.utc_dt(2019, 1, 10, 0, 0, 0),
),
(
util_helper.utc_dt(2019, 1, 11, 0, 0, 0),
util_helper.utc_dt(2019, 1, 12, 0, 0, 0),
),
(util_helper.utc_dt(2019, 1, 13, 0, 0, 0), None),
)
events = api_helper.generate_instance_events(self.instance_plain,
powered_times)
random.shuffle(events)
runs = util.normalize_runs(events)
self.assertEquals(len(runs), len(powered_times))
sorted_runs = sorted(runs, key=lambda r: r.start_time)
for index, run in enumerate(sorted_runs):
self.assertEquals(run.start_time, powered_times[index][0])
self.assertEquals(run.end_time, powered_times[index][1])
self.assertEquals(run.instance_id, self.instance_plain.id)
self.assertEquals(run.image_id, self.image_plain.id)
def test_generate_azure_events_default_and_no_times(self):
"""Assert generation of an azure InstanceEvent."""
account = helper.generate_cloud_account(cloud_type="azure")
instance = helper.generate_instance(account, cloud_type="azure")
events = helper.generate_instance_events(instance,
tuple(),
cloud_type="azure")
self.assertEqual(len(events), 0)
def test_normalize_instance_on_never_off(self):
"""Test normalize_runs for an instance that starts but never stops."""
powered_times = ((util_helper.utc_dt(2019, 1, 9, 0, 0, 0), None), )
events = api_helper.generate_instance_events(self.instance_plain,
powered_times)
runs = util.normalize_runs(events)
self.assertEquals(len(runs), 1)
run = runs[0]
self.assertEquals(run.start_time, powered_times[0][0])
self.assertEquals(run.end_time, powered_times[0][1])
self.assertEquals(run.instance_id, self.instance_plain.id)
self.assertEquals(run.image_id, self.image_plain.id)
def test_normalize_multiple_instances_on_off(self):
"""Test normalize_runs for events for multiple instances and images."""
rhel_powered_times = ((
util_helper.utc_dt(2019, 1, 9, 0, 0, 0),
util_helper.utc_dt(2019, 1, 10, 0, 0, 0),
), )
ocp_powered_times = ((
util_helper.utc_dt(2019, 1, 8, 0, 0, 0),
util_helper.utc_dt(2019, 1, 11, 0, 0, 0),
), )
rhel_events = api_helper.generate_instance_events(
self.instance_rhel, rhel_powered_times)
ocp_events = api_helper.generate_instance_events(
self.instance_ocp, ocp_powered_times)
# force some slight out-of-order shuffling of incoming events.
events = rhel_events[:-1] + ocp_events[::-1] + rhel_events[-1:]
random.shuffle(events)
runs = util.normalize_runs(events)
self.assertEquals(len(runs), 2)
rhel_run = runs[0] if runs[0].rhel else runs[1]
ocp_run = runs[0] if runs[0].openshift else runs[1]
self.assertTrue(rhel_run.rhel)
self.assertFalse(rhel_run.openshift)
self.assertEquals(rhel_run.start_time, rhel_powered_times[0][0])
self.assertEquals(rhel_run.end_time, rhel_powered_times[0][1])
self.assertEquals(rhel_run.instance_id, self.instance_rhel.id)
self.assertEquals(rhel_run.image_id, self.image_rhel.id)
self.assertFalse(ocp_run.rhel)
self.assertTrue(ocp_run.openshift)
self.assertEquals(ocp_run.start_time, ocp_powered_times[0][0])
self.assertEquals(ocp_run.end_time, ocp_powered_times[0][1])
self.assertEquals(ocp_run.instance_id, self.instance_ocp.id)
self.assertEquals(ocp_run.image_id, self.image_ocp.id)
def generate_events(self, powered_times, instance, image):
"""
Generate events saved to the DB and returned.
Args:
powered_times (list[tuple]): Time periods instance is powered on.
instance (Instance): which instance has the events.
image (MachineImage): which image seen in the events.
Returns:
list[InstanceEvent]: The list of events
"""
events = api_helper.generate_instance_events(
instance,
powered_times,
image.content_object.ec2_ami_id,
)
return events
def test_generate_aws_events_with_some_times(self):
"""Assert generation of InstanceEvents with some times."""
account = helper.generate_cloud_account()
instance = helper.generate_instance(account)
powered_times = (
(None, util_helper.utc_dt(2017, 1, 1)),
(util_helper.utc_dt(2017, 1, 2), util_helper.utc_dt(2017, 1, 3)),
(util_helper.utc_dt(2017, 1, 4), None),
)
events = helper.generate_instance_events(instance, powered_times)
self.assertEqual(len(events), 4)
self.assertEqual(events[0].occurred_at, powered_times[0][1])
self.assertEqual(events[1].occurred_at, powered_times[1][0])
self.assertEqual(events[2].occurred_at, powered_times[1][1])
self.assertEqual(events[3].occurred_at, powered_times[2][0])
self.assertEqual(events[0].event_type, InstanceEvent.TYPE.power_off)
self.assertEqual(events[1].event_type, InstanceEvent.TYPE.power_on)
self.assertEqual(events[2].event_type, InstanceEvent.TYPE.power_off)
self.assertEqual(events[3].event_type, InstanceEvent.TYPE.power_on)
self.assertIsNotNone(events[0].content_object.instance_type)
self.assertIsNotNone(events[0].content_object.subnet)
# Assert they all have the same AMI, subnet, and instance type.
for event in events[1:]:
self.assertEqual(
event.content_object.instance_type,
events[0].content_object.instance_type,
)
self.assertEqual(
event.content_object.subnet, events[0].content_object.subnet
)
if event.event_type != event.TYPE.power_off:
# power_off events typically don't have an image defined.
# the second event should be power_on and should have an
# image defined we can compare with.
self.assertEqual(
event.instance.machine_image, events[1].instance.machine_image
)
def test_generate_aws_events_default_and_no_times(self):
"""Assert generation of InstanceEvents with minimal args."""
account = helper.generate_cloud_account()
instance = helper.generate_instance(account)
events = helper.generate_instance_events(instance, tuple())
self.assertEqual(len(events), 0)
def __init__(self):
"""Initialize all the data for the examples."""
api_helper.generate_instance_type_definitions(cloud_type="aws")
api_helper.generate_instance_type_definitions(cloud_type="azure")
self.customer_account_number = "100001"
self.customer_user = util_helper.get_test_user(
self.customer_account_number, is_superuser=False)
self.customer_user.date_joined = util_helper.utc_dt(
2019, 1, 1, 0, 0, 0)
self.customer_user.save()
self.customer_client = api_helper.SandboxedRestClient()
self.customer_client._force_authenticate(self.customer_user)
self.internal_client = api_helper.SandboxedRestClient(
api_root="/internal/api/cloudigrade/v1")
self.internal_client._force_authenticate(self.customer_user)
self.customer_arn = util_helper.generate_dummy_arn()
# Times to use for various account and event activity.
self.now = get_now()
self.this_morning = self.now.replace(hour=0,
minute=0,
second=0,
microsecond=0)
self.yesterday = self.this_morning - timedelta(days=1)
self.last_month = self.this_morning - timedelta(days=31)
self.last_week = self.this_morning - timedelta(days=7)
self.three_days_ago = self.this_morning - timedelta(days=3)
self.two_days_ago = self.this_morning - timedelta(days=2)
self.two_weeks_ago = self.this_morning - timedelta(weeks=2)
self.tomorrow = self.this_morning + timedelta(days=1)
self.next_week = self.this_morning + timedelta(weeks=1)
######################################
# Generate AWS data for the customer user.
self.aws_customer_account = api_helper.generate_cloud_account(
arn=util_helper.generate_dummy_arn(),
user=self.customer_user,
name="greatest account ever",
created_at=self.two_weeks_ago,
)
self.azure_customer_account = api_helper.generate_cloud_account(
user=self.customer_user,
name="meh account",
created_at=self.two_weeks_ago,
cloud_type="azure",
azure_subscription_id=str(seeded_uuid4()),
azure_tenant_id=str(seeded_uuid4()),
)
self.customer_instances = [
api_helper.generate_instance(self.aws_customer_account),
api_helper.generate_instance(self.aws_customer_account),
api_helper.generate_instance(self.aws_customer_account),
api_helper.generate_instance(self.azure_customer_account,
cloud_type="azure"),
api_helper.generate_instance(self.azure_customer_account,
cloud_type="azure"),
api_helper.generate_instance(self.azure_customer_account,
cloud_type="azure"),
]
# Generate events so we can see customer activity in the responses.
# These events represent all customer instances starting one week ago,
# stopping two days ago, and starting again yesterday.
self.events = []
for instance in self.customer_instances[:2]:
self.events.extend(
api_helper.generate_instance_events(
instance,
[
(self.last_week, self.three_days_ago),
(self.yesterday, None),
],
))
for instance in self.customer_instances[3:6]:
self.events.extend(
api_helper.generate_instance_events(
instance,
[
(self.last_week, self.three_days_ago),
(self.yesterday, None),
],
cloud_type="azure",
))
# Build the runs for the created events.
# Note: this crude and *direct* implementation of Run-saving should be
# replaced as we continue porting pilot functionality and (eventually)
# better general-purpose Run-handling functions materialize.
normalized_runs = normalize_runs(models.InstanceEvent.objects.all())
for normalized_run in normalized_runs:
run = models.Run(
start_time=normalized_run.start_time,
end_time=normalized_run.end_time,
machineimage_id=normalized_run.image_id,
instance_id=normalized_run.instance_id,
instance_type=normalized_run.instance_type,
memory=normalized_run.instance_memory,
vcpu=normalized_run.instance_vcpu,
)
run.save()
# Force all images to have RHEL detected ("7.7")
self.images = list(
set(instance.machine_image for instance in self.customer_instances
if instance.machine_image is not None))
for image in self.images:
image.inspection_json = json.dumps({
"rhel_enabled_repos_found": True,
"rhel_version": "7.7",
"syspurpose": {
"role": "Red Hat Enterprise Linux Server",
"service_level_agreement": "Premium",
"usage": "Development/Test",
},
})
image.status = image.INSPECTED
image.region = "us-east-1"
image.save()
# Pre-calculate concurrent usage data for upcoming requests.
# Calculate each day since "last week" (oldest date we use in example requests).
the_date = self.last_week.date()
one_day_delta = timedelta(days=1)
# while the_date <= self.this_morning.date():
while the_date <= self.next_week.date():
task_id = f"calculate-concurrent-usage-{seeded_uuid4()}"
models.ConcurrentUsageCalculationTask.objects.create(
user_id=self.customer_user.id,
date=the_date.isoformat(),
task_id=task_id,
status=models.ConcurrentUsageCalculationTask.COMPLETE,
)
calculate_max_concurrent_usage(the_date, self.customer_user.id)
the_date = the_date + one_day_delta
def test_process_instance_event_duplicate_start(
self,
mock_calculate_concurrent_usage_task,
mock_get_last_scheduled_concurrent_usage_calculation_task,
):
"""
Test that recalculate works when a duplicate start event is introduced.
Initial Runs (5,7):
[ ### ]
New power on event at (1,) results in the run being updated:
[####### ]
New power on event at (3,) results in the run not being updated:
[####### ]
"""
instance_type = "t1.potato"
instance = api_helper.generate_instance(self.account)
run_time = [(
util_helper.utc_dt(2018, 1, 5, 0, 0, 0),
util_helper.utc_dt(2018, 1, 7, 0, 0, 0),
)]
instance_events = api_helper.generate_instance_events(
instance, run_time, instance_type=instance_type)
api_helper.recalculate_runs_from_events(instance_events)
first_start = util_helper.utc_dt(2018, 1, 1, 0, 0, 0)
instance_event = api_helper.generate_single_instance_event(
instance=instance,
occurred_at=first_start,
event_type=InstanceEvent.TYPE.power_on,
instance_type=instance_type,
)
tasks.process_instance_event(instance_event)
runs = list(Run.objects.all())
self.assertEqual(1, len(runs))
self.assertEqual(run_time[0][1], runs[0].end_time)
self.assertEqual(first_start, runs[0].start_time)
second_start = util_helper.utc_dt(2018, 1, 3, 0, 0, 0)
duplicate_start_event = api_helper.generate_single_instance_event(
instance=instance,
occurred_at=second_start,
event_type=InstanceEvent.TYPE.power_on,
instance_type=instance_type,
)
tasks.process_instance_event(duplicate_start_event)
runs = list(Run.objects.all())
self.assertEqual(1, len(runs))
self.assertEqual(run_time[0][1], runs[0].end_time)
self.assertEqual(first_start, runs[0].start_time)
