def test_reboot_server_wrong_status(self, server_status):
"""
Attempt to reboot a server while in a status that doesn't allow it
"""
server = OpenStackServerFactory(status=server_status)
with self.assertRaises(WrongStateException):
server.reboot()
def test_sleep_until_state_changes(self, condition, mock_sleep,
mock_update_status):
"""
Check if sleep_until behaves correctly if condition to wait for
is unfulfilled initially.
"""
server = OpenStackServerFactory()
status_queue = [
server._status_to_building,
server._status_to_booting,
server._status_to_ready,
]
status_queue.reverse() # To be able to use pop()
def update_status():
""" Simulate status progression """
status_queue.pop()()
mock_update_status.side_effect = update_status
# Sleep until condition is fulfilled.
# Use a small value for "timeout" to ensure that we can fail quickly
# if server can not reach desired status because transition logic is broken:
server.sleep_until(lambda: getattr(server.status, condition['name']),
timeout=5)
self.assertEqual(server.status, condition['expected_status'])
self.assertEqual(mock_sleep.call_count,
condition['required_transitions'] - 1)
def test_vm_created(self, openstack_id, server_status):
"""
Test that server correctly reports that a VM has been created for it
"""
server = OpenStackServerFactory(openstack_id=openstack_id,
status=server_status)
self.assertTrue(server.vm_created)
def test_log_delete(self, mock_consul):
"""
Check `log_entries` output for combination of instance & server logs
"""
# Clear out existing log entries to make sure we're starting with a clean slate:
for log_entry in LogEntry.objects.all():
log_entry.delete()
server1 = self.server
server2 = OpenStackServerFactory(openstack_id='vm2_id')
self.instance.logger.info('Line #1, on instance')
server1.logger.info('Line #2, on server 1')
server2.logger.info('Line #3, on server 2')
self.assertEqual(LogEntry.objects.count(), 3)
# Delete server 1:
server1_id = server1.pk
server1.delete()
# Now its log entry should be deleted:
entries = LogEntry.objects.order_by('pk').all().values_list('text', flat=True)
for entry_text in entries:
self.assertNotIn('Line #2', entry_text)
self.assertIn('Line #1, on instance', entries[0])
self.assertIn('Line #3, on server 2', entries[1])
self.assertIn(
'Deleted 1 log entries for deleted OpenStack VM instance with ID {}'.format(server1_id),
entries[2]
)
def test_status_transitions(self):
"""
Test that status transitions work as expected for different server workflows
"""
# Normal workflow
server = OpenStackServerFactory()
self.assertEqual(server.status, ServerStatus.Pending)
self._assert_status_conditions(server)
server._status_to_building()
self.assertEqual(server.status, ServerStatus.Building)
self._assert_status_conditions(server)
server._status_to_unknown()
self.assertEqual(server.status, ServerStatus.Unknown)
self._assert_status_conditions(server)
server._status_to_building()
self.assertEqual(server.status, ServerStatus.Building)
self._assert_status_conditions(server)
server._status_to_booting()
self.assertEqual(server.status, ServerStatus.Booting)
self._assert_status_conditions(server, vm_available=True)
server._status_to_unknown()
self.assertEqual(server.status, ServerStatus.Unknown)
self._assert_status_conditions(server)
server._status_to_booting()
self.assertEqual(server.status, ServerStatus.Booting)
self._assert_status_conditions(server, vm_available=True)
server._status_to_ready()
self.assertEqual(server.status, ServerStatus.Ready)
self._assert_status_conditions(server,
is_steady_state=True,
accepts_ssh_commands=True,
vm_available=True)
server._status_to_unknown()
self.assertEqual(server.status, ServerStatus.Unknown)
self._assert_status_conditions(server)
server._status_to_ready()
self.assertEqual(server.status, ServerStatus.Ready)
self._assert_status_conditions(server,
is_steady_state=True,
accepts_ssh_commands=True,
vm_available=True)
server._status_to_terminated()
self.assertEqual(server.status, ServerStatus.Terminated)
self._assert_status_conditions(server, is_steady_state=True)
# Server creation fails
instance_bad_server = BuildingOpenStackServerFactory()
instance_bad_server._status_to_build_failed()
self.assertEqual(instance_bad_server.status, ServerStatus.BuildFailed)
self._assert_status_conditions(server, is_steady_state=True)
def test_get_log_entries(self):
"""
GET - Log entries
"""
self.api_client.login(username='******', password='******')
instance = OpenEdXInstanceFactory(sub_domain='instance0')
server = OpenStackServerFactory(openstack_id="vm0", instance=instance)
instance.logger.info("info")
instance.logger.error("error")
server.logger.info("info")
server.logger.error("error")
response = self.api_client.get('/api/v1/openedxinstance/{pk}/'.format(pk=instance.pk))
self.assertEqual(response.status_code, status.HTTP_200_OK)
expected_list = [
{'level': 'INFO', 'text': 'instance.models.instance | instance=instance0 | info'},
{'level': 'ERROR', 'text': 'instance.models.instance | instance=instance0 | error'},
{'level': 'INFO', 'text': 'instance.models.server | instance=instance0,server=vm0 | info'},
{'level': 'ERROR', 'text': 'instance.models.server | instance=instance0,server=vm0 | error'},
]
self.assertEqual(len(expected_list), len(response.data['log_entries']))
for expected_entry, log_entry in zip(expected_list, response.data['log_entries']):
self.assertEqual(expected_entry['level'], log_entry['level'])
self.assertEqual(expected_entry['text'], log_entry['text'])
def setUp(self):
"""
Set up an instance and server to use for testing.
"""
super().setUp()
self.instance = OpenEdXInstanceFactory(sub_domain='my.instance')
self.server = OpenStackServerFactory(instance=self.instance, openstack_id='vm1_id')
def test_sleep_until_condition_already_fulfilled(self, mock_sleep, mock_update_status):
"""
Check if sleep_until behaves correctly if condition to wait for
is already fulfilled.
"""
conditions = [
lambda: server.status.is_steady_state,
lambda: server.status.accepts_ssh_commands,
]
for condition in conditions:
server = OpenStackServerFactory()
status_queue = [
server._status_to_started,
server._status_to_active,
server._status_to_booted,
]
# Transition to state fulfilling condition
for state_transition in status_queue:
server._transition(state_transition, progress=ServerProgress.Success)
# Sleep until condition is fulfilled.
# Use a small value for "timeout" to ensure that we can fail quickly
# if server can not reach desired status because transition logic is broken:
server.sleep_until(condition, timeout=5)
self.assertEqual(server.status, ServerStatus.Booted)
self.assertEqual(server.progress, ServerProgress.Success)
self.assertEqual(mock_sleep.call_count, 0)
def test_terminate_server_vm_created(self, openstack_id, server_status):
"""
Terminate a server with a VM
"""
server = OpenStackServerFactory(openstack_id=openstack_id, status=server_status)
server.terminate()
self.assertEqual(server.status, ServerStatus.Terminated)
server.os_server.delete.assert_called_once_with()
def test_sleep_until_invalid_timeout(self):
"""
Check if sleep_until behaves correctly when passed an invalid timeout value.
"""
server = OpenStackServerFactory()
for value in (-1, 0):
with self.assertRaises(AssertionError):
server.sleep_until(lambda: server.status.accepts_ssh_commands, timeout=value)
def test_os_server(self, mock_create_server):
"""
Get the os_server attribute of a new server
This should ensure the server is started to be able to return a value
"""
mock_create_server.return_value.id = 'pending-server-id'
server = OpenStackServerFactory()
self.assertEqual(server.os_server, server.nova.servers.get.return_value)
self.assertEqual(server.nova.mock_calls, [call.servers.get('pending-server-id')])
def test_update_status_pending(self, mock_create_server):
"""
Update status while the server is pending
"""
mock_create_server.return_value.id = 'pending-server-id'
server = OpenStackServerFactory()
self.assertEqual(server.status, ServerStatus.Pending)
self.assertIsInstance(server.update_status(), ServerStatus.Pending)
self.assertEqual(server.status, ServerStatus.Pending)
def test_provision_not_ready(self):
"""
POST /:id/provision - Status not ready
"""
self.api_client.login(username='******', password='******')
instance = OpenEdXInstanceFactory()
OpenStackServerFactory(instance=instance, progress=OpenStackServer.Progress.Running)
self.assertEqual(instance.progress, OpenStackServer.Progress.Running)
response = self.api_client.post('/api/v1/openedxinstance/{pk}/provision/'.format(pk=instance.pk))
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_new_server(self):
"""
New OpenStackServer object
"""
self.assertFalse(OpenStackServer.objects.all())
server = OpenStackServerFactory()
self.assertEqual(OpenStackServer.objects.get().pk, server.pk)
self.assertEqual(str(server), 'New OpenStack Server')
self.assertEqual(server.status, ServerStatus.New)
self.assertEqual(server.progress, ServerProgress.Running)
def test_update_status_new(self, mock_create_server):
"""
Update status while the server is new
"""
mock_create_server.return_value.id = 'new-server-id'
server = OpenStackServerFactory(os_server_fixture='openstack/api_server_1_building.json')
self.assertEqual(server.status, ServerStatus.New)
self.assertEqual(server.progress, ServerProgress.Running)
self.assertIsInstance(server.update_status(), ServerStatus.Active)
self.assertEqual(server.progress, ServerProgress.Running)
def test_log_delete_num_queries(self, mock_consul):
"""
Check that the LogEntry.on_post_delete handler doesn't do more queries than necessary.
"""
server = OpenStackServerFactory() # Can't use self.server since deletion of it cascades to self.app_server
with self.assertNumQueries(3):
# We expect one query to check for a related appserver, one to delete the server, one to delete the LogEntry
server.delete()
log_entry = LogEntry.objects.create(text='blah')
with self.assertNumQueries(1):
log_entry.delete()
def test_new_server(self):
"""
New OpenStackServer object
"""
self.assertFalse(OpenStackServer.objects.all())
server = OpenStackServerFactory()
created_server = OpenStackServer.objects.get()
self.assertEqual(created_server.pk, server.pk)
self.assertIsInstance(created_server.status, ServerStatus.Pending)
self.assertEqual(str(server), 'Pending OpenStack Server')
self.assertEqual(server.status, ServerStatus.Pending)
def test_terminate_server_vm_unavailable(self, server_status):
"""
Terminate a server without a VM
"""
server = OpenStackServerFactory(status=server_status)
try:
server.terminate()
except AssertionError:
self.fail('Termination logic tried to operate on non-existent VM.')
else:
self.assertEqual(server.status, ServerStatus.Terminated)
def test_terminate_new_server(self):
"""
Terminate a server with a 'new' status
"""
server = OpenStackServerFactory()
server.terminate()
self.assertEqual(server.status, ServerStatus.Terminated)
self.assertEqual(server.progress, ServerProgress.Success)
server.terminate() # This shouldn't change anything
self.assertEqual(server.status, ServerStatus.Terminated)
self.assertEqual(server.progress, ServerProgress.Success)
self.assertFalse(server.nova.mock_calls)
def test_get_details(self):
"""
GET - Detailed attributes
"""
self.api_client.login(username='******', password='******')
test_openstack_id = 'test-openstack-id'
server = OpenStackServerFactory(openstack_id=test_openstack_id)
response = self.api_client.get('/api/v1/openstackserver/{pk}/'.format(pk=server.id))
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.check_serialized_server(response.data, server)
self.assertEqual(response.data['openstack_id'], test_openstack_id)
def test_start_server_fails(self, mock_create_server):
"""
Check if 'start' behaves correctly when server creation fails
"""
mock_create_server.side_effect = novaclient.exceptions.ClientException(400)
server = OpenStackServerFactory()
self.assertEqual(server.status, ServerStatus.Pending)
server.start()
server = OpenStackServer.objects.get(pk=server.pk)
self.assertEqual(server.status, ServerStatus.BuildFailed)
def test_invalid_status_transitions(self, transition):
"""
Test that invalid status transitions raise exception
"""
# TODO: Get pylint to see state as an iterable
invalid_from_states = (state for state in ServerStatus.states #pylint: disable=not-an-iterable
if state not in transition['from_states'])
for invalid_from_state in invalid_from_states:
instance = OpenStackServerFactory(status=invalid_from_state)
self.assertEqual(instance.status, invalid_from_state)
with self.assertRaises(WrongStateException):
getattr(instance, transition['name'])()
def test_get_superuser(self):
"""
GET - Authenticated access through a superuser account.
"""
self.api_client.login(username='******', password='******')
response = self.api_client.get('/api/v1/openstackserver/')
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data, [])
server = OpenStackServerFactory()
response = self.api_client.get('/api/v1/openstackserver/')
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.check_serialized_server(response.data[0], server)
def test_sleep_until_timeout(self, mock_sleep, mock_update_status):
"""
Check if sleep_until behaves correctly if condition to wait for
is unfulfilled when timeout is reached.
"""
server = OpenStackServerFactory()
def update_status():
""" Simulate status progression """
server._transition(server._status_to_started, progress=ServerProgress.Success)
mock_update_status.side_effect = update_status
with self.assertRaises(TimeoutError):
server.sleep_until(lambda: server.status.accepts_ssh_commands, timeout=1)
self.assertEqual(mock_sleep.call_count, 1)
def test_terminate_server_not_found(self, openstack_id, server_status):
"""
Terminate a server for which the corresponding VM doesn't exist anymore
"""
server = OpenStackServerFactory(openstack_id=openstack_id, status=server_status)
def raise_not_found(): #pylint: disable=missing-docstring
raise novaclient.exceptions.NotFound('not-found')
server.os_server.delete.side_effect = raise_not_found
server.logger = Mock()
mock_logger = server.logger
server.terminate()
self.assertEqual(server.status, ServerStatus.Terminated)
server.os_server.delete.assert_called_once_with()
mock_logger.error.assert_called_once_with(AnyStringMatching('Error while attempting to terminate server'))
def test_sleep_until_timeout(self, mock_sleep, mock_update_status):
"""
Check if sleep_until behaves correctly if condition to wait for
is unfulfilled when timeout is reached.
"""
server = OpenStackServerFactory()
def update_status():
""" Simulate status progression """
server._status_to_building()
mock_update_status.side_effect = update_status
with self.assertRaises(TimeoutError) as timeout_error:
server.sleep_until(lambda: server.status.accepts_ssh_commands, timeout=1)
self.assertEqual(mock_sleep.call_count, 1)
self.assertIn("Waited 0.01", timeout_error.exception)
def test_provision(self, mock_provision_instance, mock_get_commit_id_from_ref):
"""
POST /:id/provision
"""
self.api_client.login(username='******', password='******')
instance = OpenEdXInstanceFactory(commit_id='0' * 40, branch_name='api-branch', fork_name='api/repo')
OpenStackServerFactory(instance=instance,
status=OpenStackServer.Status.Ready,
progress=OpenStackServer.Progress.Success)
mock_get_commit_id_from_ref.return_value = '1' * 40
response = self.api_client.post('/api/v1/openedxinstance/{pk}/provision/'.format(pk=instance.pk))
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data, {'status': 'Instance provisioning started'})
self.assertEqual(mock_provision_instance.call_count, 1)
self.assertEqual(OpenEdXInstance.objects.get(pk=instance.pk).commit_id, '1' * 40)
self.assertEqual(mock_get_commit_id_from_ref.mock_calls, [
call('api/repo', 'api-branch', ref_type='heads'),
])
def test_terminate_server_openstack_api_error(self, exception):
"""
Terminate a server when there are errors connecting to the OpenStack API
"""
server = OpenStackServerFactory()
def raise_openstack_api_error(): #pylint: disable=missing-docstring
raise exception
server.os_server.delete.side_effect = raise_openstack_api_error
server.logger = Mock()
mock_logger = server.logger
server.terminate()
self.assertEqual(server.status, ServerStatus.Unknown)
server.os_server.delete.assert_called_once_with()
mock_logger.error.assert_called_once_with(
AnyStringMatching('Unable to reach the OpenStack API due to'),
exception)
def test_start_server(self, mock_create_server):
"""
Start a new server
"""
mock_create_server.return_value.id = 'pending-server-id'
server = OpenStackServerFactory()
self.assertEqual(server.status, ServerStatus.Pending)
server.start()
mock_create_server.assert_called_once_with(
server.nova,
AnyStringMatching(r'test-inst-\d+'),
flavor_selector=settings.OPENSTACK_SANDBOX_FLAVOR,
image_selector=settings.OPENSTACK_SANDBOX_BASE_IMAGE,
key_name=settings.OPENSTACK_SANDBOX_SSH_KEYNAME,
)
server = OpenStackServer.objects.get(pk=server.pk)
self.assertEqual(server.status, ServerStatus.Building)
self.assertEqual(server.openstack_id, 'pending-server-id')
self.assertEqual(str(server), 'pending-server-id')
def test_sleep_until_steady_state(self, mock_building_is_steady_state, mock_update_status):
"""
Check if sleep_until behaves correctly if condition to wait for
can not be fulfilled because server is in a steady state
(that doesn't fulfill the condition).
"""
server = OpenStackServerFactory()
def update_status():
""" Simulate status progression """
server._status_to_building()
mock_update_status.side_effect = update_status
# Pretend that Status.Building (which doesn't accept SSH commands) is a steady state
mock_building_is_steady_state.return_value = True
with self.assertRaises(SteadyStateException):
# Try to sleep until condition is fulfilled.
# Use a small value for "timeout" to ensure that we can fail quickly
# if server can not reach desired status because transition logic is broken:
server.sleep_until(lambda: server.status.accepts_ssh_commands, timeout=5)
