def test_launch_in_other_region(self, mock_get_nova_client):
"""
Test launching an appserver in a non-default region.
"""
instance = OpenEdXInstanceFactory(openstack_region="elsewhere")
make_test_appserver(instance)
mock_get_nova_client.assert_called_once_with("elsewhere")
def create_user_with_trial_instance(self):
"""
Returns a trial instance for a user
:param key: String to append to properties.
:return: A trial instance
"""
user, _ = get_user_model().objects.get_or_create(
username=str(self.instance_counter) + 'test',
email=str(self.instance_counter) + '*****@*****.**')
instance = OpenEdXInstanceFactory(
successfully_provisioned=True,
betatestapplication=BetaTestApplication())
make_test_appserver(instance=instance, is_active=True)
BetaTestApplication.objects.create(
user=user,
public_contact_email=str(self.instance_counter) +
'*****@*****.**',
subdomain=str(self.instance_counter) + 'betatestdomain',
instance_name=instance.name,
status=BetaTestApplication.ACCEPTED,
instance=instance)
self.instance_counter += 1
return instance
def test_get_servers_from_different_org(self):
"""
GET - A non-superuser instance manager from organization 1 can't find servers from organization 2
(that is, servers belonging to instances owned by organization 2).
"""
self.api_client.login(username='******', password='******')
# Instance 1 belongs to user4's organization (which is organization2)
instance1 = OpenEdXInstanceFactory()
instance1.ref.creator = self.user4.profile
instance1.ref.owner = self.organization2
instance1.save()
app_server_i1 = make_test_appserver(instance=instance1)
# Instance 2 doesn't belong to user4's organization (organization2). It was created by another user (user1)
instance2 = OpenEdXInstanceFactory()
instance2.ref.creator = self.user1.profile
instance2.ref.owner = self.organization
instance2.save()
app_server_i2 = make_test_appserver(instance=instance2)
# Only the first server should be listed
response = self.api_client.get('/api/v1/openedx_appserver/')
data_entries = response.data[0].items()
self.assertIn(('id', app_server_i1.pk), data_entries)
self.assertNotIn(('id', app_server_i2.pk), data_entries)
# Only the first server should be directly accessible
response = self.api_client.get(
'/api/v1/openedx_appserver/{pk}/'.format(pk=app_server_i1.pk))
self.assertEqual(response.status_code, status.HTTP_200_OK)
response = self.api_client.get(
'/api/v1/openedx_appserver/{pk}/'.format(pk=app_server_i2.pk))
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
def test_secret_key_settings_no_key(self):
"""
Test that secret key settings are empty if the master key is not set.
"""
instance = OpenEdXInstanceFactory()
make_test_appserver(instance)
instance.secret_key_b64encoded = ''
instance.save()
self.assertEqual(instance.get_secret_key_settings(), '')
def test_get_instances_to_upgrade(self):
"""
Test get_instances_to_upgrade with real instances.
"""
# Initalize the expected_instances list with create three instances
expected_instances = []
for iidx in range(3):
instance = OpenEdXInstanceFactory(
openedx_release=DummyInstanceUpgrade.INITIAL_RELEASE,
use_ephemeral_databases=False)
# Create iidx appservers, and set to Running
for aidx in range(iidx + 1):
app_server = make_test_appserver(instance)
app_server._status_to_waiting_for_server()
app_server._status_to_configuring_server()
app_server._status_to_running()
# Activate every other appserver (0, 2, ..)
app_server.is_active = not bool(aidx % 2)
app_server.save()
# The instance is expected in the list
expected_instances.append(instance)
# Create a third instance using an incorrect release, but an active, running appserver
instance = OpenEdXInstanceFactory(openedx_release="not-this-release",
use_ephemeral_databases=False)
# Create iidx appservers, and set to Running
app_server = make_test_appserver(instance)
app_server._status_to_waiting_for_server()
app_server._status_to_configuring_server()
app_server._status_to_running()
app_server.is_active = True
app_server.save()
# Create a fourth instance with no active appservers
instance = OpenEdXInstanceFactory(
openedx_release=DummyInstanceUpgrade.INITIAL_RELEASE,
use_ephemeral_databases=False)
# Create iidx appservers, and set to Running
app_server = make_test_appserver(instance)
app_server._status_to_waiting_for_server()
app_server._status_to_configuring_server()
app_server._status_to_running()
app_server.is_active = False
app_server.save()
# Create a fifth instance with appservers at all
instance = OpenEdXInstanceFactory(
openedx_release=DummyInstanceUpgrade.INITIAL_RELEASE,
use_ephemeral_databases=False)
actual_instances = self.upgrader.get_instances_to_upgrade()
self.assertEqual(list(actual_instances.order_by('pk')),
expected_instances)
def test_check_security_groups(self, mock_sync_security_group_rules,
mock_get_openstack_connection, mock_consul):
"""
Test that check_security_groups() can create and synchronize security groups
"""
# We simulate the existence of these network security groups on the OpenStack cloud:
existing_groups = ["group_a", "group_b"]
new_security_group = Mock()
def mocked_find_security_group(name_or_id):
""" Mock openstack network.find_security_group """
if name_or_id in existing_groups:
result = Mock()
result.name = name_or_id
return result
else:
return None
def mocked_create_security_group(**args):
""" Mock openstack network.create_security_group """
new_security_group.__dict__.update(**args)
return new_security_group
network = mock_get_openstack_connection().network
network.find_security_group.side_effect = mocked_find_security_group
network.create_security_group.side_effect = mocked_create_security_group
instance = OpenEdXInstanceFactory(
additional_security_groups=["group_a", "group_b"])
app_server = make_test_appserver(instance)
# Call check_security_groups():
app_server.check_security_groups()
# the default group doesn't exist, so we expect it was created:
network.create_security_group.assert_called_once_with(
name=settings.OPENEDX_APPSERVER_SECURITY_GROUP_NAME)
# we also expect that its description was set:
expected_description = "Security group for Open EdX AppServers. Managed automatically by OpenCraft IM."
network.update_security_group.assert_called_once_with(
new_security_group, description=expected_description)
# We expect that the group was synced with the configured rules:
mock_sync_security_group_rules.assert_called_once_with(
new_security_group,
OPENEDX_APPSERVER_SECURITY_GROUP_RULES,
network=network)
# Now, if we change the additional groups, we expect to get an exception:
instance.additional_security_groups = ["invalid"]
instance.save()
app_server = make_test_appserver(instance)
with self.assertRaisesRegex(
Exception,
"Unable to find the OpenStack network security group called 'invalid'."
):
app_server.check_security_groups()
def test_commit_changes_fail_running_appserver(self, mock_consul):
"""
Test that committing changes fails when a user is new.
"""
self.client.force_login(self.user_with_instance)
instance = self._setup_user_instanace()
make_test_appserver(instance, status=Status.ConfiguringServer)
response = self.client.post(
reverse('api:v2:openedx-instance-config-commit-changes',
args=(self.instance_config.pk, )))
self.assertEqual(response.status_code, 400)
self.assertIn("Instance launch already in progress",
response.content.decode('utf-8'))
def test_commit_changes_force_running_appserver(self,
mock_create_new_deployment,
mock_consul):
"""
Test that committing changes fails when a user is new.
"""
self.client.force_login(self.user_with_instance)
instance = self._setup_user_instance()
make_test_appserver(instance, status=Status.ConfiguringServer)
url = reverse('api:v2:openedx-instance-deployment-list')
response = self.client.post(f"{url}?force=true",
data={"id": self.instance_config.id})
self.assertEqual(response.status_code, 200)
mock_create_new_deployment.assert_called()
def test_get_instance_charges(self, charges_details_mock, mock_consul):
"""
Test that an instance is going to generate charges for all of its
terminated and running appservers only and make sure that servers
from other instances are not included in the subtotals, total, etc.
"""
desired_instance = OpenEdXInstanceFactory()
make_test_appserver(desired_instance, status=AppServerStatus.Running)
make_test_appserver(desired_instance,
status=AppServerStatus.Terminated)
make_test_appserver(desired_instance,
status=AppServerStatus.ConfigurationFailed)
another_instance = OpenEdXInstanceFactory()
make_test_appserver(instance=another_instance)
invoice_month = self._generate_invoice_date()
expected_charges_details = {
'name': 'Mock AppServer charge',
'billing_start': invoice_month,
'billing_end': invoice_month,
'days': 20,
'charge': 12,
}
charges_details_mock.return_value = expected_charges_details
appservers_charges, appservers_total = get_instance_charges(
desired_instance, invoice_month)
self.assertEqual(len(appservers_charges), 2)
for charge in appservers_charges:
self.assertEqual(charge, expected_charges_details)
self.assertEqual(appservers_total,
2 * expected_charges_details['charge'])
def test_commit_changes_force_running_appserver(self, mock_spawn_appserver,
mock_consul):
"""
Test that committing changes fails when a user is new.
"""
self.client.force_login(self.user_with_instance)
instance = self._setup_user_instanace()
make_test_appserver(instance, status=Status.ConfiguringServer)
url = reverse(
'api:v2:openedx-instance-config-commit-changes',
args=(self.instance_config.pk, ),
)
response = self.client.post(f"{url}?force=true")
self.assertEqual(response.status_code, 200)
mock_spawn_appserver.assert_called()
def test_ansible_settings_swift(self):
"""
Verify Swift Ansible configuration when Swift is enabled.
"""
instance = OpenEdXInstanceFactory(use_ephemeral_databases=False)
appserver = make_test_appserver(instance)
self.check_ansible_settings(appserver)
def test_run_playbook_logging(self, mock_inventory_str, mock_run_playbook):
"""
Ensure logging routines are working on _run_playbook method
"""
stdout_r, stdout_w = os.pipe()
stderr_r, stderr_w = os.pipe()
with open(stdout_r,
'rb', buffering=0) as stdout, open(stderr_r,
'rb',
buffering=0) as stderr:
mock_run_playbook.return_value.__enter__.return_value.stdout = stdout
mock_run_playbook.return_value.__enter__.return_value.stderr = stderr
mock_run_playbook.return_value.__enter__.return_value.returncode = 0
os.write(stdout_w, b'Hello\n')
os.close(stdout_w)
os.write(stderr_w, b'Hi\n')
os.close(stderr_w)
appserver = make_test_appserver()
playbook = Playbook(source_repo='dummy',
playbook_path='dummy',
requirements_path='dummy',
version='dummy',
variables='dummy')
log, returncode = appserver._run_playbook("/tmp/test/working/dir/",
playbook)
self.assertCountEqual(log, ['Hello', 'Hi'])
self.assertEqual(returncode, 0)
def test_ansible_settings_mongo_ephemeral(self):
"""
Don't add mysql ansible vars for ephemeral databases
"""
self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=True)
appserver = make_test_appserver(self.instance)
self.check_mongo_vars_not_set(appserver)
def test_provisioning(self, playbook_returncode, mock_open_repo,
mock_inventory, mock_run_playbook,
mock_poll_streams):
"""The appserver gets provisioned with the appropriate playbooks. Failure causes later playbooks to not run."""
appserver = make_test_appserver()
working_dir = '/cloned/configuration-repo/path'
mock_open_repo.return_value.__enter__.return_value.working_dir = working_dir
mock_run_playbook.return_value.__enter__.return_value.returncode = playbook_returncode
appserver.run_ansible_playbooks()
self.assertIn(
call(
requirements_path='{}/requirements.txt'.format(working_dir),
inventory_str=mock_inventory,
vars_str=appserver.configuration_settings,
playbook_path='{}/playbooks'.format(working_dir),
playbook_name='edx_sandbox.yml',
username='******',
), mock_run_playbook.mock_calls)
assert_func = self.assertIn if playbook_returncode == 0 else self.assertNotIn
assert_func(
call(
requirements_path='{}/requirements.txt'.format(working_dir),
inventory_str=mock_inventory,
vars_str=appserver.create_common_configuration_settings(),
playbook_path='{}/playbooks'.format(working_dir),
playbook_name='appserver.yml',
username='******',
), mock_run_playbook.mock_calls)
def test_ansible_settings_mongo_ephemeral(self):
"""
Don't add mysql ansible vars for ephemeral databases
"""
self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=True)
appserver = make_test_appserver(self.instance)
self.check_mongo_vars_not_set(appserver)
def test_ansible_settings_swift_ephemeral(self):
"""
Verify Swift Ansible configuration is not included when using ephemeral databases.
"""
instance = OpenEdXInstanceFactory(use_ephemeral_databases=True)
appserver = make_test_appserver(instance)
self.check_ansible_settings(appserver, expected=False)
def test_ansible_settings_swift(self):
"""
Verify Swift Ansible configuration when Swift is enabled.
"""
instance = OpenEdXInstanceFactory(use_ephemeral_databases=False)
appserver = make_test_appserver(instance)
self.check_ansible_settings(appserver)
def test_one_attempt_default_fail(self, task_function, mock_provision,
mock_spawn, mock_consul):
"""
Test that by default, the spawn_appserver task will not re-try provisioning, even when failing.
"""
instance = OpenEdXInstanceFactory()
# Disable mocking of retry-enabled spawn_appserver
self.spawn_appserver_patcher.stop()
self.addCleanup(self.spawn_appserver_patcher.start)
# Mock successful provisioning
mock_provision.return_value = False
mock_spawn.return_value = make_test_appserver(instance)
self.call_task_function(task_function, instance)
# Check mocked functions call count
self.assertEqual(mock_spawn.call_count, 1)
self.assertEqual(mock_provision.call_count, 1)
# Confirm logs
self.assertTrue(
any("Spawning new AppServer, attempt 1 of 1" in log.text
for log in instance.log_entries))
def setUp(self):
"""
Set up an instance and server to use for testing.
"""
super().setUp()
with patch(
'instance.tests.models.factories.openedx_instance.OpenEdXInstance._write_metadata_to_consul',
return_value=(1, True)
):
self.instance = OpenEdXInstanceFactory(sub_domain='my.instance', name="Test Instance 1")
self.app_server = make_test_appserver(instance=self.instance)
self.server = self.app_server.server
# Override the VM names for consistency:
patcher = patch('instance.models.server.OpenStackServer.name', new='test-vm-name')
self.addCleanup(patcher.stop)
patcher.start()
# Expected log line prefixes based on the above:
self.instance_prefix = 'instance.models.instance | instance={} (Test Instance 1) | '.format(
self.instance.ref.pk
)
self.appserver_prefix = (
'instance.models.appserver | instance={} (Test Instance 1),app_server={} (AppServer 1) | '.format(
self.instance.ref.pk, self.app_server.pk
)
)
self.server_prefix = 'instance.models.server | server={} (test-vm-name) | '.format(self.app_server.server.pk)
def test_postfix_queue_settings_present(self):
"""
Check that ansible vars for postfix_queue role are set correctly.
"""
instance = OpenEdXInstanceFactory(
sub_domain='test.postfix.queue',
email='*****@*****.**',
external_lms_domain='lms.myinstance.org')
appserver = make_test_appserver(instance)
configuration_vars = yaml.load(appserver.configuration_settings)
self.assertEqual(
configuration_vars['POSTFIX_QUEUE_EXTERNAL_SMTP_HOST'],
'smtp.myhost.com')
self.assertEqual(
configuration_vars['POSTFIX_QUEUE_EXTERNAL_SMTP_PORT'], '2525')
self.assertEqual(
configuration_vars['POSTFIX_QUEUE_EXTERNAL_SMTP_USER'], 'smtpuser')
self.assertEqual(
configuration_vars['POSTFIX_QUEUE_EXTERNAL_SMTP_PASSWORD'],
'smtppass')
self.assertEqual(configuration_vars['POSTFIX_QUEUE_HEADER_CHECKS'],
'/^From:(.*)$/ PREPEND Reply-To:$1')
self.assertEqual(
configuration_vars['POSTFIX_QUEUE_SENDER_CANONICAL_MAPS'],
'@myinstance.org [email protected]')
def test_num_attempts_successful(self, mock_provision, mock_spawn):
"""
Test that if num_attempts > 1, the spawn_appserver task will stop trying to provision
after a successful attempt.
"""
instance = OpenEdXInstanceFactory()
# Disable mocking of retry-enabled spawn_appserver
self.spawn_appserver_patcher.stop()
self.addCleanup(self.spawn_appserver_patcher.start)
# Mock successful provisioning
mock_provision.return_value = True
mock_spawn.return_value = make_test_appserver(instance)
tasks.spawn_appserver(instance.ref.pk,
num_attempts=3,
mark_active_on_success=True)
# Check mocked functions call count
self.assertEqual(mock_spawn.call_count, 1)
self.assertEqual(mock_provision.call_count, 1)
self.assertEqual(self.mock_make_appserver_active.call_count, 1)
# Confirm logs
self.assertTrue(
any("Spawning new AppServer, attempt 1 of 3" in log.text
for log in instance.log_entries))
self.assertFalse(
any("Spawning new AppServer, attempt 2 of 3" in log.text
for log in instance.log_entries))
self.assertFalse(
any("Spawning new AppServer, attempt 3 of 3" in log.text
for log in instance.log_entries))
def test_get_authenticated(self):
"""
GET - Authenticated - instance manager users allowed access
"""
self.api_client.login(username='******', password='******')
response = self.api_client.get('/api/v1/openedx_appserver/')
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data, [])
app_server = make_test_appserver()
response = self.api_client.get('/api/v1/openedx_appserver/')
self.assertEqual(response.status_code, status.HTTP_200_OK)
data = response.data[0].items()
self.assertIn(('id', app_server.pk), data)
self.assertIn(('api_url', 'http://testserver/api/v1/openedx_appserver/{pk}/'.format(pk=app_server.pk)), data)
self.assertIn(('name', 'AppServer 1'), data)
# Status fields:
self.assertIn(('status', 'new'), data)
self.assertIn(('status_name', 'New'), data)
self.assertIn(('status_description', 'Newly created'), data)
self.assertIn(('is_steady', True), data)
self.assertIn(('is_healthy', True), data)
# Created/modified date:
self.assertIn('created', response.data[0])
self.assertIn('modified', response.data[0])
# Other details should not be in the list view:
self.assertNotIn('instance', response.data[0])
self.assertNotIn('server', response.data[0])
self.assertNotIn('configuration_settings', response.data[0])
self.assertNotIn('edx_platform_commit', response.data[0])
self.assertNotIn('log_entries', response.data[0])
self.assertNotIn('log_error_entries', response.data[0])
def test_cannot_reprovision(self, mocks):
"""
Once an AppServer's provision() method has been called once, it cannot be called ever
again. Instead, a new AppServer must be created.
"""
app_server = make_test_appserver()
self.assertEqual(app_server.status, AppServerStatus.New)
app_server.provision()
self.assertEqual(app_server.status, AppServerStatus.Running)
with self.assertRaises(WrongStateException):
app_server.provision()
# Double-check for various states other than New:
invalid_from_states = (
AppServerStatus.WaitingForServer,
AppServerStatus.ConfiguringServer,
AppServerStatus.Error,
AppServerStatus.ConfigurationFailed,
AppServerStatus.Running,
AppServerStatus.Terminated
)
for invalid_from_state in invalid_from_states:
# Hack to force the app server into a specific state:
OpenEdXAppServer.objects.filter(pk=app_server.pk).update(_status=invalid_from_state.state_id)
app_server = OpenEdXAppServer.objects.get(pk=app_server.pk)
with self.assertRaises(WrongStateException):
app_server.provision()
def test_cannot_reprovision(self, mocks):
"""
Once an AppServer's provision() method has been called once, it cannot be called ever
again. Instead, a new AppServer must be created.
"""
app_server = make_test_appserver()
self.assertEqual(app_server.status, AppServerStatus.New)
app_server.provision()
self.assertEqual(app_server.status, AppServerStatus.Running)
with self.assertRaises(WrongStateException):
app_server.provision()
# Double-check for various states other than New:
invalid_from_states = (AppServerStatus.WaitingForServer,
AppServerStatus.ConfiguringServer,
AppServerStatus.Error,
AppServerStatus.ConfigurationFailed,
AppServerStatus.Running,
AppServerStatus.Terminated)
for invalid_from_state in invalid_from_states:
# Hack to force the app server into a specific state:
OpenEdXAppServer.objects.filter(pk=app_server.pk).update(
_status=invalid_from_state.state_id)
app_server = OpenEdXAppServer.objects.get(pk=app_server.pk)
with self.assertRaises(WrongStateException):
app_server.provision()
def test_hearbeat_active_succeeds(self):
""" Test that heartbeat_active method returns true when request to hearbeat is 200"""
appserver = make_test_appserver()
responses.add(responses.OPTIONS,
'http://{}/heartbeat'.format(appserver.server.public_ip),
status=200)
self.assertTrue(appserver.heartbeat_active())
def test_hero_cover_image_set(self):
"""
Test that when the hero cover image is set, the corresponding ansible variables are generated.
"""
OpenEdXInstanceFactory(name='Test hero cover image',
deploy_simpletheme=True,
theme_config={
'version': 1,
'link-color': '#123456'
})
instance = OpenEdXInstance.objects.get()
user = get_user_model().objects.create_user('betatestuser',
'*****@*****.**')
application = self.make_test_application(instance, user)
application.hero_cover_image = 'hero_cover.png'
application.save()
appserver = make_test_appserver(instance)
ansible_theme_vars = instance.get_theme_settings()
ansible_vars = appserver.configuration_settings
for variables in (ansible_theme_vars, ansible_vars):
parsed_vars = yaml.load(variables, Loader=yaml.SafeLoader) or {}
self.assertIn('SIMPLETHEME_STATIC_FILES_URLS', parsed_vars)
self.assertTrue(
any(item['dest'] == 'lms/static/images/hero_cover.png' for item
in parsed_vars['SIMPLETHEME_STATIC_FILES_URLS']), )
self.assertIn('SIMPLETHEME_SASS_OVERRIDES', parsed_vars)
self.assertTrue(
any(item['variable'] == 'homepage-bg-image'
and item['value'] == 'url("../images/hero_cover.png")'
for item in parsed_vars['SIMPLETHEME_SASS_OVERRIDES']), )
def test_get_playbooks(self, mock_consul):
"""
Verify correct list of playbooks is provided for spawning an appserver.
"""
instance = OpenEdXInstanceFactory()
user = get_user_model().objects.create_user(username='******',
email='*****@*****.**')
instance.lms_users.add(user)
appserver = make_test_appserver(instance)
# By default there should be four playbooks:
# - OpenEdX provisioning playbook,
# - LMS users playbook,
# - enable bulk emails playbook,
# - and the OCIM service ansible playbook.
playbooks = appserver.get_playbooks()
self.assertEqual(len(playbooks), 4)
self.assertEqual(playbooks[0], appserver.default_playbook())
self.assertEqual(playbooks[1], appserver.lms_user_creation_playbook())
self.assertEqual(playbooks[2], appserver.enable_bulk_emails_playbook())
self.assertEqual(playbooks[3].source_repo,
settings.ANSIBLE_APPSERVER_REPO)
self.assertEqual(playbooks[3].playbook_path,
settings.ANSIBLE_APPSERVER_PLAYBOOK)
# Once the instance has been successfully provisioned, the "enable bulk emails" playbooks is no longer run.
instance.successfully_provisioned = True
instance.save()
playbooks = appserver.get_playbooks()
self.assertEqual(len(playbooks), 3)
self.assertTrue(
appserver.enable_bulk_emails_playbook() not in playbooks)
def test_ansible_settings_swift(self, mock_consul):
"""
Verify Swift Ansible configuration when Swift is enabled.
"""
instance = OpenEdXInstanceFactory()
appserver = make_test_appserver(instance)
self.check_ansible_settings(appserver)
def test_make_active(self, mock_run_playbook, mock_public_ip):
"""
POST /api/v1/openedx_appserver/:id/make_active/ - Make this OpenEdXAppServer active
for its given instance.
This can be done at any time; the AppServer must be healthy but "New",
"WaitingForServer", etc. are all considered healthy states, so the AppServer does not
necessarily have to be fully provisioned and online.
"""
self.api_client.login(username='******', password='******')
instance = OpenEdXInstanceFactory(edx_platform_commit='1' * 40)
server = ReadyOpenStackServerFactory()
app_server = make_test_appserver(instance=instance, server=server)
self.assertFalse(instance.get_active_appservers().exists())
response = self.api_client.post(
'/api/v1/openedx_appserver/{pk}/make_active/'.format(
pk=app_server.pk))
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data,
{'status': 'App server activation initiated.'})
self.assertEqual(mock_run_playbook.call_count, 1)
instance.refresh_from_db()
self.assertEqual(list(instance.get_active_appservers().all()),
[app_server])
app_server.refresh_from_db()
self.assertTrue(app_server.is_active)
def test_configuration_site_configuration_settings(self):
"""
Test that the 'configuration_site_configuration_settings' field has the correct value set when
there are static content overrides.
"""
instance = OpenEdXInstanceFactory()
instance.static_content_overrides = {
'version': 0,
'static_template_about_content': 'Hello world!',
'homepage_overlay_html': 'Welcome to the LMS!',
}
instance.save()
appserver = make_test_appserver(instance)
expected_values = {
'EDXAPP_SITE_CONFIGURATION': [{
'values': {
'static_template_about_content': 'Hello world!',
'homepage_overlay_html': 'Welcome to the LMS!',
}
}]
}
self.assertEqual(
yaml.safe_load(
appserver.configuration_site_configuration_settings),
expected_values)
def test_provision_failed_email(self, mock_consul):
"""
Tests that provision_failed sends email when called from normal program flow
"""
additional_monitoring_emails = ['additionalmonitoring@localhost']
failure_emails = ['provisionfailed@localhost']
appserver = make_test_appserver()
appserver.instance.additional_monitoring_emails = additional_monitoring_emails
appserver.instance.provisioning_failure_notification_emails = failure_emails
reason = "something went wrong"
log_lines = ["log line1", "log_line2"]
appserver.provision_failed_email(reason, log_lines)
expected_subject = OpenEdXAppServer.EmailSubject.PROVISION_FAILED.format(
name=appserver.name, instance_name=appserver.instance.name,
)
# failure_emails isn't included here because they get a different type of email (an urgent one)
expected_recipients = [admin_tuple[1] for admin_tuple in settings.ADMINS]
self.assertEqual(len(django_mail.outbox), 1)
mail = django_mail.outbox[0]
self.assertIn(expected_subject, mail.subject)
self.assertIn(appserver.name, mail.body)
self.assertIn(appserver.instance.name, mail.body)
self.assertIn(reason, mail.body)
self.assertEqual(mail.from_email, settings.SERVER_EMAIL)
self.assertEqual(mail.to, expected_recipients)
self.assertEqual(len(mail.attachments), 1)
self.assertEqual(mail.attachments[0], ("provision.log", "\n".join(log_lines), "text/plain"))
def test_provision_failed_email(self):
"""
Tests that provision_failed sends email when called from normal program flow
"""
appserver = make_test_appserver()
reason = "something went wrong"
log_lines = ["log line1", "log_line2"]
appserver.provision_failed_email(reason, log_lines)
expected_subject = OpenEdXAppServer.EmailSubject.PROVISION_FAILED.format(
name=appserver.name,
instance_name=appserver.instance.name,
)
expected_recipients = [
admin_tuple[1] for admin_tuple in settings.ADMINS
]
self.assertEqual(len(django_mail.outbox), 1)
mail = django_mail.outbox[0]
self.assertIn(expected_subject, mail.subject)
self.assertIn(appserver.name, mail.body)
self.assertIn(appserver.instance.name, mail.body)
self.assertIn(reason, mail.body)
self.assertEqual(mail.from_email, settings.SERVER_EMAIL)
self.assertEqual(mail.to, expected_recipients)
self.assertEqual(len(mail.attachments), 1)
self.assertEqual(mail.attachments[0],
("provision.log", "\n".join(log_lines), "text/plain"))
def test_ansible_settings_swift_disabled(self, mock_consul):
"""
Verify Swift Ansible configuration is not included when Swift is disabled.
"""
instance = OpenEdXInstanceFactory()
appserver = make_test_appserver(instance)
self.check_ansible_settings(appserver, expected=False)
def test_cancel_first_deployment_fails(self, mock_create_new_deployment,
mock_consul):
"""
Test that trying to stop the first provisioning fails.
"""
self.client.force_login(self.user_with_instance)
instance = self._setup_user_instance()
make_test_appserver(instance, status=Status.ConfiguringServer)
url = reverse(
'api:v2:openedx-instance-deployment-detail',
args=(self.instance_config.pk, ),
)
response = self.client.delete(url)
self.assertEqual(response.status_code, 400)
def test_provision_failed_email(self):
"""
Tests that provision_failed sends email when called from normal program flow
"""
appserver = make_test_appserver()
reason = "something went wrong"
log_lines = ["log line1", "log_line2"]
appserver.provision_failed_email(reason, log_lines)
expected_subject = OpenEdXAppServer.EmailSubject.PROVISION_FAILED.format(
name=appserver.name, instance_name=appserver.instance.name,
)
expected_recipients = [admin_tuple[1] for admin_tuple in settings.ADMINS]
self.assertEqual(len(django_mail.outbox), 1)
mail = django_mail.outbox[0]
self.assertIn(expected_subject, mail.subject)
self.assertIn(appserver.name, mail.body)
self.assertIn(appserver.instance.name, mail.body)
self.assertIn(reason, mail.body)
self.assertEqual(mail.from_email, settings.SERVER_EMAIL)
self.assertEqual(mail.to, expected_recipients)
self.assertEqual(len(mail.attachments), 1)
self.assertEqual(mail.attachments[0], ("provision.log", "\n".join(log_lines), "text/plain"))
def test_ansible_settings_swift_ephemeral(self):
"""
Verify Swift Ansible configuration is not included when using ephemeral databases.
"""
instance = OpenEdXInstanceFactory(use_ephemeral_databases=True)
appserver = make_test_appserver(instance)
self.check_ansible_settings(appserver, expected=False)
def test_get_details(self):
"""
GET - Detailed attributes
"""
self.api_client.login(username='******', password='******')
instance = OpenEdXInstanceFactory(sub_domain='domain.api')
app_server = make_test_appserver(instance)
instance.active_appserver = app_server # Outside of tests, use set_appserver_active() instead
instance.save()
response = self.api_client.get('/api/v1/instance/{pk}/'.format(pk=instance.ref.pk))
self.assertEqual(response.status_code, status.HTTP_200_OK)
instance_data = response.data.items()
self.assertIn(('domain', 'domain.api.example.com'), instance_data)
self.assertIn(('is_shut_down', False), instance_data)
self.assertIn(('name', instance.name), instance_data)
self.assertIn(('url', 'http://domain.api.example.com/'), instance_data)
self.assertIn(('studio_url', 'http://studio-domain.api.example.com/'), instance_data)
self.assertIn(
('edx_platform_repository_url', 'https://github.com/{}.git'.format(settings.DEFAULT_FORK)),
instance_data
)
self.assertIn(('edx_platform_commit', 'master'), instance_data)
# AppServer info:
self.assertIn(('appserver_count', 1), instance_data)
self.assertIn('active_appserver', response.data)
self.assertIn('newest_appserver', response.data)
for key in ('active_appserver', 'newest_appserver'):
app_server_data = response.data[key]
self.assertEqual(app_server_data['id'], app_server.pk)
self.assertEqual(
app_server_data['api_url'], 'http://testserver/api/v1/openedx_appserver/{pk}/'.format(pk=app_server.pk)
)
self.assertEqual(app_server_data['status'], 'new')
def test_youtube_api_key_unset(self):
"""
Check that EDXAPP_YOUTUBE_API_KEY is set to None by default.
"""
instance = OpenEdXInstanceFactory(sub_domain='youtube.apikey', use_ephemeral_databases=True)
appserver = make_test_appserver(instance)
configuration_vars = yaml.load(appserver.configuration_settings)
self.assertIsNone(configuration_vars['EDXAPP_YOUTUBE_API_KEY'])
def test_inventory_str_no_server(self, mocks):
"""
Ansible inventory string - should raise an exception if the server has no public IP
"""
appserver = make_test_appserver()
with self.assertRaises(RuntimeError) as context:
self.assertEqual(appserver.inventory_str, '[app]\n')
self.assertEqual(str(context.exception), "Cannot prepare to run playbooks when server has no public IP.")
def test_github_admin_username_list_default(self):
"""
By default, no admin should be configured
"""
appserver = make_test_appserver()
self.assertEqual(appserver.github_admin_organizations, [])
self.assertEqual(appserver.github_admin_users, [])
self.assertEqual(appserver.github_admin_username_list, [])
self.assertNotIn('COMMON_USER_INFO', appserver.configuration_settings)
def test_ansible_settings_no_mongo_server(self):
"""
Don't add mongo ansible vars if instance has no MongoDB server
"""
self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False)
self.instance.mongodb_server = None
self.instance.save()
appserver = make_test_appserver(self.instance)
self.check_mongo_vars_not_set(appserver)
def test_inventory_str(self, mocks):
"""
Ansible inventory string - should contain the public IP of the AppServer's VM
"""
mocks.mock_create_server.side_effect = [Mock(id='test-inventory-server'), None]
mocks.os_server_manager.add_fixture('test-inventory-server', 'openstack/api_server_2_active.json')
appserver = make_test_appserver()
appserver.provision() # This is when the server gets created
self.assertEqual(appserver.inventory_str, '[app]\n192.168.100.200')
def test_get_details_permission_denied(self, username, message):
"""
GET - Detailed attributes - anonymous, basic, and staff users denied access
"""
if username:
self.api_client.login(username=username, password='******')
app_server = make_test_appserver()
response = self.api_client.get('/api/v1/openedx_appserver/{pk}/'.format(pk=app_server.pk))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertEqual(response.data, {'detail': message})
def test_provision_unhandled_exception(self, mocks):
"""
Make sure that if there is an unhandled exception during provisioning, the provision()
method should return False and send an email.
"""
mocks.mock_run_ansible_playbooks.side_effect = Exception('Something went catastrophically wrong')
appserver = make_test_appserver()
result = appserver.provision()
self.assertFalse(result)
mocks.mock_provision_failed_email.assert_called_once_with("AppServer deploy failed: unhandled exception")
def test_get_log_entries(self):
"""
GET - Log entries
"""
self.api_client.login(username='******', password='******')
instance = OpenEdXInstanceFactory(name="Log Tester Instance")
app_server = make_test_appserver(instance)
server = app_server.server
app_server.logger.info("info")
app_server.logger.error("error")
server.logger.info("info")
server.logger.error("error")
response = self.api_client.get('/api/v1/openedx_appserver/{pk}/'.format(pk=app_server.pk))
self.assertEqual(response.status_code, status.HTTP_200_OK)
expected_list = [
{
'level': 'INFO',
'text': (
'instance.models.appserver | '
'instance={inst_id} (Log Tester Inst),app_server={as_id} (AppServer 1) | info'
),
},
{
'level': 'ERROR',
'text': (
'instance.models.appserver | '
'instance={inst_id} (Log Tester Inst),app_server={as_id} (AppServer 1) | error'
),
},
{
'level': 'INFO',
'text': 'instance.models.server | server={server_name} | info',
},
{
'level': 'ERROR',
'text': 'instance.models.server | server={server_name} | error',
},
{
'level': 'INFO',
'text': 'instance.models.server | server={server_name} |'
' Starting server (status=Pending [pending])...'
},
{
'level': 'ERROR',
'text': 'instance.models.server | server={server_name} |'
' Failed to start server: Not found (HTTP 404)'
},
]
self.check_log_list(
expected_list, response.data['log_entries'],
inst_id=instance.ref.id, as_id=app_server.pk, server_name=server.name,
)
def test_lms_user_settings(self):
"""
Test that lms_user_settings are initialised correctly for new AppServers.
"""
instance = OpenEdXInstanceFactory(use_ephemeral_databases=True)
user = get_user_model().objects.create_user(username='******', email='*****@*****.**')
instance.lms_users.add(user)
appserver = make_test_appserver(instance)
ansible_settings = yaml.load(appserver.lms_user_settings)
self.assertEqual(len(ansible_settings['django_users']), 1)
self.assertEqual(ansible_settings['django_users'][0]['username'], user.username)
self.assertEqual(ansible_settings['django_groups'], [])
def test_configuration_extra_settings(self):
"""
Add extra settings in ansible vars, which can override existing settings
"""
instance = OpenEdXInstanceFactory(
name='Vars Instance',
email='*****@*****.**',
configuration_extra_settings='EDXAPP_PLATFORM_NAME: "Overridden!"',
)
appserver = make_test_appserver(instance)
self.assertIn('EDXAPP_PLATFORM_NAME: Overridden!', appserver.configuration_settings)
self.assertNotIn('Vars Instance', appserver.configuration_settings)
self.assertIn("EDXAPP_CONTACT_EMAIL: [email protected]", appserver.configuration_settings)
def test_postfix_queue_settings_absent(self):
"""
Check that ansible vars for postfix_queue role are not present when SMTP relay host is not configured.
"""
instance = OpenEdXInstanceFactory(sub_domain='test.no.postfix.queue', use_ephemeral_databases=True)
appserver = make_test_appserver(instance)
configuration_vars = yaml.load(appserver.configuration_settings)
self.assertNotIn('POSTFIX_QUEUE_EXTERNAL_SMTP_HOST', configuration_vars)
self.assertNotIn('POSTFIX_QUEUE_EXTERNAL_SMTP_PORT', configuration_vars)
self.assertNotIn('POSTFIX_QUEUE_EXTERNAL_SMTP_USER', configuration_vars)
self.assertNotIn('POSTFIX_QUEUE_EXTERNAL_SMTP_PASSWORD', configuration_vars)
self.assertNotIn('POSTFIX_QUEUE_HEADER_CHECKS', configuration_vars)
self.assertNotIn('POSTFIX_QUEUE_SENDER_CANONICAL_MAPS', configuration_vars)
def test_status_transitions(self):
"""
Test that status transitions work as expected for different app server workflows
"""
# Normal workflow
app_server = make_test_appserver()
self.assertEqual(app_server.status, AppServerStatus.New)
self._assert_status_conditions(app_server)
app_server._status_to_waiting_for_server()
self.assertEqual(app_server.status, AppServerStatus.WaitingForServer)
self._assert_status_conditions(app_server, is_steady_state=False)
app_server._status_to_configuring_server()
self.assertEqual(app_server.status, AppServerStatus.ConfiguringServer)
self._assert_status_conditions(app_server, is_steady_state=False)
app_server._status_to_running()
self.assertEqual(app_server.status, AppServerStatus.Running)
self._assert_status_conditions(app_server)
app_server._status_to_terminated()
self.assertEqual(app_server.status, AppServerStatus.Terminated)
self._assert_status_conditions(app_server)
# Server creation fails
app_server_bad_server = make_test_appserver()
app_server_bad_server._status_to_waiting_for_server()
app_server_bad_server._status_to_error()
self.assertEqual(app_server_bad_server.status, AppServerStatus.Error)
self._assert_status_conditions(app_server_bad_server, is_healthy_state=False)
# Provisioning fails
app_server_provisioning_failed = make_test_appserver()
app_server_provisioning_failed._status_to_waiting_for_server()
app_server_provisioning_failed._status_to_configuring_server()
app_server_provisioning_failed._status_to_configuration_failed()
self.assertEqual(app_server_provisioning_failed.status, AppServerStatus.ConfigurationFailed)
self._assert_status_conditions(app_server_provisioning_failed, is_healthy_state=False)
def test_ansible_s3_settings(self):
"""
Test that get_storage_settings() includes S3 vars, and that they get passed on to the
AppServer
"""
instance = OpenEdXInstanceFactory(
s3_access_key='test-s3-access-key',
s3_secret_access_key='test-s3-secret-access-key',
s3_bucket_name='test-s3-bucket-name',
use_ephemeral_databases=False,
)
self.check_s3_vars(instance.get_storage_settings())
appserver = make_test_appserver(instance)
self.check_s3_vars(appserver.configuration_settings)
def test_invalid_status_transitions(self, transition):
"""
Test that invalid status transitions raise exception
"""
# TODO: Get pylint to see state as an iterable
invalid_from_states = (state for state in AppServerStatus.states #pylint: disable=not-an-iterable
if state not in transition['from_states'])
for invalid_from_state in invalid_from_states:
appserver = make_test_appserver()
# Hack to force the status:
OpenEdXAppServer.objects.filter(pk=appserver.pk).update(_status=invalid_from_state.state_id)
appserver = OpenEdXAppServer.objects.get(pk=appserver.pk)
self.assertEqual(appserver.status, invalid_from_state)
with self.assertRaises(WrongStateException):
getattr(appserver, transition['name'])()
def test_provision_build_failed(self, mocks):
"""
Run provisioning sequence failing server creation on purpose to make sure
server and instance statuses will be set accordingly.
"""
appserver = make_test_appserver()
self.assertEqual(appserver.status, AppServerStatus.New)
self.assertEqual(appserver.server.status, Server.Status.Pending)
mocks.mock_create_server.side_effect = novaclient.exceptions.ClientException(400)
result = appserver.provision()
self.assertFalse(result)
self.assertEqual(appserver.status, AppServerStatus.Error)
self.assertEqual(appserver.server.status, Server.Status.BuildFailed)
mocks.mock_provision_failed_email.assert_called_once_with('Unable to start an OpenStack server')
def test_provision_failed(self, mocks):
"""
Run provisioning sequence failing the deployment on purpose to make sure
server and instance statuses will be set accordingly.
"""
log_lines = ['log']
mocks.mock_run_ansible_playbooks.return_value = (log_lines, 1)
appserver = make_test_appserver()
self.assertEqual(appserver.status, AppServerStatus.New)
self.assertEqual(appserver.server.status, Server.Status.Pending)
result = appserver.provision()
self.assertFalse(result)
self.assertEqual(appserver.status, AppServerStatus.ConfigurationFailed)
self.assertEqual(appserver.server.status, Server.Status.Ready)
mocks.mock_provision_failed_email.assert_called_once_with(
"AppServer deploy failed: Ansible play exited with non-zero exit code", log_lines
)
def test_ansible_settings_mongo(self):
"""
Add mongo ansible vars if instance has a MongoDB server
"""
self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False)
appserver = make_test_appserver(self.instance)
ansible_vars = appserver.configuration_settings
self.assertIn('EDXAPP_MONGO_USER: {0}'.format(self.instance.mongo_user), ansible_vars)
self.assertIn('EDXAPP_MONGO_PASSWORD: {0}'.format(self.instance.mongo_pass), ansible_vars)
self.assertIn('EDXAPP_MONGO_HOSTS: [mongo.opencraft.com]', ansible_vars)
self.assertIn('EDXAPP_MONGO_PORT: {0}'.format(MONGODB_SERVER_DEFAULT_PORT), ansible_vars)
self.assertIn('EDXAPP_MONGO_DB_NAME: {0}'.format(self.instance.mongo_database_name), ansible_vars)
self.assertIn('FORUM_MONGO_USER: {0}'.format(self.instance.mongo_user), ansible_vars)
self.assertIn('FORUM_MONGO_PASSWORD: {0}'.format(self.instance.mongo_pass), ansible_vars)
self.assertIn('FORUM_MONGO_HOSTS: [mongo.opencraft.com]', ansible_vars)
self.assertIn('FORUM_MONGO_PORT: {0}'.format(MONGODB_SERVER_DEFAULT_PORT), ansible_vars)
self.assertIn('FORUM_MONGO_DATABASE: {0}'.format(self.instance.forum_database_name), ansible_vars)
def test_provision(self, mocks):
"""
Run provisioning sequence
"""
mocks.mock_run_ansible_playbooks.return_value = (['log'], 0)
mocks.mock_create_server.side_effect = [Mock(id='test-run-provisioning-server'), None]
mocks.os_server_manager.add_fixture('test-run-provisioning-server', 'openstack/api_server_2_active.json')
mock_reboot = mocks.os_server_manager.get_os_server('test-run-provisioning-server').reboot
appserver = make_test_appserver()
self.assertEqual(appserver.status, AppServerStatus.New)
self.assertEqual(appserver.server.status, Server.Status.Pending)
result = appserver.provision()
self.assertTrue(result)
self.assertEqual(appserver.status, AppServerStatus.Running)
self.assertEqual(appserver.server.status, Server.Status.Ready)
self.assertEqual(mocks.mock_run_ansible_playbooks.call_count, 1)
self.assertEqual(mock_reboot.call_count, 1)
def test_github_admin_username_list(self, mock_get_username_list):
"""
When Github admin users are set, they should end up in the Ansible configuration.
"""
mock_get_username_list.side_effect = {
'test-org1': ['jane', 'joey'],
'test-org2': ['jess', 'jack'],
}.get
instance = OpenEdXInstanceFactory(
github_admin_organizations=['test-org1', 'test-org2'],
github_admin_users=['jean', 'john'],
)
all_names = ['jane', 'joey', 'jess', 'jack', 'jean', 'john']
appserver = make_test_appserver(instance)
self.assertEqual(appserver.github_admin_username_list, all_names)
ansible_settings = yaml.load(appserver.configuration_settings)
self.assertEqual(ansible_settings['COMMON_USER_INFO'], [
{'name': name, 'github': True, 'type': 'admin'} for name in all_names
])
def test_run_playbook_logging(self, mock_inventory_str, mock_run_playbook):
"""
Ensure logging routines are working on _run_playbook method
"""
stdout_r, stdout_w = os.pipe()
stderr_r, stderr_w = os.pipe()
with open(stdout_r, 'rb', buffering=0) as stdout, open(stderr_r, 'rb', buffering=0) as stderr:
mock_run_playbook.return_value.__enter__.return_value.stdout = stdout
mock_run_playbook.return_value.__enter__.return_value.stderr = stderr
mock_run_playbook.return_value.__enter__.return_value.returncode = 0
os.write(stdout_w, b'Hello\n')
os.close(stdout_w)
os.write(stderr_w, b'Hi\n')
os.close(stderr_w)
appserver = make_test_appserver()
playbook = Playbook(source_repo='dummy', playbook_path='dummy', requirements_path='dummy', version='dummy',
variables='dummy')
log, returncode = appserver._run_playbook("/tmp/test/working/dir/", playbook)
self.assertCountEqual(log, ['Hello', 'Hi'])
self.assertEqual(returncode, 0)
def test_provisioning(
self, playbook_returncode, mock_open_repo, mock_inventory, mock_run_playbook, mock_poll_streams):
"""
Test instance provisioning
"""
appserver = make_test_appserver()
working_dir = '/cloned/configuration-repo/path'
mock_open_repo.return_value.__enter__.return_value.working_dir = working_dir
mock_run_playbook.return_value.__enter__.return_value.returncode = playbook_returncode
appserver.run_ansible_playbooks()
self.assertIn(call(
requirements_path='{}/requirements.txt'.format(working_dir),
inventory_str=mock_inventory,
vars_str=appserver.configuration_settings,
playbook_path='{}/playbooks'.format(working_dir),
playbook_name='edx_sandbox.yml',
username='******',
), mock_run_playbook.mock_calls)
def _create_appserver(self, instance, status, created=None):
"""
Return appserver for `instance` that has `status`, and (optionally) was `created` on a specific date.
Note that this method does not set the status of the VM (OpenStackServer)
that is associated with the app server.
Client code is expected to take care of that itself (if necessary).
"""
appserver = make_test_appserver(instance)
if created:
appserver.created = created
appserver.save()
if status == AppServerStatus.Running:
self._set_appserver_running(appserver)
if status == AppServerStatus.ConfigurationFailed:
self._set_appserver_configuration_failed(appserver)
elif status == AppServerStatus.Terminated:
self._set_appserver_terminated(appserver)
return appserver
def test_make_active(self, mock_run_playbook):
"""
POST /api/v1/openedx_appserver/:id/make_active/ - Make this OpenEdXAppServer active
for its given instance.
This can be done at any time; the AppServer must be healthy but "New",
"WaitingForServer", etc. are all considered healthy states, so the AppServer does not
necessarily have to be fully provisioned and online.
"""
self.api_client.login(username='******', password='******')
instance = OpenEdXInstanceFactory(edx_platform_commit='1' * 40, use_ephemeral_databases=True)
app_server = make_test_appserver(instance)
self.assertEqual(instance.active_appserver, None)
response = self.api_client.post('/api/v1/openedx_appserver/{pk}/make_active/'.format(pk=app_server.pk))
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data, {'status': 'App server activation initiated.'})
self.assertEqual(mock_run_playbook.call_count, 1)
instance.refresh_from_db()
self.assertEqual(instance.active_appserver, app_server)