def test_terminate_obsolete_appservers_no_active(self):
"""
Test that `terminate_obsolete_appservers` does not terminate any app servers
if an instance does not have an active app server.
"""
instance = OpenEdXInstanceFactory()
reference_date = timezone.now()
# Create app servers
obsolete_appserver = self._create_running_appserver(instance, reference_date - timedelta(days=5))
obsolete_appserver_failed = self._create_failed_appserver(instance, reference_date - timedelta(days=5))
recent_appserver = self._create_running_appserver(instance, reference_date - timedelta(days=1))
recent_appserver_failed = self._create_failed_appserver(instance, reference_date - timedelta(days=1))
appserver = self._create_running_appserver(instance, reference_date)
appserver_failed = self._create_failed_appserver(instance, reference_date)
newer_appserver = self._create_running_appserver(instance, reference_date + timedelta(days=3))
newer_appserver_failed = self._create_failed_appserver(instance, reference_date + timedelta(days=3))
# Terminate app servers
instance.terminate_obsolete_appservers()
# Check status of app servers (should be unchanged)
self._assert_status([
(obsolete_appserver, AppServerStatus.Running, ServerStatus.Pending),
(obsolete_appserver_failed, AppServerStatus.ConfigurationFailed, ServerStatus.Pending),
(recent_appserver, AppServerStatus.Running, ServerStatus.Pending),
(recent_appserver_failed, AppServerStatus.ConfigurationFailed, ServerStatus.Pending),
(appserver, AppServerStatus.Running, ServerStatus.Pending),
(appserver_failed, AppServerStatus.ConfigurationFailed, ServerStatus.Pending),
(newer_appserver, AppServerStatus.Running, ServerStatus.Pending),
(newer_appserver_failed, AppServerStatus.ConfigurationFailed, ServerStatus.Pending),
])
def test_get_details(self):
"""
GET - Detailed attributes
"""
self.api_client.login(username='******', password='******')
instance = OpenEdXInstanceFactory(sub_domain='domain.api')
app_server = make_test_appserver(instance)
instance.active_appserver = app_server # Outside of tests, use set_appserver_active() instead
instance.save()
response = self.api_client.get('/api/v1/instance/{pk}/'.format(pk=instance.ref.pk))
self.assertEqual(response.status_code, status.HTTP_200_OK)
instance_data = response.data.items()
self.assertIn(('domain', 'domain.api.example.com'), instance_data)
self.assertIn(('is_shut_down', False), instance_data)
self.assertIn(('name', instance.name), instance_data)
self.assertIn(('url', 'http://domain.api.example.com/'), instance_data)
self.assertIn(('studio_url', 'http://studio-domain.api.example.com/'), instance_data)
self.assertIn(
('edx_platform_repository_url', 'https://github.com/{}.git'.format(settings.DEFAULT_FORK)),
instance_data
)
self.assertIn(('edx_platform_commit', 'master'), instance_data)
# AppServer info:
self.assertIn(('appserver_count', 1), instance_data)
self.assertIn('active_appserver', response.data)
self.assertIn('newest_appserver', response.data)
for key in ('active_appserver', 'newest_appserver'):
app_server_data = response.data[key]
self.assertEqual(app_server_data['id'], app_server.pk)
self.assertEqual(
app_server_data['api_url'], 'http://testserver/api/v1/openedx_appserver/{pk}/'.format(pk=app_server.pk)
)
self.assertEqual(app_server_data['status'], 'new')
def test_enable_monitoring(self, mock_newrelic):
"""
Check that the `enable_monitoring` method creates New Relic Synthetics
monitors for each of the instance's public urls, and enables email
alerts.
"""
monitor_ids = [str(uuid4()) for i in range(3)]
mock_newrelic.get_synthetics_monitors.return_value = []
mock_newrelic.create_synthetics_monitor.side_effect = monitor_ids
instance = OpenEdXInstanceFactory()
instance.enable_monitoring()
# Check that the monitors have been created
mock_newrelic.delete_synthetics_monitor.assert_not_called()
mock_newrelic.create_synthetics_monitor.assert_has_calls([
call(instance.url),
call(instance.studio_url),
call(instance.lms_preview_url),
], any_order=True)
self.assertCountEqual(
instance.new_relic_availability_monitors.values_list('pk', flat=True),
monitor_ids
)
# Check that alert emails have been set up
mock_newrelic.add_synthetics_email_alerts.assert_has_calls([
call(monitor_id, ['*****@*****.**'])
for monitor_id in monitor_ids
], any_order=True)
def test_preliminary_page_not_configured(self):
"""
Test that get_preliminary_page_config() returns an empty configuration if
PRELIMINARY_PAGE_SERVER_IP is not set.
"""
instance = OpenEdXInstanceFactory()
self.assertEqual(instance.get_preliminary_page_config(instance.ref.pk), ([], []))
def test_spawn_appserver_detailed(self, mocks):
"""
Test spawning an AppServer in more detail; this is partially an integration test
Unlike test_spawn_appserver(), this test does not mock the .provision() method, so the
AppServer will go through the motions of provisioning and end up with the appropriate
status.
Note that OpenEdXInstance does not include auto-retry support or auto-activation upon
success; those behaviors are implemented at the task level in tasks.py and tested in
test_tasks.py
"""
mocks.mock_create_server.side_effect = [Mock(id='test-run-provisioning-server'), None]
mocks.os_server_manager.add_fixture('test-run-provisioning-server', 'openstack/api_server_2_active.json')
instance = OpenEdXInstanceFactory(
sub_domain='test.spawn',
use_ephemeral_databases=True,
)
self.assertEqual(instance.appserver_set.count(), 0)
self.assertIsNone(instance.active_appserver)
appserver_id = instance.spawn_appserver()
self.assertIsNotNone(appserver_id)
self.assertEqual(instance.appserver_set.count(), 1)
self.assertIsNone(instance.active_appserver)
appserver = instance.appserver_set.get(pk=appserver_id)
self.assertEqual(appserver.status, AppServerStatus.Running)
self.assertEqual(appserver.server.status, Server.Status.Ready)
def test_delete_instance(self, mocks, delete_by_ref, *mock_methods):
"""
Test that an instance can be deleted directly or by its InstanceReference.
"""
instance = OpenEdXInstanceFactory(sub_domain='test.deletion', use_ephemeral_databases=True)
instance_ref = instance.ref
appserver = OpenEdXAppServer.objects.get(pk=instance.spawn_appserver())
for method in mock_methods:
self.assertEqual(
method.call_count, 0,
'{} should not have been called'.format(method._mock_name)
)
# Now delete the instance, either using InstanceReference or the OpenEdXInstance class:
if delete_by_ref:
instance_ref.delete()
else:
instance.delete()
for method in mock_methods:
self.assertEqual(
method.call_count, 1,
'{} should have been called exactly once'.format(method._mock_name)
)
with self.assertRaises(OpenEdXInstance.DoesNotExist):
OpenEdXInstance.objects.get(pk=instance.pk)
with self.assertRaises(InstanceReference.DoesNotExist):
instance_ref.refresh_from_db()
with self.assertRaises(OpenEdXAppServer.DoesNotExist):
appserver.refresh_from_db()
def test_set_appserver_active(self, mocks, mock_enable_monitoring):
"""
Check that monitoring is enabled when an appserver is activated.
"""
instance = OpenEdXInstanceFactory()
appserver_id = instance.spawn_appserver()
instance.set_appserver_active(appserver_id)
self.assertEqual(mock_enable_monitoring.call_count, 1)
def test_swift_disabled(self, create_swift_container):
"""
Verify disabling Swift provisioning works.
"""
instance = OpenEdXInstanceFactory(use_ephemeral_databases=False)
instance.provision_swift()
self.assertIs(instance.swift_provisioned, False)
self.assertFalse(create_swift_container.called)
def test_instance_subdomain(self):
"""
Subdomain used by an existing instance.
"""
OpenEdXInstanceFactory.create(
sub_domain=self.form_data['subdomain'],
)
self._assert_registration_fails(self.form_data, expected_errors={
'subdomain': ['This domain is already taken.'],
})
def make_test_appserver(instance=None):
"""
Factory method to create an OpenEdXAppServer (and OpenStackServer).
"""
if not instance:
instance = OpenEdXInstanceFactory()
if not instance.load_balancing_server:
instance.load_balancing_server = LoadBalancingServer.objects.select_random()
instance.save()
return instance._create_owned_appserver()
def test_ansible_s3_settings_ephemeral(self):
"""
Test that get_storage_settings() does not include S3 vars when in ephemeral mode
"""
instance = OpenEdXInstanceFactory(
s3_access_key='test-s3-access-key',
s3_secret_access_key='test-s3-secret-access-key',
s3_bucket_name='test-s3-bucket-name',
use_ephemeral_databases=True,
)
self.assertEqual(instance.get_storage_settings(), '')
def test_domain_url(self):
"""
Domain and URL attributes
"""
instance = OpenEdXInstanceFactory(
internal_lms_domain='sample.example.org', name='Sample Instance'
)
internal_lms_domain = 'sample.example.org'
internal_lms_preview_domain = 'preview-sample.example.org'
internal_studio_domain = 'studio-sample.example.org'
self.assertEqual(instance.internal_lms_domain, internal_lms_domain)
self.assertEqual(instance.internal_lms_preview_domain, internal_lms_preview_domain)
self.assertEqual(instance.internal_studio_domain, internal_studio_domain)
# External domains are empty by default.
self.assertEqual(instance.external_lms_domain, '')
self.assertEqual(instance.external_lms_preview_domain, '')
self.assertEqual(instance.external_studio_domain, '')
# When external domain is empty, main domains/URLs equal internal domains.
self.assertEqual(instance.domain, internal_lms_domain)
self.assertEqual(instance.lms_preview_domain, internal_lms_preview_domain)
self.assertEqual(instance.studio_domain, internal_studio_domain)
self.assertEqual(instance.studio_domain_nginx_regex, r'~^(studio\-sample\.example\.org)$')
self.assertEqual(instance.url, 'http://{}/'.format(internal_lms_domain))
self.assertEqual(instance.lms_preview_url, 'http://{}/'.format(internal_lms_preview_domain))
self.assertEqual(instance.studio_url, 'http://{}/'.format(internal_studio_domain))
self.assertEqual(str(instance), 'Sample Instance (sample.example.org)')
# External domains take precedence over internal domains.
external_lms_domain = 'external.domain.com'
external_lms_preview_domain = 'lms-preview.external.domain.com'
external_studio_domain = 'external-studio.domain.com'
instance.external_lms_domain = external_lms_domain
instance.external_lms_preview_domain = external_lms_preview_domain
instance.external_studio_domain = external_studio_domain
# Internal domains are still the same.
self.assertEqual(instance.internal_lms_domain, internal_lms_domain)
self.assertEqual(instance.internal_lms_preview_domain, internal_lms_preview_domain)
self.assertEqual(instance.internal_studio_domain, internal_studio_domain)
# Default domains will now equal external domains.
self.assertEqual(instance.domain, external_lms_domain)
self.assertEqual(instance.lms_preview_domain, external_lms_preview_domain)
self.assertEqual(instance.studio_domain, external_studio_domain)
self.assertEqual(
instance.studio_domain_nginx_regex,
r'~^(external\-studio\.domain\.com|studio\-sample\.example\.org)$'
)
self.assertEqual(instance.url, 'http://{}/'.format(external_lms_domain))
self.assertEqual(instance.lms_preview_url, 'http://{}/'.format(external_lms_preview_domain))
self.assertEqual(instance.studio_url, 'http://{}/'.format(external_studio_domain))
self.assertEqual(str(instance), 'Sample Instance (external.domain.com)')
# URLs respect the protocol setting.
instance.protocol = 'https'
self.assertEqual(instance.url, 'https://{}/'.format(external_lms_domain))
self.assertEqual(instance.lms_preview_url, 'https://{}/'.format(external_lms_preview_domain))
self.assertEqual(instance.studio_url, 'https://{}/'.format(external_studio_domain))
def test_spawn_appserver_with_lms_users(self, mocks, mock_provision):
"""
Provision an AppServer with a user added to lms_users.
"""
instance = OpenEdXInstanceFactory(sub_domain='test.spawn', use_ephemeral_databases=True)
user = get_user_model().objects.create_user(username='******', email='*****@*****.**')
instance.lms_users.add(user)
appserver_id = instance.spawn_appserver()
appserver = instance.appserver_set.get(pk=appserver_id)
self.assertEqual(appserver.lms_users.count(), 1)
self.assertEqual(appserver.lms_users.get(), user)
self.assertTrue(appserver.lms_user_settings)
def test_provision_swift(self, create_swift_container):
"""
Test provisioning Swift containers, and that they are provisioned only once.
"""
instance = OpenEdXInstanceFactory(use_ephemeral_databases=False)
instance.provision_swift()
self.check_swift(instance, create_swift_container)
# Provision again without resetting the mock. The assertCountEqual assertion will verify
# that the container isn't provisioned again.
instance.provision_swift()
self.check_swift(instance, create_swift_container)
def test_forum_api_key(self, mocks, mock_provision):
"""
Ensure the FORUM_API_KEY matches EDXAPP_COMMENTS_SERVICE_KEY
"""
instance = OpenEdXInstanceFactory(sub_domain='test.forum_api_key', use_ephemeral_databases=True)
appserver_id = instance.spawn_appserver()
appserver = instance.appserver_set.get(pk=appserver_id)
configuration_vars = yaml.load(appserver.configuration_settings)
api_key = configuration_vars['EDXAPP_COMMENTS_SERVICE_KEY']
self.assertIsNot(api_key, '')
self.assertIsNotNone(api_key)
self.assertEqual(configuration_vars['FORUM_API_KEY'], api_key)
def test_shut_down(self, mock_reconfigure, mock_disable_monitoring, mock_remove_dns_records):
"""
Test that `shut_down` method terminates all app servers belonging to an instance
and disables monitoring.
"""
instance = OpenEdXInstanceFactory()
instance.load_balancing_server = LoadBalancingServer.objects.select_random()
instance.save()
reference_date = timezone.now()
# Create app servers
obsolete_appserver = self._create_running_appserver(instance, reference_date - timedelta(days=5))
obsolete_appserver_failed = self._create_failed_appserver(instance, reference_date - timedelta(days=5))
recent_appserver = self._create_running_appserver(instance, reference_date - timedelta(days=1))
recent_appserver_failed = self._create_failed_appserver(instance, reference_date - timedelta(days=1))
active_appserver = self._create_running_appserver(instance, reference_date)
newer_appserver = self._create_running_appserver(instance, reference_date + timedelta(days=3))
newer_appserver_failed = self._create_failed_appserver(instance, reference_date + timedelta(days=3))
# Set single app server active
instance.active_appserver = active_appserver
instance.save()
active_appserver.instance.refresh_from_db()
self.assertEqual(mock_reconfigure.call_count, 0)
self.assertEqual(mock_disable_monitoring.call_count, 0)
self.assertEqual(mock_remove_dns_records.call_count, 0)
# Shut down instance
instance.shut_down()
self.assertEqual(mock_reconfigure.call_count, 1)
self.assertEqual(mock_disable_monitoring.call_count, 1)
self.assertEqual(mock_remove_dns_records.call_count, 1)
# Check status of running app servers
self._assert_status([
(obsolete_appserver, AppServerStatus.Terminated, ServerStatus.Terminated),
(recent_appserver, AppServerStatus.Terminated, ServerStatus.Terminated),
(active_appserver, AppServerStatus.Terminated, ServerStatus.Terminated),
(newer_appserver, AppServerStatus.Terminated, ServerStatus.Terminated),
])
# Check status of failed app servers:
# AppServerStatus.Terminated is reserved for instances that were running successfully at some point,
# so app servers with AppServerStatus.ConfigurationFailed will still have that status
# after `shut_down` calls `terminate_vm` on them.
# However, the VM (OpenStackServer) that an app server is associated with
# *should* have ServerStatus.Terminated if the app server was old enough to be terminated.
self._assert_status([
(obsolete_appserver_failed, AppServerStatus.ConfigurationFailed, ServerStatus.Terminated),
(recent_appserver_failed, AppServerStatus.ConfigurationFailed, ServerStatus.Terminated),
(newer_appserver_failed, AppServerStatus.ConfigurationFailed, ServerStatus.Terminated),
])
def test_disable_monitoring(self, mock_newrelic): # pylint: disable=no-self-use
"""
Check that the `disable_monitoring` method removes any New Relic
Synthetics monitors for this instance.
"""
monitor_ids = [str(uuid4()) for i in range(3)]
instance = OpenEdXInstanceFactory()
for monitor_id in monitor_ids:
instance.new_relic_availability_monitors.create(pk=monitor_id)
instance.disable_monitoring()
mock_newrelic.delete_synthetics_monitor.assert_has_calls([
call(monitor_id) for monitor_id in monitor_ids
], any_order=True)
def test_newrelic_configuration(self, mocks, mock_provision):
"""
Check that newrelic ansible vars are set correctly
"""
instance = OpenEdXInstanceFactory(sub_domain='test.newrelic', use_ephemeral_databases=True)
appserver_id = instance.spawn_appserver()
appserver = instance.appserver_set.get(pk=appserver_id)
configuration_vars = yaml.load(appserver.configuration_settings)
self.assertIs(configuration_vars['COMMON_ENABLE_NEWRELIC'], True)
self.assertIs(configuration_vars['COMMON_ENABLE_NEWRELIC_APP'], True)
self.assertEqual(configuration_vars['COMMON_ENVIRONMENT'], 'opencraft')
self.assertEqual(configuration_vars['COMMON_DEPLOYMENT'], instance.internal_lms_domain)
self.assertEqual(configuration_vars['NEWRELIC_LICENSE_KEY'], 'newrelic-key')
def test_ansible_s3_settings(self):
"""
Test that get_storage_settings() includes S3 vars, and that they get passed on to the
AppServer
"""
instance = OpenEdXInstanceFactory(
s3_access_key='test-s3-access-key',
s3_secret_access_key='test-s3-secret-access-key',
s3_bucket_name='test-s3-bucket-name',
use_ephemeral_databases=False,
)
self.check_s3_vars(instance.get_storage_settings())
appserver = make_test_appserver(instance)
self.check_s3_vars(appserver.configuration_settings)
def test_set_appserver_active(self, mocks):
"""
Test set_appserver_active()
"""
instance = OpenEdXInstanceFactory(internal_lms_domain='test.activate.opencraft.co.uk',
use_ephemeral_databases=True)
appserver_id = instance.spawn_appserver()
instance.set_appserver_active(appserver_id)
instance.refresh_from_db()
self.assertEqual(instance.active_appserver.pk, appserver_id)
self.assertEqual(mocks.mock_load_balancer_run_playbook.call_count, 2)
instance.set_appserver_inactive()
instance.refresh_from_db()
self.assertIsNone(instance.active_appserver)
self.assertEqual(mocks.mock_load_balancer_run_playbook.call_count, 3)
def test_provision_mongo(self):
"""
Provision mongo databases
"""
self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False)
self.instance.provision_mongo()
self.check_mongo()
def test_create_instance_no_default_mongo_url(self):
"""
Test that creating an instance with persistent databases raises an exception
if DEFAULT_INSTANCE_MONGO_URL not set.
"""
with self.assertRaises(MongoDBServer.DoesNotExist):
self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False)
def test_ansible_settings_mongo_ephemeral(self):
"""
Don't add mysql ansible vars for ephemeral databases
"""
self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=True)
appserver = make_test_appserver(self.instance)
self.check_mongo_vars_not_set(appserver)
def test_spawn_appserver_names(self, mocks, mock_provision):
"""
Run spawn_appserver() sequence multiple times and check names of resulting app servers
"""
instance = OpenEdXInstanceFactory(sub_domain='test.spawn_names', use_ephemeral_databases=True)
appserver_id = instance.spawn_appserver()
appserver = instance.appserver_set.get(pk=appserver_id)
self.assertEqual(appserver.name, "AppServer 1")
appserver_id = instance.spawn_appserver()
appserver = instance.appserver_set.get(pk=appserver_id)
self.assertEqual(appserver.name, "AppServer 2")
appserver_id = instance.spawn_appserver()
appserver = instance.appserver_set.get(pk=appserver_id)
self.assertEqual(appserver.name, "AppServer 3")
def test_spawn_appserver_with_external_databases(self, mocks, mock_provision):
"""
Run spawn_appserver() sequence, with external databases
"""
instance = OpenEdXInstanceFactory(sub_domain='test.persistent', use_ephemeral_databases=False)
appserver_id = instance.spawn_appserver()
self.assertEqual(mocks.mock_provision_mysql.call_count, 1)
self.assertEqual(mocks.mock_provision_mongo.call_count, 1)
self.assertEqual(mocks.mock_provision_swift.call_count, 1)
appserver = instance.appserver_set.get(pk=appserver_id)
ansible_vars = yaml.load(appserver.configuration_settings)
for setting in ('EDXAPP_MYSQL_USER', 'EDXAPP_MONGO_PASSWORD',
'EDXAPP_MONGO_USER', 'EDXAPP_MONGO_PASSWORD',
'EDXAPP_SWIFT_USERNAME', 'EDXAPP_SWIFT_KEY'):
self.assertTrue(ansible_vars[setting])
def test__get_database_suffix(self):
"""
Test that _get_database_suffix returns correct suffix for a given database.
"""
self.instance = OpenEdXInstanceFactory()
suffix = "test"
database_name = self.instance._get_mysql_database_name(suffix)
self.assertEqual(self.instance._get_database_suffix(database_name), suffix)
def test_ansible_settings_no_mysql_server(self):
"""
Don't add mysql ansible vars if instance has no MySQL server
"""
self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False)
self.instance.mysql_server = None
self.instance.save()
self.check_mysql_vars_not_set(self.instance)
def test_get_load_balancer_configuration(self, mocks):
"""
Test that the load balancer configuration gets generated correctly.
"""
instance = OpenEdXInstanceFactory(sub_domain='test.load_balancer', use_ephemeral_databases=True)
domain_names = [
"test.load_balancer.example.com",
"preview-test.load_balancer.example.com",
"studio-test.load_balancer.example.com",
]
# Test configuration for preliminary page
backend_map, config = instance.get_load_balancer_configuration()
self._check_load_balancer_configuration(
backend_map, config, domain_names, settings.PRELIMINARY_PAGE_SERVER_IP
)
# Test configuration for active appserver
appserver_id = instance.spawn_appserver()
instance.set_appserver_active(appserver_id)
backend_map, config = instance.get_load_balancer_configuration()
self._check_load_balancer_configuration(
backend_map, config, domain_names, instance.active_appserver.server.public_ip
)
# Test configuration in case an active appserver doesn't have a public IP address anymore.
# This might happen if the OpenStack server dies or gets modified from the outside, but it
# is not expected to happen under normal circumstances. We deconfigure the backend and log
# an error in this case.
with patch('instance.openstack.get_server_public_address', return_value=None), \
self.assertLogs("instance.models.instance", "ERROR"):
self.assertEqual(instance.get_load_balancer_configuration(), ([], []))
def test_secret_key_settings(self):
"""
Test the YAML settings returned by SecretKeyInstanceMixin.
"""
instance = OpenEdXInstanceFactory()
settings = yaml.load(instance.get_secret_key_settings())
# Test that all keys are hex-encoded strings.
for secret_key in settings.values():
codecs.decode(secret_key, "hex")
# Make sure all independent secret keys are all different
independent_secrets = set(settings[var] for var in OPENEDX_SECRET_KEYS)
self.assertEqual(len(independent_secrets), len(OPENEDX_SECRET_KEYS))
# Verify that API client keys are set to the matching server key.
for to_var, from_var in OPENEDX_SHARED_KEYS.items():
self.assertEqual(settings[to_var], settings[from_var])
def test_ansible_settings_no_mongo_server(self):
"""
Don't add mongo ansible vars if instance has no MongoDB server
"""
self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False)
self.instance.mongodb_server = None
self.instance.save()
appserver = make_test_appserver(self.instance)
self.check_mongo_vars_not_set(appserver)
def setUp(self):
super().setUp()
with patch(
'instance.tests.models.factories.openedx_instance.OpenEdXInstance._write_metadata_to_consul',
return_value=(1, True)):
self.instance = OpenEdXInstanceFactory()
def test_deprovision_s3(self, s3_connection, iam_connection):
"""
Test s3 deprovisioning succeeds
"""
instance = OpenEdXInstanceFactory()
instance.storage_type = StorageContainer.S3_STORAGE
instance.s3_access_key = 'test'
instance.s3_secret_access_key = 'test'
instance.s3_bucket_name = 'test'
instance.s3_region = 'test'
instance.provision_s3()
instance.deprovision_s3()
instance.refresh_from_db()
self.assertEqual(instance.s3_bucket_name, "")
self.assertEqual(instance.s3_access_key, "")
self.assertEqual(instance.s3_secret_access_key, "")
# We always want to preserve information about a client's preferred region, so s3_region should not be empty.
self.assertEqual(instance.s3_region, "test")
def test_deprovision_s3_delete_bucket_fails(self, s3_connection,
connect_iam):
"""
Test s3 deprovisioning fails on delete_bucket
"""
s3_connection = s3_connection()
s3_connection.delete_bucket.side_effect = boto.exception.S3ResponseError(
403, "Forbidden")
instance = OpenEdXInstanceFactory()
instance.storage_type = StorageContainer.S3_STORAGE
instance.s3_access_key = 'test'
instance.s3_secret_access_key = 'test'
instance.s3_bucket_name = 'test'
instance.s3_region = 'test'
with self.assertLogs("instance.models.instance"):
instance.deprovision_s3()
instance.refresh_from_db()
# Since it failed deleting the bucket, s3_bucket_name should not be empty
self.assertEqual(instance.s3_bucket_name, "test")
# We always want to preserve information about a client's preferred region, so s3_region should not be empty.
self.assertEqual(instance.s3_region, "test")
self.assertEqual(instance.s3_secret_access_key, "")
self.assertEqual(instance.s3_access_key, "")
def test_s3_region_default_value(self):
"""
Test the default value for the S3 region
"""
instance = OpenEdXInstanceFactory()
self.assertEqual(instance.s3_region, settings.AWS_S3_DEFAULT_REGION)
def test_ansible_settings_mysql(self, mocks):
"""
Test that get_database_settings produces correct settings for MySQL databases
"""
# Delete MySQLServer object created during the migrations to allow the settings override to
# take effect.
MySQLServer.objects.all().delete()
self.instance = OpenEdXInstanceFactory()
expected_host = "mysql.opencraft.com"
expected_port = MYSQL_SERVER_DEFAULT_PORT
def make_flat_group_info(var_names=None,
database=None,
include_port=True):
""" Return dict containing info for a flat group of variables """
group_info = {}
if var_names:
group_info["vars"] = var_names
# Compute and insert values
name = self.instance._get_mysql_database_name(database["name"])
user = self.instance._get_mysql_user_name(database["user"])
password = self.instance._get_mysql_pass(user)
values = [name, user, password, expected_host]
if include_port:
values.append(expected_port)
group_info["values"] = values
return group_info
def make_nested_group_info(var_names, databases):
""" Return dict containing info for a nested group of variables """
group_info = {"vars": var_names}
# Compute and insert values
for database in databases:
database["name"] = self.instance._get_mysql_database_name(
database["name"])
database["user"] = self.instance._get_mysql_user_name(
database["user"])
database["password"] = self.instance._get_mysql_pass(
database["user"])
values = [database["name"] for database in databases]
values.append({
database.get("id", "default"): dict(
ENGINE='django.db.backends.mysql',
NAME=database["name"],
USER=database["user"],
PASSWORD=database["password"],
HOST=expected_host,
PORT=expected_port,
**database.get("additional_settings", {}),
)
for database in databases
})
group_info["values"] = values
return group_info
# Load instance settings
db_vars = yaml.load(self.instance.get_database_settings())
# Check instance settings for common users
self.check_common_users(self.instance, db_vars)
# Check service-specific instance settings
var_groups = {
"EDXAPP_MYSQL_":
make_flat_group_info(database={
"name": "edxapp",
"user": "******"
}),
"XQUEUE_MYSQL_":
make_flat_group_info(database={
"name": "xqueue",
"user": "******"
}),
"EDXAPP_MYSQL_CSMH_":
make_flat_group_info(database={
"name": "edxapp_csmh",
"user": "******"
}),
"NOTIFIER_DATABASE_":
make_flat_group_info(
var_names=["NAME", "USER", "PASSWORD", "HOST", "PORT"],
database={
"name": "notifier",
"user": "******"
}),
"EDX_NOTES_API_MYSQL_":
make_flat_group_info(
var_names=["DB_NAME", "DB_USER", "DB_PASS", "HOST"],
database={
"name": "edx_notes_api",
"user": "******"
},
include_port=False),
"ECOMMERCE_": {
"vars": ["DATABASES"],
"values": [{
"default": {
"ENGINE": 'django.db.backends.mysql',
"NAME": "{{ ECOMMERCE_DATABASE_NAME }}",
"USER": "******",
"PASSWORD": "******",
"HOST": "{{ ECOMMERCE_DATABASE_HOST }}",
"PORT": expected_port,
"ATOMIC_REQUESTS": True,
"CONN_MAX_AGE": 0
}
}]
},
"ECOMMERCE_DATABASE_":
make_flat_group_info(
var_names=["NAME", "USER", "PASSWORD", "HOST"],
database={
"name": "ecommerce",
"user": "******"
}),
"PROGRAMS_":
make_nested_group_info(["DEFAULT_DB_NAME", "DATABASES"], [{
"name": "programs",
"user": "******",
"additional_settings": {
"ATOMIC_REQUESTS": True,
"CONN_MAX_AGE": 0,
}
}]),
"INSIGHTS_":
make_nested_group_info(["DATABASE_NAME", "DATABASES"],
[{
"name": "dashboard",
"user": "******"
}]),
"ANALYTICS_API_":
make_nested_group_info(
["DEFAULT_DB_NAME", "REPORTS_DB_NAME", "DATABASES"
], [{
"name": "analytics_api",
"user": "******"
}, {
"id": "reports",
"name": "reports",
"user": "******"
}]),
}
for group_prefix, group_info in var_groups.items():
values = group_info["values"]
if "vars" in group_info:
self.check_vars(self.instance,
db_vars,
group_prefix,
var_names=group_info["vars"],
values=values)
else:
self.check_vars(self.instance,
db_vars,
group_prefix,
values=values)
class MySQLInstanceTestCase(TestCase):
"""
Test cases for MySQLInstanceMixin and OpenEdXDatabaseMixin
"""
def setUp(self):
super().setUp()
self.instance = None
def tearDown(self):
if self.instance:
self.instance.deprovision_mysql()
super().tearDown()
def _assert_privileges(self, database):
"""
Assert that relevant users can access database
"""
database_name = database["name"]
user = database["user"]
additional_users = [
user["name"] for user in database.get("additional_users", [])
]
global_users = [
self.instance.migrate_user, self.instance.read_only_user
]
users = [user] + additional_users + global_users
for user in users:
password = self.instance._get_mysql_pass(user)
# Pass password using MYSQL_PWD environment variable rather than the --password
# parameter so that mysql command doesn't print a security warning.
env = {'MYSQL_PWD': password}
mysql_cmd = "mysql -h 127.0.0.1 -u {user} -e 'SHOW TABLES' {db_name}".format(
user=user, db_name=database_name)
tables = subprocess.call(mysql_cmd, shell=True, env=env)
self.assertEqual(tables, 0)
def check_mysql(self):
"""
Check that the mysql databases and users have been created
"""
self.assertIs(self.instance.mysql_provisioned, True)
self.assertTrue(self.instance.mysql_user)
self.assertTrue(self.instance.mysql_pass)
databases = subprocess.check_output(
"mysql -h 127.0.0.1 -u root -e 'SHOW DATABASES'",
shell=True).decode()
for database in self.instance.mysql_databases:
# Check if database exists
database_name = database["name"]
self.assertIn(database_name, databases)
# Check if relevant users can access it
self._assert_privileges(database)
def check_mysql_vars_not_set(self, instance):
"""
Check that the given instance does not point to a mysql database
"""
db_vars_str = instance.get_database_settings()
for var in ('EDXAPP_MYSQL_USER', 'EDXAPP_MYSQL_PASSWORD',
'EDXAPP_MYSQL_HOST', 'EDXAPP_MYSQL_PORT',
'EDXAPP_MYSQL_DB_NAME', 'COMMON_MYSQL_MIGRATE_USER',
'COMMON_MYSQL_MIGRATE_PASS'):
self.assertNotIn(var, db_vars_str)
def check_common_users(self, instance, db_vars):
"""
Check that instance settings contain correct information about common users.
"""
self.assertEqual(db_vars['COMMON_MYSQL_MIGRATE_USER'],
instance.migrate_user)
self.assertEqual(db_vars['COMMON_MYSQL_MIGRATE_PASS'],
instance._get_mysql_pass(instance.migrate_user))
self.assertEqual(db_vars['COMMON_MYSQL_READ_ONLY_USER'],
instance.read_only_user)
self.assertEqual(db_vars['COMMON_MYSQL_READ_ONLY_PASS'],
instance._get_mysql_pass(instance.read_only_user))
self.assertEqual(db_vars['COMMON_MYSQL_ADMIN_USER'],
instance.admin_user)
self.assertEqual(db_vars['COMMON_MYSQL_ADMIN_PASS'],
instance._get_mysql_pass(instance.admin_user))
def check_vars(self,
instance,
db_vars,
prefix,
var_names=None,
values=None):
"""
Check that instance settings contain correct values for vars that start with prefix.
"""
if var_names is None:
var_names = ["DB_NAME", "USER", "PASSWORD", "HOST", "PORT"]
instance_settings = zip(var_names, values)
for var_name, value in instance_settings:
var_name = prefix + var_name
self.assertEqual(db_vars[var_name], value)
def test__get_mysql_database_name(self):
"""
Test that _get_mysql_database_name correctly builds database names.
"""
self.instance = OpenEdXInstanceFactory()
# Database name should be a combination of database_name and custom suffix
suffix = "test"
database_name = self.instance._get_mysql_database_name(suffix)
expected_database_name = "{0}_{1}".format(self.instance.database_name,
suffix)
self.assertEqual(database_name, expected_database_name)
# Using suffix that exceeds maximum length should raise an error
suffix = "long-long-long-long-long-long-long-long-long-long-long-long-suffix"
with self.assertRaises(AssertionError):
self.instance._get_mysql_database_name(suffix)
def test__get_mysql_user_name(self):
"""
Test that _get_mysql_user_name correctly builds user names.
"""
self.instance = OpenEdXInstanceFactory()
# User name should be a combination of mysql_user and custom suffix
suffix = "test"
user_name = self.instance._get_mysql_user_name(suffix)
expected_user_name = "{0}_{1}".format(self.instance.mysql_user, suffix)
self.assertEqual(user_name, expected_user_name)
# Using suffix that exceeds maximum length should raise an error
suffix = "long-long-long-suffix"
with self.assertRaises(AssertionError):
self.instance._get_mysql_user_name(suffix)
def test__get_mysql_pass(self):
"""
Test behavior of _get_mysql_pass.
It should:
- generate passwords of appropriate length
- generate different passwords for different users
- behave deterministically, i.e., return the same password for a given user
every time it is called with that user
"""
self.instance = OpenEdXInstanceFactory()
user1 = "user1"
pass1 = self.instance._get_mysql_pass(user1)
user2 = "user2"
pass2 = self.instance._get_mysql_pass(user2)
self.assertEqual(len(pass1), 64)
self.assertEqual(len(pass2), 64)
self.assertFalse(pass1 == pass2)
self.assertEqual(pass1, self.instance._get_mysql_pass(user1))
self.assertEqual(pass2, self.instance._get_mysql_pass(user2))
def test__get_mysql_pass_from_dbname(self):
"""
Test that _get_mysql_pass_from_dbname meets the same criteria as _get_mysql_pass
"""
self.instance = OpenEdXInstanceFactory()
database1 = "database1"
pass1 = self.instance._get_mysql_pass_from_dbname(database1)
database2 = "database2"
pass2 = self.instance._get_mysql_pass_from_dbname(database2)
self.assertEqual(len(pass1), 64)
self.assertEqual(len(pass2), 64)
self.assertFalse(pass1 == pass2)
self.assertEqual(pass1,
self.instance._get_mysql_pass_from_dbname(database1))
self.assertEqual(pass2,
self.instance._get_mysql_pass_from_dbname(database2))
def test_provision_mysql(self):
"""
Provision mysql database
"""
self.instance = OpenEdXInstanceFactory()
self.instance.provision_mysql()
self.check_mysql()
def test_provision_mysql_weird_domain(self):
"""
Make sure that database names are escaped correctly
"""
sub_domain = 'really.really.really.really.long.subdomain'
base_domain = 'this-is-a-really-unusual-domain-แปลกมาก.com'
internal_lms_domain = '{}.{}'.format(sub_domain, base_domain)
self.instance = OpenEdXInstanceFactory(
internal_lms_domain=internal_lms_domain)
self.instance.provision_mysql()
self.check_mysql()
def test_provision_mysql_again(self):
"""
Only create the database once
"""
self.instance = OpenEdXInstanceFactory()
self.instance.provision_mysql()
self.assertIs(self.instance.mysql_provisioned, True)
mysql_user = self.instance.mysql_user
mysql_pass = self.instance.mysql_pass
self.instance.provision_mysql()
self.assertEqual(self.instance.mysql_user, mysql_user)
self.assertEqual(self.instance.mysql_pass, mysql_pass)
self.check_mysql()
def test_provision_mysql_no_mysql_server(self):
"""
Don't provision a mysql database if instance has no MySQL server
"""
self.instance = OpenEdXInstanceFactory()
self.instance.mysql_server = None
self.instance.save()
self.instance.provision_mysql()
databases = subprocess.check_output(
"mysql -h 127.0.0.1 -u root -e 'SHOW DATABASES'",
shell=True).decode()
for database in self.instance.mysql_databases:
self.assertNotIn(database["name"], databases)
@patch_services
@override_settings(
DEFAULT_INSTANCE_MYSQL_URL='mysql://*****:*****@mysql.opencraft.com')
def test_ansible_settings_mysql(self, mocks):
"""
Test that get_database_settings produces correct settings for MySQL databases
"""
# Delete MySQLServer object created during the migrations to allow the settings override to
# take effect.
MySQLServer.objects.all().delete()
self.instance = OpenEdXInstanceFactory()
expected_host = "mysql.opencraft.com"
expected_port = MYSQL_SERVER_DEFAULT_PORT
def make_flat_group_info(var_names=None,
database=None,
include_port=True):
""" Return dict containing info for a flat group of variables """
group_info = {}
if var_names:
group_info["vars"] = var_names
# Compute and insert values
name = self.instance._get_mysql_database_name(database["name"])
user = self.instance._get_mysql_user_name(database["user"])
password = self.instance._get_mysql_pass(user)
values = [name, user, password, expected_host]
if include_port:
values.append(expected_port)
group_info["values"] = values
return group_info
def make_nested_group_info(var_names, databases):
""" Return dict containing info for a nested group of variables """
group_info = {"vars": var_names}
# Compute and insert values
for database in databases:
database["name"] = self.instance._get_mysql_database_name(
database["name"])
database["user"] = self.instance._get_mysql_user_name(
database["user"])
database["password"] = self.instance._get_mysql_pass(
database["user"])
values = [database["name"] for database in databases]
values.append({
database.get("id", "default"): dict(
ENGINE='django.db.backends.mysql',
NAME=database["name"],
USER=database["user"],
PASSWORD=database["password"],
HOST=expected_host,
PORT=expected_port,
**database.get("additional_settings", {}),
)
for database in databases
})
group_info["values"] = values
return group_info
# Load instance settings
db_vars = yaml.load(self.instance.get_database_settings())
# Check instance settings for common users
self.check_common_users(self.instance, db_vars)
# Check service-specific instance settings
var_groups = {
"EDXAPP_MYSQL_":
make_flat_group_info(database={
"name": "edxapp",
"user": "******"
}),
"XQUEUE_MYSQL_":
make_flat_group_info(database={
"name": "xqueue",
"user": "******"
}),
"EDXAPP_MYSQL_CSMH_":
make_flat_group_info(database={
"name": "edxapp_csmh",
"user": "******"
}),
"NOTIFIER_DATABASE_":
make_flat_group_info(
var_names=["NAME", "USER", "PASSWORD", "HOST", "PORT"],
database={
"name": "notifier",
"user": "******"
}),
"EDX_NOTES_API_MYSQL_":
make_flat_group_info(
var_names=["DB_NAME", "DB_USER", "DB_PASS", "HOST"],
database={
"name": "edx_notes_api",
"user": "******"
},
include_port=False),
"ECOMMERCE_": {
"vars": ["DATABASES"],
"values": [{
"default": {
"ENGINE": 'django.db.backends.mysql',
"NAME": "{{ ECOMMERCE_DATABASE_NAME }}",
"USER": "******",
"PASSWORD": "******",
"HOST": "{{ ECOMMERCE_DATABASE_HOST }}",
"PORT": expected_port,
"ATOMIC_REQUESTS": True,
"CONN_MAX_AGE": 0
}
}]
},
"ECOMMERCE_DATABASE_":
make_flat_group_info(
var_names=["NAME", "USER", "PASSWORD", "HOST"],
database={
"name": "ecommerce",
"user": "******"
}),
"PROGRAMS_":
make_nested_group_info(["DEFAULT_DB_NAME", "DATABASES"], [{
"name": "programs",
"user": "******",
"additional_settings": {
"ATOMIC_REQUESTS": True,
"CONN_MAX_AGE": 0,
}
}]),
"INSIGHTS_":
make_nested_group_info(["DATABASE_NAME", "DATABASES"],
[{
"name": "dashboard",
"user": "******"
}]),
"ANALYTICS_API_":
make_nested_group_info(
["DEFAULT_DB_NAME", "REPORTS_DB_NAME", "DATABASES"
], [{
"name": "analytics_api",
"user": "******"
}, {
"id": "reports",
"name": "reports",
"user": "******"
}]),
}
for group_prefix, group_info in var_groups.items():
values = group_info["values"]
if "vars" in group_info:
self.check_vars(self.instance,
db_vars,
group_prefix,
var_names=group_info["vars"],
values=values)
else:
self.check_vars(self.instance,
db_vars,
group_prefix,
values=values)
def test_ansible_settings_no_mysql_server(self):
"""
Don't add mysql ansible vars if instance has no MySQL server
"""
self.instance = OpenEdXInstanceFactory()
self.instance.mysql_server = None
self.instance.save()
self.check_mysql_vars_not_set(self.instance)
@patch('instance.models.mixins.database._get_mysql_cursor',
return_value=Mock())
@patch('instance.models.mixins.database._drop_database')
@patch('instance.models.mixins.database._drop_user')
def test_deprovision_mysql(self, mock_drop_user, mock_drop_database,
mock_get_cursor):
"""
Test deprovision_mysql does correct calls.
"""
self.instance = OpenEdXInstanceFactory()
self.instance.mysql_provisioned = True
self.instance.deprovision_mysql()
for database in self.instance.mysql_databases:
mock_drop_database.assert_any_call(mock_get_cursor(),
database["name"])
mock_drop_user.assert_any_call(mock_get_cursor(), database["user"])
for user in self.instance.global_users:
mock_drop_user.assert_any_call(mock_get_cursor(), user)
@patch('instance.models.mixins.database._get_mysql_cursor',
return_value=Mock())
@patch('instance.models.mixins.database._drop_database',
side_effect=MySQLError())
@patch('instance.models.mixins.database._drop_user')
def test_ignore_errors_deprovision_mysql(self, mock_drop_user,
mock_drop_database,
mock_get_cursor):
"""
Test mysql is set as deprovision when ignoring errors.
"""
self.instance = OpenEdXInstanceFactory()
self.instance.mysql_provisioned = True
self.instance.deprovision_mysql(ignore_errors=True)
self.assertFalse(self.instance.mysql_provisioned)
class RabbitMQInstanceTestCase(TestCase):
"""
Test cases for RabbitMQInstanceMixin
"""
def setUp(self):
super().setUp()
self.instance = OpenEdXInstanceFactory()
@responses.activate
@ddt.data(('GET', ['overview'], '/api/overview'),
('PUT', ['users', 'testuser'], '/api/users/testuser'),
('DELETE', ['permissions', '/some_vhost', 'testuser'
], '/api/permissions/%2Fsome_vhost/testuser'))
@ddt.unpack
def test_rabbitmq_request(self, method, url_parts, expected_url):
"""
Test to make sure the _rabbitmq_request parameters form the correct URLs
"""
url = '{service_url}{path}'.format(
service_url=self.instance.rabbitmq_server.api_url,
path=expected_url)
expected_body = {
'info': 'This is a mocked request to URL {url}'.format(url=url)
}
# Mock the URL with a uniquely identifying body so that we can verify that the
# correct URL is formed and called.
responses.add(method, url, json=expected_body)
self.instance = OpenEdXInstanceFactory()
response = self.instance._rabbitmq_request(method.lower(), *url_parts)
self.assertDictEqual(response.json(), expected_body)
@responses.activate
def test_provision_rabbitmq(self):
"""
Record the calls to the RabbitMQ API and make sure a new vhost along with
two new users are created during provision and deleted during deprobision.
The use of `responses.RequestsMock` raises an exception during context deconstruction
if any of the URLs added to the `responses` object aren't ever called. Also,
if any RabbitMQ API URLs are called that haven't been mocked, a `RabbitMQAPIError`
should be raised (given the default `.env.test` configuration).
So, this test should pass if and only if all of the specifically mocked URLs are
called during both provision and deprovision.
"""
rabbitmq_users = [
self.instance.rabbitmq_provider_user,
self.instance.rabbitmq_consumer_user
]
rabbitmq_vhost = urllib.parse.quote(self.instance.rabbitmq_vhost,
safe='')
vhosts_calls = ['vhosts/{}'.format(rabbitmq_vhost)]
users_calls = ['users/{}'.format(user) for user in rabbitmq_users]
permissions_calls = [
'permissions/{}/{}'.format(rabbitmq_vhost, user)
for user in rabbitmq_users
]
provision_calls = [
'{}/api/{}'.format(self.instance.rabbitmq_server.api_url, url)
for url in vhosts_calls + users_calls + permissions_calls
]
deprovision_calls = [
'{}/api/{}'.format(self.instance.rabbitmq_server.api_url, url)
for url in vhosts_calls + users_calls
]
# Spec the provisioning calls
with responses.RequestsMock() as rsps:
for url in provision_calls:
rsps.add(responses.PUT,
url,
content_type='application/json',
body='{}')
self.instance.provision_rabbitmq()
# Spec the deprovisioning calls
with responses.RequestsMock() as rsps:
for url in deprovision_calls:
rsps.add(responses.DELETE,
url,
content_type='application/json',
body='{}')
self.instance.deprovision_rabbitmq()
@responses.activate
def test_rabbitmq_api_error(self):
"""
Test that RabbitMQAPIError is thrown during auth issues
"""
with responses.RequestsMock() as rsps:
# Emulate 401 Unauthorized
rsps.add(responses.GET,
'{}/api/overview'.format(
self.instance.rabbitmq_server.api_url),
content_type='application/json',
body='{}',
status=401)
with self.assertRaises(RabbitMQAPIError):
self.instance._rabbitmq_request('get', 'overview')
@ddt.data(({
'name': 'test'
}, 'test'), ({
'name': 'test',
'description': 'test description'
}, 'test (test description)'))
@ddt.unpack
def test_string_representation(self, fields, representation):
"""
Test that the str method returns the appropriate values.
"""
rabbitmq = self.instance.rabbitmq_server
for name, value in fields.items():
setattr(rabbitmq, name, value)
rabbitmq.save()
self.assertEqual(str(rabbitmq), representation)
@patch(
'instance.models.mixins.rabbitmq.RabbitMQInstanceMixin._rabbitmq_request'
)
def test_deprovision_rabbitmq(self, mock_rabbitmq_request):
"""
Test deprovision_rabbitmq does correct calls.
"""
self.instance.rabbitmq_provisioned = True
self.instance.deprovision_rabbitmq()
mock_rabbitmq_request.assert_any_call('delete', 'vhosts',
self.instance.rabbitmq_vhost)
mock_rabbitmq_request.assert_any_call(
'delete', 'users', self.instance.rabbitmq_consumer_user.username)
mock_rabbitmq_request.assert_any_call(
'delete', 'users', self.instance.rabbitmq_provider_user.username)
@patch(
'instance.models.mixins.rabbitmq.RabbitMQInstanceMixin._rabbitmq_request',
side_effect=RabbitMQAPIError())
def test_ignore_errors_deprovision_rabbitmq(self, mock_rabbitmq_request):
"""
Test rabbitmq is set as deprovision when ignoring errors.
"""
self.instance.rabbitmq_provisioned = True
self.instance.deprovision_rabbitmq(ignore_errors=True)
self.assertFalse(self.instance.rabbitmq_provisioned)
def setUp(self):
super().setUp()
self.instance = OpenEdXInstanceFactory()
def test_colors_and_images_applied(self):
"""
Creates a beta application with asks for some colors and logo/favicon, and checks that the generated
ansible variables match those colors and images.
"""
# Create objects
OpenEdXInstanceFactory(name='Integration - test_colors_applied',
deploy_simpletheme=True)
instance = OpenEdXInstance.objects.get()
user = get_user_model().objects.create_user('betatestuser',
'*****@*****.**')
self.make_test_application(instance, user)
appserver = make_test_appserver(instance)
# Test the results
self.assertTrue(instance.deploy_simpletheme)
# We check 2 times: one time just the theme vars, next whether they're in the final list
ansible_theme_vars = instance.get_theme_settings()
ansible_vars = appserver.configuration_settings
for variables in (ansible_theme_vars, ansible_vars):
parsed_vars = yaml.load(variables) or {}
expected_settings = {
'SIMPLETHEME_ENABLE_DEPLOY':
True,
'SIMPLETHEME_SASS_OVERRIDES': [
{
'variable': 'link-color',
'value': '#003344',
},
# TODO: These are specific to Ginkgo and can be removed
# after Hawthorn upgrade
{
'variable': 'header-bg',
'value': '#caaffe',
},
{
'variable': 'footer-bg',
'value': '#ffff11',
},
# END TODO
{
'variable': 'button-color',
'value': '#001122',
},
{
'variable': 'action-primary-bg',
'value': '#001122',
},
{
'variable': 'action-secondary-bg',
'value': '#001122',
},
{
'variable': 'theme-colors',
'value': '("primary": #001122, "secondary": #001122)'
}
],
'EDXAPP_DEFAULT_SITE_THEME':
'simple-theme',
# for SIMPLETHEME_STATIC_FILES_URLS, see below
'SIMPLETHEME_EXTRA_SASS':
'''
.global-header {
background: #caaffe;
}
.wrapper-footer {
background: #ffff11;
}'''
}
for ansible_var, value in expected_settings.items():
self.assertEqual(value, parsed_vars[ansible_var])
# We check that the files are in URLs
# If this fails in local tests it can be because you don't have SWIFT upload enabled
# (check the .env or .env.test file for MEDIAFILES_SWIFT_ENABLE and login info)
files = parsed_vars['SIMPLETHEME_STATIC_FILES_URLS']
self.assertEqual(len(files), 2)
logo, favicon = files
self.assertEqual(logo['dest'], 'lms/static/images/logo.png')
self.assertEqual(favicon['dest'], 'lms/static/images/favicon.ico')
self.assertIn('opencraft_logo_small.png', logo['url'])
self.assertIn('favicon.ico', favicon['url'])
def test_rsa_key_creation(self, mock_consul):
"""
Test that we can produce public and private key pair for an
instance with a particular existing secret key.
"""
instance = OpenEdXInstanceFactory()
instance.secret_key_rsa_private = """-----BEGIN RSA PRIVATE KEY-----
MIICWwIBAAKBgQDEXX4rFLFl/eT3NJD8Y8rDcS39ynIjdYxaOHx6Q+PszU4YR6M0
3k3oMyDboIju6R8zim2JR9FWlOZTNN1MVSPOvu51CD4igNh5o+mgeBhVc+eatbvC
boDDtk/kHO0DEyebzO8oIortnh2pXF+Oyu3MdcyMFeF5xVEKqD0HQ9d05QIDAQAB
AoGAVDl9umC/zm1eXiHv5jGvcLEE9wx0dH0g3DnKOm8QPiu5SXTArhaD+AqmF03+
LetT9Ll1TiK9yZNIT3wnR2xlVLH6VuwcZ07KMUtvYLiuVGIAVf1TLs2E3zxRcrHb
TMsg15QnMFsat9yqMXSNPbqrs9tHU4hBv1k3uvkB4KWJVakCQQDjBYPaxmo8jyyJ
QzdLDh/cv20t4Q5LGB/XbfHTfJmnamToto6hEfG3Coy3/bbYhwFrWK53iKQpvkzg
vxNMlQoLAkEA3W4093Nai0V+YHVzI3fGqciSYrR4klYACqnIb+OlmLaT+Zj3Uv+d
P9BDy7frzRX9hYPQXMhYVxQBZtcF/CjCzwJADYUjkCDm7MpeDaKqJVcnAJ+J4gSY
NFKwesT6dOzjvbuxXMaage8upQcE0GRUwlpv9DOo2EeT90R1EaFvhc0OdwJAK7eV
d4Frz/FheRPXLpp4O48g76Hn6CRYj8Jjk0ujpxns7yt3MQjMeAvbRr5CLNR5oEGd
AqR/ZHnLqQ0s3lMB2wJAM3JaM2LtR3XhvQqT2vBteGB+iIWSh8cSxfWcd/vVSKIk
yF9iraiA2UvfpdwQSgXWsm7/+70kzVsb/MGl3rn63A==
-----END RSA PRIVATE KEY-----"""
instance.save()
jwk_key_pair = instance.get_jwk_key_pair()
self.assertEqual(
json.loads(jwk_key_pair.private), {
"n":
"xF1-KxSxZf3k9zSQ_GPKw3Et_cpyI3WMWjh8ekPj7M1OGEejNN5N6DMg"
"26CI7ukfM4ptiUfRVpTmUzTdTFUjzr7udQg-IoDYeaPpoHgYVXPnmrW7"
"wm6Aw7ZP5BztAxMnm8zvKCKK7Z4dqVxfjsrtzHXMjBXhecVRCqg9B0PX"
"dOU",
"kid":
"opencraft",
"kty":
"RSA",
"d":
"VDl9umC_zm1eXiHv5jGvcLEE9wx0dH0g3DnKOm8QPiu5SXTArhaD-Aqm"
"F03-LetT9Ll1TiK9yZNIT3wnR2xlVLH6VuwcZ07KMUtvYLiuVGIAVf1T"
"Ls2E3zxRcrHbTMsg15QnMFsat9yqMXSNPbqrs9tHU4hBv1k3uvkB4KWJ"
"Vak",
"p":
"4wWD2sZqPI8siUM3Sw4f3L9tLeEOSxgf123x03yZp2pk6LaOoRHxtwqM"
"t_222IcBa1iud4ikKb5M4L8TTJUKCw",
"q":
"3W4093Nai0V-YHVzI3fGqciSYrR4klYACqnIb-OlmLaT-Zj3Uv-dP9BD"
"y7frzRX9hYPQXMhYVxQBZtcF_CjCzw",
"e":
"AQAB"
})
self.assertEqual(
json.loads(jwk_key_pair.public),
{
"keys": [{
"e":
"AQAB",
"kty":
"RSA",
"kid":
"opencraft",
"n":
"xF1-KxSxZf3k9zSQ_GPKw3Et_cpyI3WMWjh8ekPj7M1OGEejNN5N"
"6DMg26CI7ukfM4ptiUfRVpTmUzTdTFUjzr7udQg-IoDYeaPpoHgY"
"VXPnmrW7wm6Aw7ZP5BztAxMnm8zvKCKK7Z4dqVxfjsrtzHXMjBXh"
"ecVRCqg9B0PXdOU"
}]
},
)
class MongoDBInstanceTestCase(TestCase):
"""
Test cases for MongoDBInstanceMixin and OpenEdXDatabaseMixin
"""
def setUp(self):
super().setUp()
self.instance = None
def tearDown(self):
if self.instance:
self.instance.deprovision_mongo()
super().tearDown()
def check_mongo(self):
"""
Check that the instance mongo user has access to the external mongo database
"""
mongo = pymongo.MongoClient(settings.DEFAULT_INSTANCE_MONGO_URL)
for database in self.instance.mongo_database_names:
self.assertTrue(mongo[database].authenticate(
self.instance.mongo_user, self.instance.mongo_pass))
def check_mongo_vars_not_set(self, appserver):
"""
Check that the given OpenEdXAppServer does not point to a mongo database
"""
for var in ('EDXAPP_MONGO_USER', 'EDXAPP_MONGO_PASSWORD'
'EDXAPP_MONGO_HOSTS', 'EDXAPP_MONGO_PORT',
'EDXAPP_MONGO_DB_NAME', 'FORUM_MONGO_USER',
'FORUM_MONGO_PASSWORD', 'FORUM_MONGO_HOSTS',
'FORUM_MONGO_PORT', 'FORUM_MONGO_DATABASE'):
self.assertNotIn(var, appserver.configuration_settings)
def check_mongo_vars_set(self,
appserver,
expected_hosts,
expected_replica_set=None):
"""
Check that the given OpenEdXAppServer is using the expected mongo settings.
"""
ansible_vars = appserver.configuration_settings
self.assertIn(
'EDXAPP_MONGO_USER: {0}'.format(self.instance.mongo_user),
ansible_vars)
self.assertIn(
'EDXAPP_MONGO_PASSWORD: {0}'.format(self.instance.mongo_pass),
ansible_vars)
self.assertIn(
'EDXAPP_MONGO_PORT: {0}'.format(MONGODB_SERVER_DEFAULT_PORT),
ansible_vars)
self.assertIn(
'EDXAPP_MONGO_DB_NAME: {0}'.format(
self.instance.mongo_database_name), ansible_vars)
# Use regex match, because sometimes the mongo hosts are unordered
self.assertRegex(ansible_vars,
r"EDXAPP_MONGO_HOSTS:\s*{0}\n".format(expected_hosts))
if expected_replica_set:
self.assertIn(
'EDXAPP_MONGO_REPLICA_SET: {0}'.format(expected_replica_set),
ansible_vars)
else:
self.assertNotIn('EDXAPP_MONGO_REPLICA_SET', ansible_vars)
self.assertIn('FORUM_MONGO_USER: {0}'.format(self.instance.mongo_user),
ansible_vars)
self.assertIn(
'FORUM_MONGO_PASSWORD: {0}'.format(self.instance.mongo_pass),
ansible_vars)
self.assertIn(
'FORUM_MONGO_PORT: {0}'.format(MONGODB_SERVER_DEFAULT_PORT),
ansible_vars)
self.assertIn(
'FORUM_MONGO_DATABASE: {0}'.format(
self.instance.forum_database_name), ansible_vars)
def test_provision_mongo(self):
"""
Provision mongo databases
"""
self.instance = OpenEdXInstanceFactory()
self.instance.provision_mongo()
self.check_mongo()
def test_provision_mongo_again(self):
"""
Only create the databases once
"""
self.instance = OpenEdXInstanceFactory()
self.instance.provision_mongo()
self.assertIs(self.instance.mongo_provisioned, True)
mongo_user = self.instance.mongo_user
mongo_pass = self.instance.mongo_pass
self.instance.provision_mongo()
self.assertEqual(self.instance.mongo_user, mongo_user)
self.assertEqual(self.instance.mongo_pass, mongo_pass)
self.check_mongo()
def test_provision_mongo_no_mongodb_server(self):
"""
Don't provision a mongo database if instance has no MongoDB server
"""
mongo = pymongo.MongoClient(settings.DEFAULT_INSTANCE_MONGO_URL)
self.instance = OpenEdXInstanceFactory()
self.instance.mongodb_server = None
self.instance.save()
self.instance.provision_mongo()
databases = mongo.database_names()
for database in self.instance.mongo_database_names:
self.assertNotIn(database, databases)
@override_settings(
DEFAULT_INSTANCE_MONGO_URL='mongodb://*****:*****@mongo.opencraft.com')
def test_ansible_settings_mongo(self):
"""
Add mongo ansible vars if instance has a MongoDB server
"""
# Delete MongoDBServer object created during the migrations to allow the settings override
# to take effect.
MongoDBServer.objects.all().delete()
self.instance = OpenEdXInstanceFactory()
appserver = make_test_appserver(self.instance)
self.check_mongo_vars_set(appserver,
expected_hosts='mongo.opencraft.com')
@override_settings(
DEFAULT_INSTANCE_MONGO_URL=None,
DEFAULT_MONGO_REPLICA_SET_NAME="test_name",
DEFAULT_MONGO_REPLICA_SET_USER="******",
DEFAULT_MONGO_REPLICA_SET_PASSWORD="******",
DEFAULT_MONGO_REPLICA_SET_PRIMARY="test.opencraft.hosting",
DEFAULT_MONGO_REPLICA_SET_HOSTS=
"test.opencraft.hosting,test1.opencraft.hosting,test2.opencraft.hosting"
)
@ddt.data(
('open-release/ficus', 'open-release/ficus'),
('open-release/ficus', 'opencraft-release/ficus'),
('open-release/ginkgo', 'open-release/ginkgo'),
)
@ddt.unpack
def test_ansible_settings_no_replica_set(self, openedx_release,
configuration_version):
"""
Prior to Hawthorn, edx configuration does not support MongoDB replica sets,
and the mongo hosts must be a single host, provided as a list of strings.
"""
# Delete MongoDBServer object created during the migrations to allow the settings override
# to take effect.
MongoDBServer.objects.all().delete()
self.instance = OpenEdXInstanceFactory(
openedx_release=openedx_release,
configuration_version=configuration_version)
appserver = make_test_appserver(self.instance)
self.check_mongo_vars_set(appserver,
expected_hosts="\n- test.opencraft.hosting")
@override_settings(
DEFAULT_INSTANCE_MONGO_URL=None,
DEFAULT_MONGO_REPLICA_SET_NAME="test_name",
DEFAULT_MONGO_REPLICA_SET_USER="******",
DEFAULT_MONGO_REPLICA_SET_PASSWORD="******",
DEFAULT_MONGO_REPLICA_SET_PRIMARY="test.opencraft.hosting",
DEFAULT_MONGO_REPLICA_SET_HOSTS=
"test.opencraft.hosting,test1.opencraft.hosting,test2.opencraft.hosting"
)
@ddt.data(
('open-release/ginkgo', 'opencraft-release/ginkgo'),
(settings.OPENEDX_RELEASE_STABLE_REF,
settings.STABLE_CONFIGURATION_VERSION),
(settings.DEFAULT_OPENEDX_RELEASE,
settings.DEFAULT_CONFIGURATION_VERSION),
)
@ddt.unpack
def test_ansible_settings_use_replica_set(self, openedx_release,
configuration_version):
"""
Add mongo ansible vars if instance has a MongoDB replica set
Also, the mongo hosts are provied as a comma-separated string.
"""
# Delete MongoDBServer object created during the migrations to allow the settings override
# to take effect.
MongoDBServer.objects.all().delete()
self.instance = OpenEdXInstanceFactory(
openedx_release=openedx_release,
configuration_version=configuration_version)
appserver = make_test_appserver(self.instance)
self.check_mongo_vars_set(appserver,
expected_hosts=r'test\d?.opencraft.hosting,'
r'test\d?.opencraft.hosting,'
r'test\d?.opencraft.hosting',
expected_replica_set='test_name')
def test_ansible_settings_no_mongo_server(self):
"""
Don't add mongo ansible vars if instance has no MongoDB server
"""
self.instance = OpenEdXInstanceFactory()
self.instance.mongodb_server = None
self.instance.save()
appserver = make_test_appserver(self.instance)
self.check_mongo_vars_not_set(appserver)
@override_settings(
DEFAULT_INSTANCE_MONGO_URL=None,
DEFAULT_MONGO_REPLICA_SET_NAME="test_name",
DEFAULT_MONGO_REPLICA_SET_USER="******",
DEFAULT_MONGO_REPLICA_SET_PASSWORD="******",
DEFAULT_MONGO_REPLICA_SET_PRIMARY="test.opencraft.hosting",
DEFAULT_MONGO_REPLICA_SET_HOSTS=
"test.opencraft.hosting,test1.opencraft.hosting,test2.opencraft.hosting"
)
def test__get_main_database_url(self):
"""
Main database url should be extracted from primary replica set MongoDBServer
"""
self.instance = OpenEdXInstanceFactory()
self.assertEqual(self.instance._get_main_database_url(),
"mongodb://*****:*****@test.opencraft.hosting")
@patch(
'instance.models.mixins.database.MongoDBInstanceMixin._get_main_database_url'
)
@patch('instance.models.mixins.database.pymongo.MongoClient',
autospec=True)
def test_deprovision_mongo(self, mock_mongo_client_cls,
mock_get_main_db_url):
"""
Test deprovision_mongo calls drop_database.
"""
self.instance = OpenEdXInstanceFactory()
self.instance.mongo_provisioned = True
self.instance.deprovision_mongo()
for database in self.instance.mongo_database_names:
mock_mongo_client_cls().drop_database.assert_any_call(database)
@patch('instance.models.mixins.database.pymongo.MongoClient',
autospec=True)
@patch(
'instance.models.mixins.database.MongoDBInstanceMixin._get_main_database_url'
)
@patch('instance.models.mixins.database.pymongo.MongoClient.drop_database',
side_effect=PyMongoError())
def test_ignore_errors_deprovision_mongo(self, mock_mongo_client_cls,
*mock_methods):
"""
Test mongo is set as deprovision when ignoring errors.
"""
self.instance = OpenEdXInstanceFactory()
self.instance.mongo_provisioned = True
self.instance.deprovision_mongo(ignore_errors=True)
self.assertFalse(self.instance.mongo_provisioned)