class MySQLInstanceTestCase(TestCase): """ Test cases for MySQLInstanceMixin and OpenEdXDatabaseMixin """ def setUp(self): super().setUp() self.instance = None def tearDown(self): if self.instance: with patch( 'instance.tests.models.factories.openedx_instance.OpenEdXInstance._write_metadata_to_consul', return_value=(1, True) ): self.instance.deprovision_mysql() super().tearDown() def _assert_privileges(self, database): """ Assert that relevant users can access database """ database_name = database["name"] user = database["user"] additional_users = [user["name"] for user in database.get("additional_users", [])] global_users = [self.instance.migrate_user, self.instance.read_only_user] users = [user] + additional_users + global_users for user in users: password = self.instance._get_mysql_pass(user) # Pass password using MYSQL_PWD environment variable rather than the --password # parameter so that mysql command doesn't print a security warning. env = {'MYSQL_PWD': password} mysql_cmd = "mysql -h 127.0.0.1 -u {user} -e 'SHOW TABLES' {db_name}".format(user=user, db_name=database_name) tables = subprocess.call(mysql_cmd, shell=True, env=env) self.assertEqual(tables, 0) def check_mysql(self): """ Check that the mysql databases and users have been created """ self.assertIs(self.instance.mysql_provisioned, True) self.assertTrue(self.instance.mysql_user) self.assertTrue(self.instance.mysql_pass) databases = subprocess.check_output("mysql -h 127.0.0.1 -u root -e 'SHOW DATABASES'", shell=True).decode() for database in self.instance.mysql_databases: # Check if database exists database_name = database["name"] self.assertIn(database_name, databases) # Check if relevant users can access it self._assert_privileges(database) def check_mysql_vars_not_set(self, instance): """ Check that the given instance does not point to a mysql database """ db_vars_str = instance.get_database_settings() for var in ('EDXAPP_MYSQL_USER', 'EDXAPP_MYSQL_PASSWORD', 'EDXAPP_MYSQL_HOST', 'EDXAPP_MYSQL_PORT', 'EDXAPP_MYSQL_DB_NAME', 'COMMON_MYSQL_MIGRATE_USER', 'COMMON_MYSQL_MIGRATE_PASS'): self.assertNotIn(var, db_vars_str) def check_common_users(self, instance, db_vars): """ Check that instance settings contain correct information about common users. """ self.assertEqual(db_vars['COMMON_MYSQL_MIGRATE_USER'], instance.migrate_user) self.assertEqual(db_vars['COMMON_MYSQL_MIGRATE_PASS'], instance._get_mysql_pass(instance.migrate_user)) self.assertEqual(db_vars['COMMON_MYSQL_READ_ONLY_USER'], instance.read_only_user) self.assertEqual(db_vars['COMMON_MYSQL_READ_ONLY_PASS'], instance._get_mysql_pass(instance.read_only_user)) self.assertEqual(db_vars['COMMON_MYSQL_ADMIN_USER'], instance.admin_user) self.assertEqual(db_vars['COMMON_MYSQL_ADMIN_PASS'], instance._get_mysql_pass(instance.admin_user)) def check_vars(self, instance, db_vars, prefix, var_names=None, values=None): """ Check that instance settings contain correct values for vars that start with prefix. """ if var_names is None: var_names = ["DB_NAME", "USER", "PASSWORD", "HOST", "PORT"] instance_settings = zip(var_names, values) for var_name, value in instance_settings: var_name = prefix + var_name self.assertEqual(db_vars[var_name], value) def test__get_mysql_database_name(self, mock_consul): """ Test that _get_mysql_database_name correctly builds database names. """ self.instance = OpenEdXInstanceFactory() # Database name should be a combination of database_name and custom suffix suffix = "test" database_name = self.instance._get_mysql_database_name(suffix) expected_database_name = "{0}_{1}".format(self.instance.database_name, suffix) self.assertEqual(database_name, expected_database_name) # Using suffix that exceeds maximum length should raise an error suffix = "long-long-long-long-long-long-long-long-long-long-long-long-suffix" with self.assertRaises(AssertionError): self.instance._get_mysql_database_name(suffix) def test__get_mysql_user_name(self, mock_consul): """ Test that _get_mysql_user_name correctly builds user names. """ self.instance = OpenEdXInstanceFactory() # User name should be a combination of mysql_user and custom suffix suffix = "test" user_name = self.instance._get_mysql_user_name(suffix) expected_user_name = "{0}_{1}".format(self.instance.mysql_user, suffix) self.assertEqual(user_name, expected_user_name) # Using suffix that exceeds maximum length should raise an error suffix = "long-long-long-suffix" with self.assertRaises(AssertionError): self.instance._get_mysql_user_name(suffix) def test__get_mysql_pass(self, mock_consul): """ Test behavior of _get_mysql_pass. It should: - generate passwords of appropriate length - generate different passwords for different users - behave deterministically, i.e., return the same password for a given user every time it is called with that user """ self.instance = OpenEdXInstanceFactory() user1 = "user1" pass1 = self.instance._get_mysql_pass(user1) user2 = "user2" pass2 = self.instance._get_mysql_pass(user2) self.assertEqual(len(pass1), 64) self.assertEqual(len(pass2), 64) self.assertFalse(pass1 == pass2) self.assertEqual(pass1, self.instance._get_mysql_pass(user1)) self.assertEqual(pass2, self.instance._get_mysql_pass(user2)) def test__get_mysql_pass_from_dbname(self, mock_consul): """ Test that _get_mysql_pass_from_dbname meets the same criteria as _get_mysql_pass """ self.instance = OpenEdXInstanceFactory() database1 = "database1" pass1 = self.instance._get_mysql_pass_from_dbname(database1) database2 = "database2" pass2 = self.instance._get_mysql_pass_from_dbname(database2) self.assertEqual(len(pass1), 64) self.assertEqual(len(pass2), 64) self.assertFalse(pass1 == pass2) self.assertEqual(pass1, self.instance._get_mysql_pass_from_dbname(database1)) self.assertEqual(pass2, self.instance._get_mysql_pass_from_dbname(database2)) def test_provision_mysql(self, mock_consul): """ Provision mysql database """ self.instance = OpenEdXInstanceFactory() self.instance.provision_mysql() self.check_mysql() def test_provision_mysql_weird_domain(self, mock_consul): """ Make sure that database names are escaped correctly """ sub_domain = 'really.really.really.really.long.subdomain' base_domain = 'this-is-a-really-unusual-domain-แปลกมาก.com' internal_lms_domain = '{}.{}'.format(sub_domain, base_domain) self.instance = OpenEdXInstanceFactory(internal_lms_domain=internal_lms_domain) self.instance.provision_mysql() self.check_mysql() def test_provision_mysql_again(self, mock_consul): """ Only create the database once """ self.instance = OpenEdXInstanceFactory() self.instance.provision_mysql() self.assertIs(self.instance.mysql_provisioned, True) mysql_user = self.instance.mysql_user mysql_pass = self.instance.mysql_pass self.instance.provision_mysql() self.assertEqual(self.instance.mysql_user, mysql_user) self.assertEqual(self.instance.mysql_pass, mysql_pass) self.check_mysql() def test_provision_mysql_no_mysql_server(self, mock_consul): """ Don't provision a mysql database if instance has no MySQL server """ self.instance = OpenEdXInstanceFactory() self.instance.mysql_server = None self.instance.save() self.instance.provision_mysql() databases = subprocess.check_output("mysql -h 127.0.0.1 -u root -e 'SHOW DATABASES'", shell=True).decode() for database in self.instance.mysql_databases: self.assertNotIn(database["name"], databases) @patch_services @override_settings(DEFAULT_INSTANCE_MYSQL_URL='mysql://*****:*****@mysql.opencraft.com') def test_ansible_settings_mysql(self, mocks, mock_consul): """ Test that get_database_settings produces correct settings for MySQL databases """ # Delete MySQLServer object created during the migrations to allow the settings override to # take effect. MySQLServer.objects.all().delete() self.instance = OpenEdXInstanceFactory() expected_host = "mysql.opencraft.com" expected_port = MYSQL_SERVER_DEFAULT_PORT def make_flat_group_info(var_names=None, database=None, include_port=True): """ Return dict containing info for a flat group of variables """ group_info = {} if var_names: group_info["vars"] = var_names # Compute and insert values name = self.instance._get_mysql_database_name(database["name"]) user = self.instance._get_mysql_user_name(database["user"]) password = self.instance._get_mysql_pass(user) values = [name, user, password, expected_host] if include_port: values.append(expected_port) group_info["values"] = values return group_info def make_nested_group_info(var_names, databases): """ Return dict containing info for a nested group of variables """ group_info = { "vars": var_names } # Compute and insert values for database in databases: database["name"] = self.instance._get_mysql_database_name(database["name"]) database["user"] = self.instance._get_mysql_user_name(database["user"]) database["password"] = self.instance._get_mysql_pass(database["user"]) values = [database["name"] for database in databases] values.append({ database.get("id", "default"): dict( ENGINE='django.db.backends.mysql', NAME=database["name"], USER=database["user"], PASSWORD=database["password"], HOST=expected_host, PORT=expected_port, **database.get("additional_settings", {}), ) for database in databases }) group_info["values"] = values return group_info # Load instance settings db_vars = yaml.load(self.instance.get_database_settings(), Loader=yaml.SafeLoader) # Check instance settings for common users self.check_common_users(self.instance, db_vars) # Check service-specific instance settings var_groups = { "EDXAPP_MYSQL_": make_flat_group_info(database={"name": "edxapp", "user": "******"}), "XQUEUE_MYSQL_": make_flat_group_info(database={"name": "xqueue", "user": "******"}), "EDXAPP_MYSQL_CSMH_": make_flat_group_info(database={"name": "edxapp_csmh", "user": "******"}), "NOTIFIER_DATABASE_": make_flat_group_info( var_names=["NAME", "USER", "PASSWORD", "HOST", "PORT"], database={"name": "notifier", "user": "******"} ), "EDX_NOTES_API_MYSQL_": make_flat_group_info( var_names=["DB_NAME", "DB_USER", "DB_PASS", "HOST"], database={"name": "edx_notes_api", "user": "******"}, include_port=False ), "ECOMMERCE_": { "vars": ["DATABASES"], "values": [{ "default": { "ENGINE": 'django.db.backends.mysql', "NAME": "{{ ECOMMERCE_DATABASE_NAME }}", "USER": "******", "PASSWORD": "******", "HOST": "{{ ECOMMERCE_DATABASE_HOST }}", "PORT": expected_port, "ATOMIC_REQUESTS": True, "CONN_MAX_AGE": 0 } }] }, "ECOMMERCE_DATABASE_": make_flat_group_info( var_names=["NAME", "USER", "PASSWORD", "HOST"], database={"name": "ecommerce", "user": "******"} ), "PROGRAMS_": make_nested_group_info( ["DEFAULT_DB_NAME", "DATABASES"], [{ "name": "programs", "user": "******", "additional_settings": { "ATOMIC_REQUESTS": True, "CONN_MAX_AGE": 0, } }] ), "INSIGHTS_": make_nested_group_info( ["DATABASE_NAME", "DATABASES"], [{"name": "dashboard", "user": "******"}] ), "ANALYTICS_API_": make_nested_group_info( ["DEFAULT_DB_NAME", "REPORTS_DB_NAME", "DATABASES"], [{"name": "analytics_api", "user": "******"}, {"id": "reports", "name": "reports", "user": "******"}] ), } for group_prefix, group_info in var_groups.items(): values = group_info["values"] if "vars" in group_info: self.check_vars(self.instance, db_vars, group_prefix, var_names=group_info["vars"], values=values) else: self.check_vars(self.instance, db_vars, group_prefix, values=values) def test_ansible_settings_no_mysql_server(self, mock_consul): """ Don't add mysql ansible vars if instance has no MySQL server """ self.instance = OpenEdXInstanceFactory() self.instance.mysql_server = None self.instance.save() self.check_mysql_vars_not_set(self.instance) @patch('instance.models.mixins.database._get_mysql_cursor', return_value=Mock()) @patch('instance.models.mixins.database._drop_database') @patch('instance.models.mixins.database._drop_user') def test_deprovision_mysql(self, mock_drop_user, mock_drop_database, mock_get_cursor, mock_consul): """ Test deprovision_mysql does correct calls. """ self.instance = OpenEdXInstanceFactory() self.instance.mysql_provisioned = True self.instance.deprovision_mysql() for database in self.instance.mysql_databases: mock_drop_database.assert_any_call(mock_get_cursor(), database["name"]) mock_drop_user.assert_any_call(mock_get_cursor(), database["user"]) for user in self.instance.global_users: mock_drop_user.assert_any_call(mock_get_cursor(), user) @patch('instance.models.mixins.database._get_mysql_cursor', return_value=Mock()) @patch('instance.models.mixins.database._drop_database', side_effect=MySQLError()) @patch('instance.models.mixins.database._drop_user') def test_ignore_errors_deprovision_mysql(self, mock_drop_user, mock_drop_database, mock_get_cursor, mock_consul): """ Test mysql is set as deprovision when ignoring errors. """ self.instance = OpenEdXInstanceFactory() self.instance.mysql_provisioned = True self.instance.deprovision_mysql(ignore_errors=True) self.assertFalse(self.instance.mysql_provisioned)