class MySQLInstanceTestCase(TestCase): """ Test cases for MySQLInstanceMixin and OpenEdXDatabaseMixin """ def setUp(self): super().setUp() self.instance = None def tearDown(self): if self.instance: self.instance.deprovision_mysql() super().tearDown() def _assert_privileges(self, database): """ Assert that relevant users can access database """ database_name = database["name"] user = database["user"] additional_users = [user["name"] for user in database.get("additional_users", [])] global_users = [self.instance.migrate_user, self.instance.read_only_user] users = [user] + additional_users + global_users for user in users: password = self.instance._get_mysql_pass(user) # Pass password using MYSQL_PWD environment variable rather than the --password # parameter so that mysql command doesn't print a security warning. env = {'MYSQL_PWD': password} mysql_cmd = "mysql -u {user} -e 'SHOW TABLES' {db_name}".format(user=user, db_name=database_name) tables = subprocess.call(mysql_cmd, shell=True, env=env) self.assertEqual(tables, 0) def check_mysql(self): """ Check that the mysql databases and users have been created """ self.assertIs(self.instance.mysql_provisioned, True) self.assertTrue(self.instance.mysql_user) self.assertTrue(self.instance.mysql_pass) databases = subprocess.check_output("mysql -u root -e 'SHOW DATABASES'", shell=True).decode() for database in self.instance.mysql_databases: # Check if database exists database_name = database["name"] self.assertIn(database_name, databases) # Check if relevant users can access it self._assert_privileges(database) def check_mysql_vars_not_set(self, instance): """ Check that the given instance does not point to a mysql database """ db_vars_str = instance.get_database_settings() for var in ('EDXAPP_MYSQL_USER', 'EDXAPP_MYSQL_PASSWORD', 'EDXAPP_MYSQL_HOST', 'EDXAPP_MYSQL_PORT', 'EDXAPP_MYSQL_DB_NAME', 'COMMON_MYSQL_MIGRATE_USER', 'COMMON_MYSQL_MIGRATE_PASS'): self.assertNotIn(var, db_vars_str) def check_common_users(self, instance, db_vars): """ Check that instance settings contain correct information about common users. """ self.assertEqual(db_vars['COMMON_MYSQL_MIGRATE_USER'], instance.migrate_user) self.assertEqual(db_vars['COMMON_MYSQL_MIGRATE_PASS'], instance._get_mysql_pass(instance.migrate_user)) self.assertEqual(db_vars['COMMON_MYSQL_READ_ONLY_USER'], instance.read_only_user) self.assertEqual(db_vars['COMMON_MYSQL_READ_ONLY_PASS'], instance._get_mysql_pass(instance.read_only_user)) self.assertEqual(db_vars['COMMON_MYSQL_ADMIN_USER'], instance.admin_user) self.assertEqual(db_vars['COMMON_MYSQL_ADMIN_PASS'], instance._get_mysql_pass(instance.admin_user)) def check_vars(self, instance, db_vars, prefix, var_names=None, values=None): """ Check that instance settings contain correct values for vars that start with prefix. """ if var_names is None: var_names = ["DB_NAME", "USER", "PASSWORD", "HOST", "PORT"] instance_settings = zip(var_names, values) for var_name, value in instance_settings: var_name = prefix + var_name self.assertEqual(db_vars[var_name], value) def test__get_mysql_database_name(self): """ Test that _get_mysql_database_name correctly builds database names. """ self.instance = OpenEdXInstanceFactory() # Database name should be a combination of database_name and custom suffix suffix = "test" database_name = self.instance._get_mysql_database_name(suffix) expected_database_name = "{0}_{1}".format(self.instance.database_name, suffix) self.assertEqual(database_name, expected_database_name) # Using suffix that exceeds maximum length should raise an error suffix = "long-long-long-long-long-long-long-long-long-long-long-long-suffix" with self.assertRaises(AssertionError): self.instance._get_mysql_database_name(suffix) def test__get_mysql_user_name(self): """ Test that _get_mysql_user_name correctly builds user names. """ self.instance = OpenEdXInstanceFactory() # User name should be a combination of mysql_user and custom suffix suffix = "test" user_name = self.instance._get_mysql_user_name(suffix) expected_user_name = "{0}_{1}".format(self.instance.mysql_user, suffix) self.assertEqual(user_name, expected_user_name) # Using suffix that exceeds maximum length should raise an error suffix = "long-long-long-suffix" with self.assertRaises(AssertionError): self.instance._get_mysql_user_name(suffix) def test__get_mysql_pass(self): """ Test behavior of _get_mysql_pass. It should: - generate passwords of appropriate length - generate different passwords for different users - behave deterministically, i.e., return the same password for a given user every time it is called with that user """ self.instance = OpenEdXInstanceFactory() user1 = "user1" pass1 = self.instance._get_mysql_pass(user1) user2 = "user2" pass2 = self.instance._get_mysql_pass(user2) self.assertEqual(len(pass1), 64) self.assertEqual(len(pass2), 64) self.assertFalse(pass1 == pass2) self.assertEqual(pass1, self.instance._get_mysql_pass(user1)) self.assertEqual(pass2, self.instance._get_mysql_pass(user2)) def test__get_database_suffix(self): """ Test that _get_database_suffix returns correct suffix for a given database. """ self.instance = OpenEdXInstanceFactory() suffix = "test" database_name = self.instance._get_mysql_database_name(suffix) self.assertEqual(self.instance._get_database_suffix(database_name), suffix) def test__get_template_vars(self): """ Test that _get_template_vars returns correct settings for a given database. """ self.instance = OpenEdXInstanceFactory() suffix = "test-db" database = { "name": self.instance._get_mysql_database_name(suffix), "user": self.instance._get_mysql_user_name(suffix), } template_vars = self.instance._get_template_vars(database) expected_template_vars = { "{}_database".format(suffix): database["name"], "{}_user".format(suffix): database["user"], "{}_pass".format(suffix): self.instance._get_mysql_pass(database["user"]) } self.assertEqual(template_vars, expected_template_vars) def test_provision_mysql(self): """ Provision mysql database """ self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False) self.instance.provision_mysql() self.check_mysql() def test_provision_mysql_weird_domain(self): """ Make sure that database names are escaped correctly """ sub_domain = 'really.really.really.really.long.subdomain' base_domain = 'this-is-a-really-long-unusual-domain-แปลกมาก.com' internal_lms_domain = '{}.{}'.format(sub_domain, base_domain) self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False, internal_lms_domain=internal_lms_domain) self.instance.provision_mysql() self.check_mysql() def test_provision_mysql_again(self): """ Only create the database once """ self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False) self.instance.provision_mysql() self.assertIs(self.instance.mysql_provisioned, True) mysql_user = self.instance.mysql_user mysql_pass = self.instance.mysql_pass self.instance.provision_mysql() self.assertEqual(self.instance.mysql_user, mysql_user) self.assertEqual(self.instance.mysql_pass, mysql_pass) self.check_mysql() def test_provision_mysql_no_mysql_server(self): """ Don't provision a mysql database if instance has no MySQL server """ self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False) self.instance.mysql_server = None self.instance.save() self.instance.provision_mysql() databases = subprocess.check_output("mysql -u root -e 'SHOW DATABASES'", shell=True).decode() for database in self.instance.mysql_databases: self.assertNotIn(database["name"], databases) @patch_services @override_settings(DEFAULT_INSTANCE_MYSQL_URL='mysql://*****:*****@mysql.opencraft.com') def test_ansible_settings_mysql(self, mocks): """ Test that get_database_settings produces correct settings for MySQL databases """ # Delete MySQLServer object created during the migrations to allow the settings override to # take effect. MySQLServer.objects.all().delete() self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False) expected_host = "mysql.opencraft.com" expected_port = MYSQL_SERVER_DEFAULT_PORT def make_flat_group_info(var_names=None, database=None, include_port=True): """ Return dict containing info for a flat group of variables """ group_info = {} if var_names: group_info["vars"] = var_names # Compute and insert values name = self.instance._get_mysql_database_name(database["name"]) user = self.instance._get_mysql_user_name(database["user"]) password = self.instance._get_mysql_pass(user) values = [name, user, password, expected_host] if include_port: values.append(expected_port) group_info["values"] = values return group_info def make_nested_group_info(var_names, databases): """ Return dict containing info for a nested group of variables """ group_info = { "vars": var_names } # Compute and insert values for database in databases: database["name"] = self.instance._get_mysql_database_name(database["name"]) database["user"] = self.instance._get_mysql_user_name(database["user"]) database["password"] = self.instance._get_mysql_pass(database["user"]) values = [database["name"] for database in databases] values.append({ database.get("id", "default"): dict( ENGINE='django.db.backends.mysql', NAME=database["name"], USER=database["user"], PASSWORD=database["password"], HOST=expected_host, PORT=expected_port, **database.get("additional_settings", {}), ) for database in databases }) group_info["values"] = values return group_info # Load instance settings db_vars = yaml.load(self.instance.get_database_settings()) # Check instance settings for common users self.check_common_users(self.instance, db_vars) # Check service-specific instance settings var_groups = { "EDXAPP_MYSQL_": make_flat_group_info(database={"name": "edxapp", "user": "******"}), "XQUEUE_MYSQL_": make_flat_group_info(database={"name": "xqueue", "user": "******"}), "EDXAPP_MYSQL_CSMH_": make_flat_group_info(database={"name": "edxapp_csmh", "user": "******"}), "NOTIFIER_DATABASE_": make_flat_group_info( var_names=["NAME", "USER", "PASSWORD", "HOST", "PORT"], database={"name": "notifier", "user": "******"} ), "EDX_NOTES_API_MYSQL_": make_flat_group_info( var_names=["DB_NAME", "DB_USER", "DB_PASS", "HOST"], database={"name": "edx_notes_api", "user": "******"}, include_port=False ), "ECOMMERCE_": make_nested_group_info( ["DEFAULT_DB_NAME", "DATABASES"], [{"name": "ecommerce", "user": "******", "additional_settings": { "ATOMIC_REQUESTS": True, "CONN_MAX_AGE": 60, }}] ), "PROGRAMS_": make_nested_group_info( ["DEFAULT_DB_NAME", "DATABASES"], [{"name": "programs", "user": "******", "additional_settings": { "ATOMIC_REQUESTS": True, "CONN_MAX_AGE": 60, }}] ), "INSIGHTS_": make_nested_group_info( ["DATABASE_NAME", "DATABASES"], [{"name": "dashboard", "user": "******"}] ), "ANALYTICS_API_": make_nested_group_info( ["DEFAULT_DB_NAME", "REPORTS_DB_NAME", "DATABASES"], [{"name": "analytics_api", "user": "******"}, {"id": "reports", "name": "reports", "user": "******"}] ), } for group_prefix, group_info in var_groups.items(): values = group_info["values"] if "vars" in group_info: self.check_vars(self.instance, db_vars, group_prefix, var_names=group_info["vars"], values=values) else: self.check_vars(self.instance, db_vars, group_prefix, values=values) def test_ansible_settings_no_mysql_server(self): """ Don't add mysql ansible vars if instance has no MySQL server """ self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False) self.instance.mysql_server = None self.instance.save() self.check_mysql_vars_not_set(self.instance) @override_settings(DEFAULT_INSTANCE_MYSQL_URL='mysql://*****:*****@mysql.opencraft.com') def test_ansible_settings_mysql_ephemeral(self): """ Don't add mysql ansible vars for ephemeral databases """ self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=True) self.check_mysql_vars_not_set(self.instance)
class MySQLInstanceTestCase(TestCase): """ Test cases for MySQLInstanceMixin and OpenEdXDatabaseMixin """ def setUp(self): super().setUp() self.instance = None def tearDown(self): if self.instance: self.instance.deprovision_mysql() super().tearDown() def _assert_privileges(self, database): """ Assert that relevant users can access database """ database_name = database["name"] user = database["user"] additional_users = [user["name"] for user in database.get("additional_users", [])] global_users = [self.instance.migrate_user, self.instance.read_only_user] users = [user] + additional_users + global_users for user in users: password = self.instance._get_mysql_pass(user) # Pass password using MYSQL_PWD environment variable rather than the --password # parameter so that mysql command doesn't print a security warning. env = {'MYSQL_PWD': password} mysql_cmd = "mysql -u {user} -e 'SHOW TABLES' {db_name}".format(user=user, db_name=database_name) tables = subprocess.call(mysql_cmd, shell=True, env=env) self.assertEqual(tables, 0) def check_mysql(self): """ Check that the mysql databases and users have been created """ self.assertIs(self.instance.mysql_provisioned, True) self.assertTrue(self.instance.mysql_user) self.assertTrue(self.instance.mysql_pass) databases = subprocess.check_output("mysql -u root -e 'SHOW DATABASES'", shell=True).decode() for database in self.instance.mysql_databases: # Check if database exists database_name = database["name"] self.assertIn(database_name, databases) # Check if relevant users can access it self._assert_privileges(database) def check_mysql_vars_not_set(self, instance): """ Check that the given instance does not point to a mysql database """ db_vars_str = instance.get_database_settings() for var in ('EDXAPP_MYSQL_USER', 'EDXAPP_MYSQL_PASSWORD', 'EDXAPP_MYSQL_HOST', 'EDXAPP_MYSQL_PORT', 'EDXAPP_MYSQL_DB_NAME', 'COMMON_MYSQL_MIGRATE_USER', 'COMMON_MYSQL_MIGRATE_PASS'): self.assertNotIn(var, db_vars_str) def check_common_users(self, instance, db_vars): """ Check that instance settings contain correct information about common users. """ self.assertEqual(db_vars['COMMON_MYSQL_MIGRATE_USER'], instance.migrate_user) self.assertEqual(db_vars['COMMON_MYSQL_MIGRATE_PASS'], instance._get_mysql_pass(instance.migrate_user)) self.assertEqual(db_vars['COMMON_MYSQL_READ_ONLY_USER'], instance.read_only_user) self.assertEqual(db_vars['COMMON_MYSQL_READ_ONLY_PASS'], instance._get_mysql_pass(instance.read_only_user)) self.assertEqual(db_vars['COMMON_MYSQL_ADMIN_USER'], instance.admin_user) self.assertEqual(db_vars['COMMON_MYSQL_ADMIN_PASS'], instance._get_mysql_pass(instance.admin_user)) def check_vars(self, instance, db_vars, prefix, var_names=None, values=None): """ Check that instance settings contain correct values for vars that start with prefix. """ if var_names is None: var_names = ["DB_NAME", "USER", "PASSWORD", "HOST", "PORT"] instance_settings = zip(var_names, values) for var_name, value in instance_settings: var_name = prefix + var_name self.assertEqual(db_vars[var_name], value) def test__get_mysql_database_name(self): """ Test that _get_mysql_database_name correctly builds database names. """ self.instance = OpenEdXInstanceFactory() # Database name should be a combination of database_name and custom suffix suffix = "test" database_name = self.instance._get_mysql_database_name(suffix) expected_database_name = "{0}_{1}".format(self.instance.database_name, suffix) self.assertEqual(database_name, expected_database_name) # Using suffix that exceeds maximum length should raise an error suffix = "long-long-long-long-long-long-long-long-long-long-long-long-suffix" with self.assertRaises(AssertionError): self.instance._get_mysql_database_name(suffix) def test__get_mysql_user_name(self): """ Test that _get_mysql_user_name correctly builds user names. """ self.instance = OpenEdXInstanceFactory() # User name should be a combination of mysql_user and custom suffix suffix = "test" user_name = self.instance._get_mysql_user_name(suffix) expected_user_name = "{0}_{1}".format(self.instance.mysql_user, suffix) self.assertEqual(user_name, expected_user_name) # Using suffix that exceeds maximum length should raise an error suffix = "long-long-long-suffix" with self.assertRaises(AssertionError): self.instance._get_mysql_user_name(suffix) def test__get_mysql_pass(self): """ Test behavior of _get_mysql_pass. It should: - generate passwords of appropriate length - generate different passwords for different users - behave deterministically, i.e., return the same password for a given user every time it is called with that user """ self.instance = OpenEdXInstanceFactory() user1 = "user1" pass1 = self.instance._get_mysql_pass(user1) user2 = "user2" pass2 = self.instance._get_mysql_pass(user2) self.assertEqual(len(pass1), 64) self.assertEqual(len(pass2), 64) self.assertFalse(pass1 == pass2) self.assertEqual(pass1, self.instance._get_mysql_pass(user1)) self.assertEqual(pass2, self.instance._get_mysql_pass(user2)) def test__get_database_suffix(self): """ Test that _get_database_suffix returns correct suffix for a given database. """ self.instance = OpenEdXInstanceFactory() suffix = "test" database_name = self.instance._get_mysql_database_name(suffix) self.assertEqual(self.instance._get_database_suffix(database_name), suffix) def test__get_template_vars(self): """ Test that _get_template_vars returns correct settings for a given database. """ self.instance = OpenEdXInstanceFactory() suffix = "test-db" database = { "name": self.instance._get_mysql_database_name(suffix), "user": self.instance._get_mysql_user_name(suffix), } template_vars = self.instance._get_template_vars(database) expected_template_vars = { "{}_database".format(suffix): database["name"], "{}_user".format(suffix): database["user"], "{}_pass".format(suffix): self.instance._get_mysql_pass(database["user"]) } self.assertEqual(template_vars, expected_template_vars) @override_settings(DEFAULT_INSTANCE_MYSQL_URL=None) def test_create_instance_no_default_mysql_url(self): """ Test that creating an instance with persistent databases raises an exception if DEFAULT_INSTANCE_MYSQL_URL not set. """ with self.assertRaises(MySQLServer.DoesNotExist): self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False) def test_provision_mysql(self): """ Provision mysql database """ self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False) self.instance.provision_mysql() self.check_mysql() def test_provision_mysql_weird_domain(self): """ Make sure that database names are escaped correctly """ sub_domain = 'really.really.really.really.long.subdomain' base_domain = 'this-is-a-really-long-unusual-domain-แปลกมาก.com' internal_lms_domain = '{}.{}'.format(sub_domain, base_domain) self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False, internal_lms_domain=internal_lms_domain) self.instance.provision_mysql() self.check_mysql() def test_provision_mysql_again(self): """ Only create the database once """ self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False) self.instance.provision_mysql() self.assertIs(self.instance.mysql_provisioned, True) mysql_user = self.instance.mysql_user mysql_pass = self.instance.mysql_pass self.instance.provision_mysql() self.assertEqual(self.instance.mysql_user, mysql_user) self.assertEqual(self.instance.mysql_pass, mysql_pass) self.check_mysql() def test_provision_mysql_no_mysql_server(self): """ Don't provision a mysql database if instance has no MySQL server """ self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False) self.instance.mysql_server = None self.instance.save() self.instance.provision_mysql() databases = subprocess.check_output("mysql -u root -e 'SHOW DATABASES'", shell=True).decode() for database in self.instance.mysql_databases: self.assertNotIn(database["name"], databases) @patch_services @override_settings(DEFAULT_INSTANCE_MYSQL_URL='mysql://*****:*****@mysql.opencraft.com') def test_ansible_settings_mysql(self, mocks): """ Test that get_database_settings produces correct settings for MySQL databases """ self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False) expected_host = "mysql.opencraft.com" expected_port = MYSQL_SERVER_DEFAULT_PORT def make_flat_group_info(var_names=None, database=None, include_port=True): """ Return dict containing info for a flat group of variables """ group_info = {} if var_names: group_info["vars"] = var_names # Compute and insert values name = self.instance._get_mysql_database_name(database["name"]) user = self.instance._get_mysql_user_name(database["user"]) password = self.instance._get_mysql_pass(user) values = [name, user, password, expected_host] if include_port: values.append(expected_port) group_info["values"] = values return group_info def make_nested_group_info(var_names, databases): """ Return dict containing info for a nested group of variables """ group_info = { "vars": var_names } # Compute and insert values for database in databases: database["name"] = self.instance._get_mysql_database_name(database["name"]) database["user"] = self.instance._get_mysql_user_name(database["user"]) database["password"] = self.instance._get_mysql_pass(database["user"]) values = [database["name"] for database in databases] values.append({ database.get("id", "default"): dict( ENGINE='django.db.backends.mysql', NAME=database["name"], USER=database["user"], PASSWORD=database["password"], HOST=expected_host, PORT=expected_port, **database.get("additional_settings", {}), ) for database in databases }) group_info["values"] = values return group_info # Load instance settings db_vars = yaml.load(self.instance.get_database_settings()) # Check instance settings for common users self.check_common_users(self.instance, db_vars) # Check service-specific instance settings var_groups = { "EDXAPP_MYSQL_": make_flat_group_info(database={"name": "edxapp", "user": "******"}), "XQUEUE_MYSQL_": make_flat_group_info(database={"name": "xqueue", "user": "******"}), "EDXAPP_MYSQL_CSMH_": make_flat_group_info(database={"name": "edxapp_csmh", "user": "******"}), "EDX_NOTES_API_MYSQL_": make_flat_group_info( var_names=["DB_NAME", "DB_USER", "DB_PASS", "HOST"], database={"name": "edx_notes_api", "user": "******"}, include_port=False ), "ECOMMERCE_": make_nested_group_info( ["DEFAULT_DB_NAME", "DATABASES"], [{"name": "ecommerce", "user": "******", "additional_settings": { "ATOMIC_REQUESTS": True, "CONN_MAX_AGE": 60, }}] ), "INSIGHTS_": make_nested_group_info( ["DATABASE_NAME", "DATABASES"], [{"name": "dashboard", "user": "******"}] ), "ANALYTICS_API_": make_nested_group_info( ["DEFAULT_DB_NAME", "REPORTS_DB_NAME", "DATABASES"], [{"name": "analytics_api", "user": "******"}, {"id": "reports", "name": "reports", "user": "******"}] ), } for group_prefix, group_info in var_groups.items(): values = group_info["values"] if "vars" in group_info: self.check_vars(self.instance, db_vars, group_prefix, var_names=group_info["vars"], values=values) else: self.check_vars(self.instance, db_vars, group_prefix, values=values) def test_ansible_settings_no_mysql_server(self): """ Don't add mysql ansible vars if instance has no MySQL server """ self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=False) self.instance.mysql_server = None self.instance.save() self.check_mysql_vars_not_set(self.instance) @override_settings(DEFAULT_INSTANCE_MYSQL_URL='mysql://*****:*****@mysql.opencraft.com') def test_ansible_settings_mysql_ephemeral(self): """ Don't add mysql ansible vars for ephemeral databases """ self.instance = OpenEdXInstanceFactory(use_ephemeral_databases=True) self.check_mysql_vars_not_set(self.instance)