class GDPRScrubUserFromFormsCouchTests(TestCase): def setUp(self): super(GDPRScrubUserFromFormsCouchTests, self).setUp() self.db = TemporaryFilesystemBlobDB() def tearDown(self): self.db.close() super(GDPRScrubUserFromFormsCouchTests, self).tearDown() def test_modify_attachment_xml_and_metadata_couch(self): form = get_simple_wrapped_form( uuid.uuid4().hex, metadata=TestFormMetadata(domain=DOMAIN), simple_form=GDPR_SIMPLE_FORM) new_form_xml = Command().update_form_data(form, NEW_USERNAME) FormAccessors(DOMAIN).modify_attachment_xml_and_metadata( form, new_form_xml, NEW_USERNAME) # Test that the metadata changed in the database actual_form_xml = form.get_attachment("form.xml").decode('utf-8') self.assertXMLEqual(EXPECTED_FORM_XML, actual_form_xml) # Test that the operations history is updated in this form refetched_form = FormAccessors(DOMAIN).get_form(form.form_id) self.assertEqual(len(refetched_form.history), 1) self.assertEqual(refetched_form.history[0].operation, "gdpr_scrub") self.assertEqual(refetched_form.metadata.username, NEW_USERNAME)
class DeleteAttachmentsFSDBTests(TestCase): def setUp(self): super(DeleteAttachmentsFSDBTests, self).setUp() self.db = TemporaryFilesystemBlobDB() def tearDown(self): self.db.close() super(DeleteAttachmentsFSDBTests, self).tearDown() def test_hard_delete_forms_and_attachments(self): forms = [create_form_for_test(DOMAIN) for i in range(3)] form_ids = [form.form_id for form in forms] forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(3, len(forms)) other_form = create_form_for_test('other_domain') self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id])) attachments = list(FormAccessorSQL.get_attachments_for_forms(form_ids, ordered=True)) self.assertEqual(3, len(attachments)) deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id]) self.assertEqual(2, deleted) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(1, len(forms)) self.assertEqual(form_ids[0], forms[0].form_id) for attachment in attachments[1:]: with self.assertRaises(AttachmentNotFound): attachment.read_content() self.assertIsNotNone(attachments[0].read_content()) other_form = FormAccessorSQL.get_form(other_form.form_id) self.assertIsNotNone(other_form.get_xml())
class GDPRScrubUserFromFormsCouchTests(TestCase): def setUp(self): super(GDPRScrubUserFromFormsCouchTests, self).setUp() self.db = TemporaryFilesystemBlobDB() def tearDown(self): self.db.close() super(GDPRScrubUserFromFormsCouchTests, self).tearDown() def test_modify_attachment_xml_and_metadata_couch(self): form = get_simple_wrapped_form(uuid.uuid4().hex, metadata=TestFormMetadata(domain=DOMAIN), simple_form=GDPR_SIMPLE_FORM) new_form_xml = Command().update_form_data(form, NEW_USERNAME) FormAccessors(DOMAIN).modify_attachment_xml_and_metadata(form, new_form_xml, NEW_USERNAME) # Test that the metadata changed in the database actual_form_xml = form.get_attachment("form.xml").decode('utf-8') self.assertXMLEqual(EXPECTED_FORM_XML, actual_form_xml) # Test that the operations history is updated in this form refetched_form = FormAccessors(DOMAIN).get_form(form.form_id) self.assertEqual(len(refetched_form.history), 1) self.assertEqual(refetched_form.history[0].operation, "gdpr_scrub") self.assertEqual(refetched_form.metadata.username, NEW_USERNAME)
class DeleteAttachmentsFSDBTests(TestCase): def setUp(self): super(DeleteAttachmentsFSDBTests, self).setUp() self.db = TemporaryFilesystemBlobDB() def tearDown(self): self.db.close() super(DeleteAttachmentsFSDBTests, self).tearDown() def test_hard_delete_forms_and_attachments(self): forms = [create_form_for_test(DOMAIN) for i in range(3)] form_ids = sorted(form.form_id for form in forms) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(3, len(forms)) other_form = create_form_for_test('other_domain') self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms( 'other_domain', [other_form.form_id])) attachments = sorted(get_blob_db().metadb.get_for_parents(form_ids), key=lambda meta: meta.parent_id) self.assertEqual(3, len(attachments)) deleted = FormAccessorSQL.hard_delete_forms( DOMAIN, form_ids[1:] + [other_form.form_id]) self.assertEqual(2, deleted) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(1, len(forms)) self.assertEqual(form_ids[0], forms[0].form_id) for attachment in attachments[1:]: with self.assertRaises(BlobNotFound): attachment.open() with attachments[0].open() as content: self.assertIsNotNone(content.read()) other_form = FormAccessorSQL.get_form(other_form.form_id) self.assertIsNotNone(other_form.get_xml())
class DeleteAttachmentsFSDBTests(TestCase): def setUp(self): super(DeleteAttachmentsFSDBTests, self).setUp() self.db = TemporaryFilesystemBlobDB() def tearDown(self): self.db.close() super(DeleteAttachmentsFSDBTests, self).tearDown() def test_hard_delete_forms_and_attachments(self): forms = [create_form_for_test(DOMAIN) for i in range(3)] form_ids = sorted(form.form_id for form in forms) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(3, len(forms)) other_form = create_form_for_test('other_domain') self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id])) attachments = sorted( get_blob_db().metadb.get_for_parents(form_ids), key=lambda meta: meta.parent_id ) self.assertEqual(3, len(attachments)) deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id]) self.assertEqual(2, deleted) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(1, len(forms)) self.assertEqual(form_ids[0], forms[0].form_id) for attachment in attachments[1:]: with self.assertRaises(BlobNotFound): attachment.open() with attachments[0].open() as content: self.assertIsNotNone(content.read()) other_form = FormAccessorSQL.get_form(other_form.form_id) self.assertIsNotNone(other_form.get_xml())
class HqTestSuiteRunner(CouchDbKitTestSuiteRunner): """ A test suite runner for Hq. On top of the couchdb testrunner, also apply all our monkeypatches to the settings. To use this, change the settings.py file to read: TEST_RUNNER = 'Hq.testrunner.HqTestSuiteRunner' """ dbs = [] def setup_test_environment(self, **kwargs): self._assert_only_test_databases_accessed() # monkey patch TEST_APPS into INSTALLED_APPS # so that tests are run for them # without having to explicitly have them in INSTALLED_APPS # weird list/tuple type issues, so force everything to tuples settings.INSTALLED_APPS = (tuple(settings.INSTALLED_APPS) + tuple(settings.TEST_APPS)) settings.CELERY_ALWAYS_EAGER = True # keep a copy of the original PILLOWTOPS setting around in case other tests want it. settings._PILLOWTOPS = settings.PILLOWTOPS settings.PILLOWTOPS = {} super(HqTestSuiteRunner, self).setup_test_environment(**kwargs) def setup_databases(self, **kwargs): from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() self._assert_is_a_test_db(settings.COUCH_DATABASE_NAME) return super(HqTestSuiteRunner, self).setup_databases(**kwargs) def teardown_databases(self, old_config, **kwargs): self.blob_db.close() for db_uri in settings.EXTRA_COUCHDB_DATABASES.values(): db = Database(db_uri) self._assert_is_a_test_db(db_uri) self._delete_db_if_exists(db) super(HqTestSuiteRunner, self).teardown_databases(old_config, **kwargs) def _assert_only_test_databases_accessed(self): original_init = Database.__init__ self_ = self def asserting_init(self, uri, create=False, server=None, **params): original_init(self, uri, create=create, server=server, **params) try: self_._assert_is_a_test_db(self.dbname) except AssertionError: db = self def request(self, *args, **kwargs): self_._assert_is_a_test_db(db.dbname) self.res.request = request Database.__init__ = asserting_init @classmethod def get_test_db_name(cls, db_uri): cls._assert_is_a_test_db(db_uri) return db_uri @staticmethod def _assert_is_a_test_db(db_uri): dbname = urlparse(db_uri).path assert dbname.lstrip('/').startswith('test_'), db_uri assert not dbname.endswith('_test'), db_uri @staticmethod def _delete_db_if_exists(db): try: db.server.delete_db(db.dbname) except ResourceNotFound: pass def get_all_test_labels(self): return [self._strip(app) for app in settings.INSTALLED_APPS if app not in settings.APPS_TO_EXCLUDE_FROM_TESTS and not app.startswith('django.')] def run_tests(self, test_labels, extra_tests=None, **kwargs): test_labels = test_labels or self.get_all_test_labels() return super(HqTestSuiteRunner, self).run_tests( test_labels, extra_tests, **kwargs ) def _strip(self, entry): app_config = AppConfig.create(entry) return app_config.label
class HqdbContext(DatabaseContext): """Database context with couchdb setup/teardown In addition to the normal django database setup/teardown, also setup/teardown couchdb databases. This is mostly copied from ``couchdbkit.ext.django.testrunner.CouchDbKitTestSuiteRunner`` """ @classmethod def verify_test_db(cls, app, uri): if '/test_' not in uri: raise ValueError("not a test db url: app=%s url=%r" % (app, uri)) return app def should_skip_test_setup(self): # FRAGILE look in sys.argv; can't get nose config from here return "--collect-only" in sys.argv def setup(self): from django.conf import settings if self.should_skip_test_setup(): return from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() # get/verify list of apps with databases to be deleted on teardown databases = getattr(settings, "COUCHDB_DATABASES", []) if isinstance(databases, (list, tuple)): # Convert old style to new style databases = {app_name: uri for app_name, uri in databases} self.apps = [self.verify_test_db(*item) for item in databases.items()] sys.__stdout__.write("\n") # newline for creating database message super(HqdbContext, self).setup() def teardown(self): if self.should_skip_test_setup(): return self.blob_db.close() # delete couch databases deleted_databases = [] skipcount = 0 for app in self.apps: app_label = app.split('.')[-1] db = loading.get_db(app_label) if db.dbname in deleted_databases: skipcount += 1 continue try: db.server.delete_db(db.dbname) deleted_databases.append(db.dbname) log.info("deleted database %s for %s", db.dbname, app_label) except ResourceNotFound: log.info("database %s not found for %s! it was probably already deleted.", db.dbname, app_label) if skipcount: log.info("skipped deleting %s app databases that were already deleted", skipcount) # HACK clean up leaked database connections from corehq.sql_db.connections import connection_manager connection_manager.dispose_all() super(HqdbContext, self).teardown()
class HqdbContext(DatabaseContext): """Database setup/teardown In addition to the normal django database setup/teardown, also setup/teardown couch databases. Database setup/teardown may be skipped, depending on the presence and value of an environment variable (`REUSE_DB`). Typical usage is `REUSE_DB=1` which means skip database setup and migrations if possible and do not teardown databases after running tests. If connection fails for any test database in `settings.DATABASES` all databases will be re-created and migrated. Other supported `REUSE_DB` values: - `REUSE_DB=reset` : drop existing, then create and migrate new test databses, but do not teardown after running tests. This is convenient when the existing databases are outdated and need to be rebuilt. - `REUSE_DB=optimize` : same as reset, but use migration optimizer to reduce the number of database migrations. - `REUSE_DB=teardown` : skip database setup; do normal teardown after running tests. """ def __init__(self, tests, runner): reuse_db = os.environ.get("REUSE_DB") self.skip_setup_for_reuse_db = reuse_db and reuse_db not in ["reset", "optimize"] self.skip_teardown_for_reuse_db = reuse_db and reuse_db != "teardown" self.optimize_migrations = AppLabelsPlugin.enabled and reuse_db in [None, "optimize"] if self.optimize_migrations: self.test_labels = AppLabelsPlugin.get_test_labels(tests) super(HqdbContext, self).__init__(tests, runner) @classmethod def verify_test_db(cls, app, uri): if '/test_' not in uri: raise ValueError("not a test db url: app=%s url=%r" % (app, uri)) return app, uri def should_skip_test_setup(self): # FRAGILE look in sys.argv; can't get nose config from here return "--collect-only" in sys.argv def setup(self): if self.should_skip_test_setup(): return if self.optimize_migrations: self.optimizer = optimize_apps_for_test_labels(self.test_labels) self.optimizer.__enter__() from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() # get/verify list of apps with databases to be deleted on teardown databases = getattr(settings, "COUCHDB_DATABASES", []) if isinstance(databases, (list, tuple)): # Convert old style to new style databases = {app_name: uri for app_name, uri in databases} self.apps = [self.verify_test_db(*item) for item in databases.items()] if self.skip_setup_for_reuse_db: from django.db import connections old_names = [] for connection in connections.all(): db = connection.settings_dict assert db["NAME"].startswith(TEST_DATABASE_PREFIX), db["NAME"] try: connection.ensure_connection() except OperationalError: break # cannot connect; resume normal setup old_names.append((connection, db["NAME"], True)) else: self.old_names = old_names, [] return # skip remaining setup sys.__stdout__.write("\n") # newline for creating database message super(HqdbContext, self).setup() def teardown(self): if self.should_skip_test_setup(): return self.blob_db.close() if self.optimize_migrations: self.optimizer.__exit__(None, None, None) if self.skip_teardown_for_reuse_db: return # delete couch databases deleted_databases = [] for app, uri in self.apps: if uri in deleted_databases: continue app_label = app.split('.')[-1] db = loading.get_db(app_label) try: db.server.delete_db(db.dbname) deleted_databases.append(uri) log.info("deleted database %s for %s", db.dbname, app_label) except ResourceNotFound: log.info("database %s not found for %s! it was probably already deleted.", db.dbname, app_label) # HACK clean up leaked database connections from corehq.sql_db.connections import connection_manager connection_manager.dispose_all() super(HqdbContext, self).teardown()
class HqdbContext(DatabaseContext): """Database setup/teardown In addition to the normal django database setup/teardown, also setup/teardown couch databases. Database setup/teardown may be skipped, depending on the presence and value of an environment variable (`REUSE_DB`). Typical usage is `REUSE_DB=1` which means skip database setup and migrations if possible and do not teardown databases after running tests. If connection fails for any test database in `settings.DATABASES` all databases will be re-created and migrated. Other supported `REUSE_DB` values: - `REUSE_DB=reset` : drop existing, then create and migrate new test databses, but do not teardown after running tests. This is convenient when the existing databases are outdated and need to be rebuilt. - `REUSE_DB=optimize` : same as reset, but use migration optimizer to reduce the number of database migrations. - `REUSE_DB=teardown` : skip database setup; do normal teardown after running tests. """ def __init__(self, tests, runner): reuse_db = os.environ.get("REUSE_DB") self.skip_setup_for_reuse_db = reuse_db and reuse_db not in [ "reset", "optimize" ] self.skip_teardown_for_reuse_db = reuse_db and reuse_db != "teardown" self.optimize_migrations = AppLabelsPlugin.enabled and reuse_db in [ None, "optimize" ] if self.optimize_migrations: self.test_labels = AppLabelsPlugin.get_test_labels(tests) super(HqdbContext, self).__init__(tests, runner) @classmethod def verify_test_db(cls, app, uri): if '/test_' not in uri: raise ValueError("not a test db url: app=%s url=%r" % (app, uri)) return app, uri def should_skip_test_setup(self): # FRAGILE look in sys.argv; can't get nose config from here return "--collect-only" in sys.argv def setup(self): if self.should_skip_test_setup(): return if self.optimize_migrations: self.optimizer = optimize_apps_for_test_labels(self.test_labels) self.optimizer.__enter__() from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() # get/verify list of apps with databases to be deleted on teardown databases = getattr(settings, "COUCHDB_DATABASES", []) if isinstance(databases, (list, tuple)): # Convert old style to new style databases = {app_name: uri for app_name, uri in databases} self.apps = [self.verify_test_db(*item) for item in databases.items()] if self.skip_setup_for_reuse_db: from django.db import connections old_names = [] for connection in connections.all(): db = connection.settings_dict assert db["NAME"].startswith(TEST_DATABASE_PREFIX), db["NAME"] try: connection.ensure_connection() except OperationalError: break # cannot connect; resume normal setup old_names.append((connection, db["NAME"], True)) else: self.old_names = old_names, [] return # skip remaining setup sys.__stdout__.write("\n") # newline for creating database message if "REUSE_DB" in os.environ: sys.__stdout__.write("REUSE_DB={REUSE_DB!r} ".format(**os.environ)) super(HqdbContext, self).setup() def teardown(self): if self.should_skip_test_setup(): return self.blob_db.close() if self.optimize_migrations: self.optimizer.__exit__(None, None, None) if self.skip_teardown_for_reuse_db: return # delete couch databases deleted_databases = [] for app, uri in self.apps: if uri in deleted_databases: continue app_label = app.split('.')[-1] db = loading.get_db(app_label) try: db.server.delete_db(db.dbname) deleted_databases.append(uri) log.info("deleted database %s for %s", db.dbname, app_label) except ResourceNotFound: log.info( "database %s not found for %s! it was probably already deleted.", db.dbname, app_label) # HACK clean up leaked database connections from corehq.sql_db.connections import connection_manager connection_manager.dispose_all() super(HqdbContext, self).teardown()
class HqdbContext(DatabaseContext): """Database setup/teardown In addition to the normal django database setup/teardown, also setup/teardown couch databases. Database setup/teardown may be skipped, depending on the presence and value of an environment variable (`REUSE_DB`). Typical usage is `REUSE_DB=1` which means skip database setup and migrations if possible and do not teardown databases after running tests. If connection fails for any test database in `settings.DATABASES` all databases will be re-created and migrated. When using REUSE_DB=1, you may also want to provide a value for the --reusedb option, either reset, flush, migrate, or teardown. ./manage.py test --help will give you a description of these. """ def __init__(self, tests, runner): reuse_db = (CmdLineParametersPlugin.get('reusedb') or string_to_boolean(os.environ.get("REUSE_DB") or "0")) self.reuse_db = reuse_db self.skip_setup_for_reuse_db = reuse_db and reuse_db != "reset" self.skip_teardown_for_reuse_db = reuse_db and reuse_db != "teardown" super(HqdbContext, self).__init__(tests, runner) def should_skip_test_setup(self): return CmdLineParametersPlugin.get('collect_only') @timelimit(480) def setup(self): if self.should_skip_test_setup(): return from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() self.old_names = self._get_databases() if self.skip_setup_for_reuse_db and self._databases_ok(): if self.reuse_db == "migrate": call_command('migrate_multi', interactive=False) if self.reuse_db == "flush": flush_databases() return # skip remaining setup if self.reuse_db == "reset": self.reset_databases() print("", file=sys.__stdout__) # newline for creating database message if self.reuse_db: print("REUSE_DB={} ".format(self.reuse_db), file=sys.__stdout__, end="") if self.skip_setup_for_reuse_db: # pass this on to the Django runner to avoid creating databases # that already exist self.runner.keepdb = True super(HqdbContext, self).setup() def reset_databases(self): self.delete_couch_databases() # tear down all databases together to avoid dependency issues teardown = [] for connection, db_name, is_first in self.old_names: try: connection.ensure_connection() teardown.append((connection, db_name, is_first)) except OperationalError: pass # ignore databases that don't exist self.runner.teardown_databases(reversed(teardown)) def _databases_ok(self): for connection, db_name, _ in self.old_names: db = connection.settings_dict assert db["NAME"].startswith(TEST_DATABASE_PREFIX), db["NAME"] try: connection.ensure_connection() except OperationalError as e: print(str(e), file=sys.__stderr__) return False return True def _get_databases(self): from django.db import connections old_names = [] test_databases, mirrored_aliases = get_unique_databases_and_mirrors() assert not mirrored_aliases, "DB mirrors not supported" for signature, (db_name, aliases) in test_databases.items(): alias = list(aliases)[0] connection = connections[alias] old_names.append((connection, db_name, True)) return old_names def delete_couch_databases(self): for db in get_all_test_dbs(): try: db.server.delete_db(db.dbname) log.info("deleted database %s", db.dbname) except ResourceNotFound: log.info( "database %s not found! it was probably already deleted.", db.dbname) def teardown(self): if self.should_skip_test_setup(): return self.blob_db.close() if self.skip_teardown_for_reuse_db: return self.delete_couch_databases() # HACK clean up leaked database connections from corehq.sql_db.connections import connection_manager connection_manager.dispose_all() # in case this was set before we want to remove it now self.runner.keepdb = False # tear down in reverse order self.old_names = reversed(self.old_names) super(HqdbContext, self).teardown()
class HqdbContext(DatabaseContext): """Database setup/teardown In addition to the normal django database setup/teardown, also setup/teardown couch databases. Database setup/teardown may be skipped, depending on the presence and value of an environment variable (`REUSE_DB`). Typical usage is `REUSE_DB=1` which means skip database setup and migrations if possible and do not teardown databases after running tests. If connection fails for any test database in `settings.DATABASES` all databases will be re-created and migrated. When using REUSE_DB=1, you may also want to provide a value for the --reusedb option, either reset, flush, migrate, or teardown. ./manage.py test --help will give you a description of these. """ def __init__(self, tests, runner): reuse_db = (CmdLineParametersPlugin.get('reusedb') or string_to_boolean(os.environ.get("REUSE_DB") or "0")) self.reuse_db = reuse_db self.skip_setup_for_reuse_db = reuse_db and reuse_db != "reset" self.skip_teardown_for_reuse_db = reuse_db and reuse_db != "teardown" super(HqdbContext, self).__init__(tests, runner) def should_skip_test_setup(self): return CmdLineParametersPlugin.get('collect_only') def setup(self): if self.should_skip_test_setup(): return from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() if self.skip_setup_for_reuse_db and self._databases_ok(): if self.reuse_db == "migrate": call_command('migrate_multi', interactive=False) if self.reuse_db == "flush": flush_databases() return # skip remaining setup if self.reuse_db == "reset": self.delete_couch_databases() sys.__stdout__.write("\n") # newline for creating database message if self.reuse_db: sys.__stdout__.write("REUSE_DB={} ".format(self.reuse_db)) super(HqdbContext, self).setup() def _databases_ok(self): from django.db import connections old_names = [] for connection in connections.all(): db = connection.settings_dict assert db["NAME"].startswith(TEST_DATABASE_PREFIX), db["NAME"] try: connection.ensure_connection() except OperationalError: return False old_names.append((connection, db["NAME"], True)) self.old_names = old_names, [] return True def delete_couch_databases(self): for db in get_all_test_dbs(): try: db.server.delete_db(db.dbname) log.info("deleted database %s", db.dbname) except ResourceNotFound: log.info("database %s not found! it was probably already deleted.", db.dbname) def teardown(self): if self.should_skip_test_setup(): return self.blob_db.close() if self.skip_teardown_for_reuse_db: return self.delete_couch_databases() # HACK clean up leaked database connections from corehq.sql_db.connections import connection_manager connection_manager.dispose_all() super(HqdbContext, self).teardown()
class HqdbContext(DatabaseContext): """Database setup/teardown In addition to the normal django database setup/teardown, also setup/teardown couch databases. Database setup/teardown may be skipped, depending on the presence and value of an environment variable (`REUSE_DB`). Typical usage is `REUSE_DB=1` which means skip database setup and migrations if possible and do not teardown databases after running tests. If connection fails for any test database in `settings.DATABASES` all databases will be re-created and migrated. When using REUSE_DB=1, you may also want to provide a value for the --reusedb option, either reset, flush, migrate, or teardown. ./manage.py test --help will give you a description of these. """ def __init__(self, tests, runner): reuse_db = (CmdLineParametersPlugin.get('reusedb') or string_to_boolean(os.environ.get("REUSE_DB") or "0")) self.reuse_db = reuse_db self.skip_setup_for_reuse_db = reuse_db and reuse_db != "reset" self.skip_teardown_for_reuse_db = reuse_db and reuse_db != "teardown" super(HqdbContext, self).__init__(tests, runner) def should_skip_test_setup(self): return CmdLineParametersPlugin.get('collect_only') def setup(self): if self.should_skip_test_setup(): return from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() self.old_names = self._get_databases() if self.skip_setup_for_reuse_db and self._databases_ok(): if self.reuse_db == "migrate": call_command('migrate_multi', interactive=False) if self.reuse_db == "flush": flush_databases() return # skip remaining setup if self.reuse_db == "reset": self.reset_databases() print("", file=sys.__stdout__) # newline for creating database message if self.reuse_db: print("REUSE_DB={} ".format(self.reuse_db), file=sys.__stdout__, end="") if self.skip_setup_for_reuse_db: # pass this on to the Django runner to avoid creating databases # that already exist self.runner.keepdb = True super(HqdbContext, self).setup() def reset_databases(self): self.delete_couch_databases() # tear down all databases together to avoid dependency issues teardown = [] for connection, db_name, is_first in self.old_names: try: connection.ensure_connection() teardown.append((connection, db_name, is_first)) except OperationalError: pass # ignore databases that don't exist self.runner.teardown_databases(reversed(teardown)) def _databases_ok(self): for connection, db_name, _ in self.old_names: db = connection.settings_dict assert db["NAME"].startswith(TEST_DATABASE_PREFIX), db["NAME"] try: connection.ensure_connection() except OperationalError as e: print(str(e), file=sys.__stderr__) return False return True def _get_databases(self): from django.db import connections old_names = [] test_databases, mirrored_aliases = get_unique_databases_and_mirrors() assert not mirrored_aliases, "DB mirrors not supported" for signature, (db_name, aliases) in test_databases.items(): alias = list(aliases)[0] connection = connections[alias] old_names.append((connection, db_name, True)) return old_names def delete_couch_databases(self): for db in get_all_test_dbs(): try: db.server.delete_db(db.dbname) log.info("deleted database %s", db.dbname) except ResourceNotFound: log.info("database %s not found! it was probably already deleted.", db.dbname) def teardown(self): if self.should_skip_test_setup(): return self.blob_db.close() if self.skip_teardown_for_reuse_db: return self.delete_couch_databases() # HACK clean up leaked database connections from corehq.sql_db.connections import connection_manager connection_manager.dispose_all() # in case this was set before we want to remove it now self.runner.keepdb = False # tear down in reverse order self.old_names = reversed(self.old_names) super(HqdbContext, self).teardown()
class TestBigBlobExport(TestCase): domain_name = 'big-blob-test-domain' def setUp(self): # psutil is in dev-requirements only. Don't bother trying to # import for the module if the test is skipped. from psutil import virtual_memory self.memory = virtual_memory().total self.db = TemporaryFilesystemBlobDB() assert get_blob_db() is self.db, (get_blob_db(), self.db) self.blob_metas = [] def tearDown(self): for meta in self.blob_metas: meta.delete() self.db.close() def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) MB = 1024**2 self.mb_block = b'\x00' * MB def mb_blocks(self): while True: yield self.mb_block def test_many_big_blobs(self): number_of_1gb_blobs = ceil(self.memory / 1024**3) + 1 for __ in range(number_of_1gb_blobs): meta = self.db.put(MockBigBlobIO(self.mb_blocks(), 1024), meta=new_meta(domain=self.domain_name, type_code=CODES.multimedia)) self.blob_metas.append(meta) with NamedTemporaryFile() as out: exporter = EXPORTERS['all_blobs'](self.domain_name) exporter.migrate(out.name, force=True) with tarfile.open(out.name, 'r:gz') as tgzfile: self.assertEqual(set(tgzfile.getnames()), {m.key for m in self.blob_metas}) def test_1_very_big_blob(self): number_of_1mb_blocks = ceil(self.memory / 1024**2) + 1 meta = self.db.put(MockBigBlobIO(self.mb_blocks(), number_of_1mb_blocks), meta=new_meta(domain=self.domain_name, type_code=CODES.multimedia)) self.blob_metas.append(meta) with NamedTemporaryFile() as out: exporter = EXPORTERS['all_blobs'](self.domain_name) exporter.migrate(out.name, force=True) with tarfile.open(out.name, 'r:gz') as tgzfile: self.assertEqual(set(tgzfile.getnames()), {m.key for m in self.blob_metas})
class TestExtendingExport(TestCase): domain_name = 'extending-export-test-domain' def setUp(self): self.db = TemporaryFilesystemBlobDB() assert get_blob_db() is self.db, (get_blob_db(), self.db) self.blob_metas = [] def tearDown(self): for meta in self.blob_metas: meta.delete() self.db.close() def test_extends(self): # First export file ... for blob in (b'ham', b'spam', b'eggs'): meta_meta = new_meta( domain=self.domain_name, type_code=CODES.multimedia, ) meta = self.db.put(BytesIO(blob), meta=meta_meta) # Naming ftw self.blob_metas.append(meta) with NamedTemporaryFile() as file_one: exporter = EXPORTERS['all_blobs'](self.domain_name) exporter.migrate(file_one.name, force=True) with tarfile.open(file_one.name, 'r:gz') as tgzfile: keys_in_file_one = set(m.key for m in self.blob_metas[-3:]) self.assertEqual(set(tgzfile.getnames()), keys_in_file_one) # Second export file extends first ... for blob in (b'foo', b'bar', b'baz'): meta_meta = new_meta( domain=self.domain_name, type_code=CODES.multimedia, ) meta = self.db.put(BytesIO(blob), meta=meta_meta) self.blob_metas.append(meta) with NamedTemporaryFile() as file_two: exporter = EXPORTERS['all_blobs'](self.domain_name) exporter.migrate( file_two.name, already_exported=keys_in_file_one, force=True, ) with tarfile.open(file_two.name, 'r:gz') as tgzfile: keys_in_file_two = set(m.key for m in self.blob_metas[-3:]) self.assertEqual(set(tgzfile.getnames()), keys_in_file_two) # Third export file extends first and second ... for blob in (b'wibble', b'wobble', b'wubble'): meta_meta = new_meta( domain=self.domain_name, type_code=CODES.multimedia, ) meta = self.db.put(BytesIO(blob), meta=meta_meta) self.blob_metas.append(meta) with NamedTemporaryFile() as file_three: exporter = EXPORTERS['all_blobs'](self.domain_name) exporter.migrate( file_three.name, already_exported=keys_in_file_one | keys_in_file_two, force=True, ) with tarfile.open(file_three.name, 'r:gz') as tgzfile: keys_in_file_three = set(m.key for m in self.blob_metas[-3:]) self.assertEqual(set(tgzfile.getnames()), keys_in_file_three)