def setup(self): if self.should_skip_test_setup(): return from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() self.old_names = self._get_databases() if self.skip_setup_for_reuse_db and self._databases_ok(): if self.reuse_db == "migrate": call_command('migrate_multi', interactive=False) if self.reuse_db == "flush": flush_databases() return # skip remaining setup if self.reuse_db == "reset": self.reset_databases() print("", file=sys.__stdout__) # newline for creating database message if self.reuse_db: print("REUSE_DB={} ".format(self.reuse_db), file=sys.__stdout__, end="") if self.skip_setup_for_reuse_db: # pass this on to the Django runner to avoid creating databases # that already exist self.runner.keepdb = True super(HqdbContext, self).setup()
def setUp(self): db1 = TemporaryFilesystemBlobDB() assert get_blob_db() is db1, (get_blob_db(), db1) data = b'binary data not valid utf-8 \xe4\x94' self.not_founds = set() self.blob_metas = [] for type_code in [CODES.form_xml, CODES.multimedia, CODES.data_export]: meta = db1.put(BytesIO(data), meta=new_meta(type_code=type_code)) lost = new_meta(type_code=type_code, content_length=42) self.blob_metas.append(meta) self.blob_metas.append(lost) lost.save() self.not_founds.add(( lost.id, lost.domain, lost.type_code, lost.parent_id, lost.key, )) self.test_size = len(self.blob_metas) db2 = TemporaryFilesystemBlobDB() self.db = TemporaryMigratingBlobDB(db2, db1) assert get_blob_db() is self.db, (get_blob_db(), self.db) discard_migration_state(self.slug)
def setup(self): if self.should_skip_test_setup(): return from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() # get/verify list of apps with databases to be deleted on teardown databases = getattr(settings, "COUCHDB_DATABASES", []) if isinstance(databases, (list, tuple)): # Convert old style to new style databases = {app_name: uri for app_name, uri in databases} self.apps = [self.verify_test_db(*item) for item in databases.items()] if self.skip_setup_for_reuse_db: from django.db import connections old_names = [] for connection in connections.all(): db = connection.settings_dict assert db["NAME"].startswith(TEST_DATABASE_PREFIX), db["NAME"] try: connection.ensure_connection() except OperationalError: break # cannot connect; resume normal setup old_names.append((connection, db["NAME"], True)) else: self.old_names = old_names, [] return # skip remaining setup sys.__stdout__.write("\n") # newline for creating database message if "REUSE_DB" in os.environ: sys.__stdout__.write("REUSE_DB={REUSE_DB!r} ".format(**os.environ)) super(HqdbContext, self).setup()
class DeleteAttachmentsFSDBTests(TestCase): def setUp(self): super(DeleteAttachmentsFSDBTests, self).setUp() self.db = TemporaryFilesystemBlobDB() def tearDown(self): self.db.close() super(DeleteAttachmentsFSDBTests, self).tearDown() def test_hard_delete_forms_and_attachments(self): forms = [create_form_for_test(DOMAIN) for i in range(3)] form_ids = [form.form_id for form in forms] forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(3, len(forms)) other_form = create_form_for_test('other_domain') self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id])) attachments = list(FormAccessorSQL.get_attachments_for_forms(form_ids, ordered=True)) self.assertEqual(3, len(attachments)) deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id]) self.assertEqual(2, deleted) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(1, len(forms)) self.assertEqual(form_ids[0], forms[0].form_id) for attachment in attachments[1:]: with self.assertRaises(AttachmentNotFound): attachment.read_content() self.assertIsNotNone(attachments[0].read_content()) other_form = FormAccessorSQL.get_form(other_form.form_id) self.assertIsNotNone(other_form.get_xml())
class GDPRScrubUserFromFormsCouchTests(TestCase): def setUp(self): super(GDPRScrubUserFromFormsCouchTests, self).setUp() self.db = TemporaryFilesystemBlobDB() def tearDown(self): self.db.close() super(GDPRScrubUserFromFormsCouchTests, self).tearDown() def test_modify_attachment_xml_and_metadata_couch(self): form = get_simple_wrapped_form(uuid.uuid4().hex, metadata=TestFormMetadata(domain=DOMAIN), simple_form=GDPR_SIMPLE_FORM) new_form_xml = Command().update_form_data(form, NEW_USERNAME) FormAccessors(DOMAIN).modify_attachment_xml_and_metadata(form, new_form_xml, NEW_USERNAME) # Test that the metadata changed in the database actual_form_xml = form.get_attachment("form.xml").decode('utf-8') self.assertXMLEqual(EXPECTED_FORM_XML, actual_form_xml) # Test that the operations history is updated in this form refetched_form = FormAccessors(DOMAIN).get_form(form.form_id) self.assertEqual(len(refetched_form.history), 1) self.assertEqual(refetched_form.history[0].operation, "gdpr_scrub") self.assertEqual(refetched_form.metadata.username, NEW_USERNAME)
class GDPRScrubUserFromFormsCouchTests(TestCase): def setUp(self): super(GDPRScrubUserFromFormsCouchTests, self).setUp() self.db = TemporaryFilesystemBlobDB() def tearDown(self): self.db.close() super(GDPRScrubUserFromFormsCouchTests, self).tearDown() def test_modify_attachment_xml_and_metadata_couch(self): form = get_simple_wrapped_form( uuid.uuid4().hex, metadata=TestFormMetadata(domain=DOMAIN), simple_form=GDPR_SIMPLE_FORM) new_form_xml = Command().update_form_data(form, NEW_USERNAME) FormAccessors(DOMAIN).modify_attachment_xml_and_metadata( form, new_form_xml, NEW_USERNAME) # Test that the metadata changed in the database actual_form_xml = form.get_attachment("form.xml").decode('utf-8') self.assertXMLEqual(EXPECTED_FORM_XML, actual_form_xml) # Test that the operations history is updated in this form refetched_form = FormAccessors(DOMAIN).get_form(form.form_id) self.assertEqual(len(refetched_form.history), 1) self.assertEqual(refetched_form.history[0].operation, "gdpr_scrub") self.assertEqual(refetched_form.metadata.username, NEW_USERNAME)
def setUp(self): # psutil is in dev-requirements only. Don't bother trying to # import for the module if the test is skipped. from psutil import virtual_memory self.memory = virtual_memory().total self.db = TemporaryFilesystemBlobDB() assert get_blob_db() is self.db, (get_blob_db(), self.db) self.blob_metas = []
def setup(self): if self.should_skip_test_setup(): return if self.optimize_migrations: self.optimizer = optimize_apps_for_test_labels(self.test_labels) self.optimizer.__enter__() from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() # get/verify list of apps with databases to be deleted on teardown databases = getattr(settings, "COUCHDB_DATABASES", []) if isinstance(databases, (list, tuple)): # Convert old style to new style databases = {app_name: uri for app_name, uri in databases} self.apps = [self.verify_test_db(*item) for item in databases.items()] if self.skip_setup_for_reuse_db: from django.db import connections old_names = [] for connection in connections.all(): db = connection.settings_dict assert db["NAME"].startswith(TEST_DATABASE_PREFIX), db["NAME"] try: connection.ensure_connection() except OperationalError: break # cannot connect; resume normal setup old_names.append((connection, db["NAME"], True)) else: self.old_names = old_names, [] return # skip remaining setup sys.__stdout__.write("\n") # newline for creating database message super(HqdbContext, self).setup()
def setUpClass(cls): with trap_extra_setup(AttributeError, msg="S3_BLOB_DB_SETTINGS not configured"): config = settings.S3_BLOB_DB_SETTINGS cls.s3db = TemporaryS3BlobDB(config) cls.fsdb = TemporaryFilesystemBlobDB() cls.db = mod.MigratingBlobDB(cls.s3db, cls.fsdb)
def setup(self): if self.should_skip_test_setup(): return from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() if self.skip_setup_for_reuse_db and self._databases_ok(): if self.reuse_db == "migrate": call_command('migrate_multi', interactive=False) if self.reuse_db == "flush": flush_databases() return # skip remaining setup if self.reuse_db == "reset": self.delete_couch_databases() print("", file=sys.__stdout__) # newline for creating database message if self.reuse_db: print("REUSE_DB={} ".format(self.reuse_db), file=sys.__stdout__, end="") super(HqdbContext, self).setup()
class DeleteAttachmentsFSDBTests(TestCase): def setUp(self): super(DeleteAttachmentsFSDBTests, self).setUp() self.db = TemporaryFilesystemBlobDB() def tearDown(self): self.db.close() super(DeleteAttachmentsFSDBTests, self).tearDown() def test_hard_delete_forms_and_attachments(self): forms = [create_form_for_test(DOMAIN) for i in range(3)] form_ids = sorted(form.form_id for form in forms) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(3, len(forms)) other_form = create_form_for_test('other_domain') self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id])) attachments = sorted( get_blob_db().metadb.get_for_parents(form_ids), key=lambda meta: meta.parent_id ) self.assertEqual(3, len(attachments)) deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id]) self.assertEqual(2, deleted) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(1, len(forms)) self.assertEqual(form_ids[0], forms[0].form_id) for attachment in attachments[1:]: with self.assertRaises(BlobNotFound): attachment.open() with attachments[0].open() as content: self.assertIsNotNone(content.read()) other_form = FormAccessorSQL.get_form(other_form.form_id) self.assertIsNotNone(other_form.get_xml())
class DeleteAttachmentsFSDBTests(TestCase): def setUp(self): super(DeleteAttachmentsFSDBTests, self).setUp() self.db = TemporaryFilesystemBlobDB() def tearDown(self): self.db.close() super(DeleteAttachmentsFSDBTests, self).tearDown() def test_hard_delete_forms_and_attachments(self): forms = [create_form_for_test(DOMAIN) for i in range(3)] form_ids = sorted(form.form_id for form in forms) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(3, len(forms)) other_form = create_form_for_test('other_domain') self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms( 'other_domain', [other_form.form_id])) attachments = sorted(get_blob_db().metadb.get_for_parents(form_ids), key=lambda meta: meta.parent_id) self.assertEqual(3, len(attachments)) deleted = FormAccessorSQL.hard_delete_forms( DOMAIN, form_ids[1:] + [other_form.form_id]) self.assertEqual(2, deleted) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(1, len(forms)) self.assertEqual(form_ids[0], forms[0].form_id) for attachment in attachments[1:]: with self.assertRaises(BlobNotFound): attachment.open() with attachments[0].open() as content: self.assertIsNotNone(content.read()) other_form = FormAccessorSQL.get_form(other_form.form_id) self.assertIsNotNone(other_form.get_xml())
def setup(self): if self.should_skip_test_setup(): return from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() # get/verify list of apps with databases to be deleted on teardown databases = getattr(settings, "COUCHDB_DATABASES", []) if isinstance(databases, (list, tuple)): # Convert old style to new style databases = {app_name: uri for app_name, uri in databases} self.apps = [self.verify_test_db(*item) for item in databases.items()] if self.skip_setup_for_reuse_db and self._databases_ok(): if self.run_migrations_for_reuse_db: call_command('migrate_multi', interactive=False) return # skip remaining setup sys.__stdout__.write("\n") # newline for creating database message if "REUSE_DB" in os.environ: sys.__stdout__.write("REUSE_DB={REUSE_DB!r} ".format(**os.environ)) super(HqdbContext, self).setup()
def import_and_verify(self, filename): from ..management.commands.run_blob_import import Command as ImportCommand expected_metas = { m for m in self.blob_metas if m.key not in self.not_found and m.domain == self.domain_name } self.assertTrue(any(m.is_compressed for m in expected_metas)) with TemporaryFilesystemBlobDB() as dest_db: assert get_blob_db() is dest_db, (get_blob_db(), dest_db) ImportCommand.handle(None, filename) for meta in expected_metas: with dest_db.get(meta=meta) as fh: self.assertEqual(fh.read(), self.data, meta.type_code)
def setup(self): from django.conf import settings if self.should_skip_test_setup(): return from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() # get/verify list of apps with databases to be deleted on teardown databases = getattr(settings, "COUCHDB_DATABASES", []) if isinstance(databases, (list, tuple)): # Convert old style to new style databases = {app_name: uri for app_name, uri in databases} self.apps = [self.verify_test_db(*item) for item in databases.items()] sys.__stdout__.write("\n") # newline for creating database message super(HqdbContext, self).setup()
def setUp(self): with trap_extra_setup(AttributeError, msg="S3_BLOB_DB_SETTINGS not configured"): config = settings.S3_BLOB_DB_SETTINGS fsdb = TemporaryFilesystemBlobDB() assert get_blob_db() is fsdb, (get_blob_db(), fsdb) self.migrate_docs = docs = [] for i in range(self.test_size): doc = SavedBasicExport(configuration=_mk_config("config-%s" % i)) doc.save() doc.set_payload(("content %s" % i).encode('utf-8')) docs.append(doc) s3db = TemporaryS3BlobDB(config) self.db = TemporaryMigratingBlobDB(s3db, fsdb) assert get_blob_db() is self.db, (get_blob_db(), self.db) BaseMigrationTest.discard_migration_state(self.slug)
def setUpClass(cls): super().setUpClass() cls.db = TemporaryFilesystemBlobDB() assert get_blob_db() is cls.db, (get_blob_db(), cls.db) data = b'binary data not valid utf-8 \xe4\x94' cls.blob_metas = [] cls.not_found = set() cls.domain_name = str(uuid.uuid4) for type_code in [CODES.form_xml, CODES.multimedia, CODES.data_export]: for domain in (cls.domain_name, str(uuid.uuid4())): meta = cls.db.put(BytesIO(data), meta=new_meta(domain=domain, type_code=type_code)) lost = new_meta(domain=domain, type_code=type_code, content_length=42) cls.blob_metas.append(meta) cls.blob_metas.append(lost) lost.save() cls.not_found.add(lost.key)
def setUp(self): lost_db = TemporaryFilesystemBlobDB() # must be created before other dbs db1 = TemporaryFilesystemBlobDB() assert get_blob_db() is db1, (get_blob_db(), db1) missing = "found.not" name = "blob.bin" data = b'binary data not valid utf-8 \xe4\x94' self.not_founds = set() self.couch_docs = [] with lost_db: for doc_type, model_class in self.couch_doc_types.items(): item = model_class() item.doc_type = doc_type item.save() item.put_attachment(data, name) with install_blob_db(lost_db): item.put_attachment(data, missing) self.not_founds.add(( doc_type, item._id, item.external_blobs[missing].id, item._blobdb_bucket(), )) item.save() self.couch_docs.append(item) def create_obj(rex): ident = random_url_id(8) args = {rex.blob_helper.id_attr: ident} fields = {getattr(f, "attname", "") for f in rex.model_class._meta.get_fields()} if "content_length" in fields: args["content_length"] = len(data) elif "length" in fields: args["length"] = len(data) item = rex.model_class(**args) save_attr = rex.model_class.__name__ + "_save" if hasattr(self, save_attr): getattr(self, save_attr)(item, rex) else: item.save() return item, ident self.sql_docs = [] for rex in (x() for x in self.sql_reindex_accessors): item, ident = create_obj(rex) helper = rex.blob_helper({"_obj_not_json": item}) db1.put(StringIO(data), ident, helper._blobdb_bucket()) self.sql_docs.append(item) lost, lost_blob_id = create_obj(rex) self.sql_docs.append(lost) self.not_founds.add(( rex.model_class.__name__, lost.id, lost_blob_id, rex.blob_helper({"_obj_not_json": lost})._blobdb_bucket(), )) self.test_size = len(self.couch_docs) + len(self.sql_docs) db2 = TemporaryFilesystemBlobDB() self.db = TemporaryMigratingBlobDB(db2, db1) assert get_blob_db() is self.db, (get_blob_db(), self.db) BaseMigrationTest.discard_migration_state(self.slug)
class TestBigBlobExport(TestCase): domain_name = 'big-blob-test-domain' def setUp(self): # psutil is in dev-requirements only. Don't bother trying to # import for the module if the test is skipped. from psutil import virtual_memory self.memory = virtual_memory().total self.db = TemporaryFilesystemBlobDB() assert get_blob_db() is self.db, (get_blob_db(), self.db) self.blob_metas = [] def tearDown(self): for meta in self.blob_metas: meta.delete() self.db.close() def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) MB = 1024**2 self.mb_block = b'\x00' * MB def mb_blocks(self): while True: yield self.mb_block def test_many_big_blobs(self): number_of_1gb_blobs = ceil(self.memory / 1024**3) + 1 for __ in range(number_of_1gb_blobs): meta = self.db.put(MockBigBlobIO(self.mb_blocks(), 1024), meta=new_meta(domain=self.domain_name, type_code=CODES.multimedia)) self.blob_metas.append(meta) with NamedTemporaryFile() as out: exporter = EXPORTERS['all_blobs'](self.domain_name) exporter.migrate(out.name, force=True) with tarfile.open(out.name, 'r:gz') as tgzfile: self.assertEqual(set(tgzfile.getnames()), {m.key for m in self.blob_metas}) def test_1_very_big_blob(self): number_of_1mb_blocks = ceil(self.memory / 1024**2) + 1 meta = self.db.put(MockBigBlobIO(self.mb_blocks(), number_of_1mb_blocks), meta=new_meta(domain=self.domain_name, type_code=CODES.multimedia)) self.blob_metas.append(meta) with NamedTemporaryFile() as out: exporter = EXPORTERS['all_blobs'](self.domain_name) exporter.migrate(out.name, force=True) with tarfile.open(out.name, 'r:gz') as tgzfile: self.assertEqual(set(tgzfile.getnames()), {m.key for m in self.blob_metas})
class HqdbContext(DatabaseContext): """Database setup/teardown In addition to the normal django database setup/teardown, also setup/teardown couch databases. Database setup/teardown may be skipped, depending on the presence and value of an environment variable (`REUSE_DB`). Typical usage is `REUSE_DB=1` which means skip database setup and migrations if possible and do not teardown databases after running tests. If connection fails for any test database in `settings.DATABASES` all databases will be re-created and migrated. Other supported `REUSE_DB` values: - `REUSE_DB=reset` : drop existing, then create and migrate new test databses, but do not teardown after running tests. This is convenient when the existing databases are outdated and need to be rebuilt. - `REUSE_DB=optimize` : same as reset, but use migration optimizer to reduce the number of database migrations. - `REUSE_DB=teardown` : skip database setup; do normal teardown after running tests. """ def __init__(self, tests, runner): reuse_db = os.environ.get("REUSE_DB") self.skip_setup_for_reuse_db = reuse_db and reuse_db not in ["reset", "optimize"] self.skip_teardown_for_reuse_db = reuse_db and reuse_db != "teardown" self.optimize_migrations = AppLabelsPlugin.enabled and reuse_db in [None, "optimize"] if self.optimize_migrations: self.test_labels = AppLabelsPlugin.get_test_labels(tests) super(HqdbContext, self).__init__(tests, runner) @classmethod def verify_test_db(cls, app, uri): if '/test_' not in uri: raise ValueError("not a test db url: app=%s url=%r" % (app, uri)) return app, uri def should_skip_test_setup(self): # FRAGILE look in sys.argv; can't get nose config from here return "--collect-only" in sys.argv def setup(self): if self.should_skip_test_setup(): return if self.optimize_migrations: self.optimizer = optimize_apps_for_test_labels(self.test_labels) self.optimizer.__enter__() from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() # get/verify list of apps with databases to be deleted on teardown databases = getattr(settings, "COUCHDB_DATABASES", []) if isinstance(databases, (list, tuple)): # Convert old style to new style databases = {app_name: uri for app_name, uri in databases} self.apps = [self.verify_test_db(*item) for item in databases.items()] if self.skip_setup_for_reuse_db: from django.db import connections old_names = [] for connection in connections.all(): db = connection.settings_dict assert db["NAME"].startswith(TEST_DATABASE_PREFIX), db["NAME"] try: connection.ensure_connection() except OperationalError: break # cannot connect; resume normal setup old_names.append((connection, db["NAME"], True)) else: self.old_names = old_names, [] return # skip remaining setup sys.__stdout__.write("\n") # newline for creating database message super(HqdbContext, self).setup() def teardown(self): if self.should_skip_test_setup(): return self.blob_db.close() if self.optimize_migrations: self.optimizer.__exit__(None, None, None) if self.skip_teardown_for_reuse_db: return # delete couch databases deleted_databases = [] for app, uri in self.apps: if uri in deleted_databases: continue app_label = app.split('.')[-1] db = loading.get_db(app_label) try: db.server.delete_db(db.dbname) deleted_databases.append(uri) log.info("deleted database %s for %s", db.dbname, app_label) except ResourceNotFound: log.info("database %s not found for %s! it was probably already deleted.", db.dbname, app_label) # HACK clean up leaked database connections from corehq.sql_db.connections import connection_manager connection_manager.dispose_all() super(HqdbContext, self).teardown()
class HqdbContext(DatabaseContext): """Database setup/teardown In addition to the normal django database setup/teardown, also setup/teardown couch databases. Database setup/teardown may be skipped, depending on the presence and value of an environment variable (`REUSE_DB`). Typical usage is `REUSE_DB=1` which means skip database setup and migrations if possible and do not teardown databases after running tests. If connection fails for any test database in `settings.DATABASES` all databases will be re-created and migrated. Other supported `REUSE_DB` values: - `REUSE_DB=reset` : drop existing, then create and migrate new test databses, but do not teardown after running tests. This is convenient when the existing databases are outdated and need to be rebuilt. - `REUSE_DB=optimize` : same as reset, but use migration optimizer to reduce the number of database migrations. - `REUSE_DB=teardown` : skip database setup; do normal teardown after running tests. """ def __init__(self, tests, runner): reuse_db = os.environ.get("REUSE_DB") self.skip_setup_for_reuse_db = reuse_db and reuse_db not in [ "reset", "optimize" ] self.skip_teardown_for_reuse_db = reuse_db and reuse_db != "teardown" self.optimize_migrations = AppLabelsPlugin.enabled and reuse_db in [ None, "optimize" ] if self.optimize_migrations: self.test_labels = AppLabelsPlugin.get_test_labels(tests) super(HqdbContext, self).__init__(tests, runner) @classmethod def verify_test_db(cls, app, uri): if '/test_' not in uri: raise ValueError("not a test db url: app=%s url=%r" % (app, uri)) return app, uri def should_skip_test_setup(self): # FRAGILE look in sys.argv; can't get nose config from here return "--collect-only" in sys.argv def setup(self): if self.should_skip_test_setup(): return if self.optimize_migrations: self.optimizer = optimize_apps_for_test_labels(self.test_labels) self.optimizer.__enter__() from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() # get/verify list of apps with databases to be deleted on teardown databases = getattr(settings, "COUCHDB_DATABASES", []) if isinstance(databases, (list, tuple)): # Convert old style to new style databases = {app_name: uri for app_name, uri in databases} self.apps = [self.verify_test_db(*item) for item in databases.items()] if self.skip_setup_for_reuse_db: from django.db import connections old_names = [] for connection in connections.all(): db = connection.settings_dict assert db["NAME"].startswith(TEST_DATABASE_PREFIX), db["NAME"] try: connection.ensure_connection() except OperationalError: break # cannot connect; resume normal setup old_names.append((connection, db["NAME"], True)) else: self.old_names = old_names, [] return # skip remaining setup sys.__stdout__.write("\n") # newline for creating database message if "REUSE_DB" in os.environ: sys.__stdout__.write("REUSE_DB={REUSE_DB!r} ".format(**os.environ)) super(HqdbContext, self).setup() def teardown(self): if self.should_skip_test_setup(): return self.blob_db.close() if self.optimize_migrations: self.optimizer.__exit__(None, None, None) if self.skip_teardown_for_reuse_db: return # delete couch databases deleted_databases = [] for app, uri in self.apps: if uri in deleted_databases: continue app_label = app.split('.')[-1] db = loading.get_db(app_label) try: db.server.delete_db(db.dbname) deleted_databases.append(uri) log.info("deleted database %s for %s", db.dbname, app_label) except ResourceNotFound: log.info( "database %s not found for %s! it was probably already deleted.", db.dbname, app_label) # HACK clean up leaked database connections from corehq.sql_db.connections import connection_manager connection_manager.dispose_all() super(HqdbContext, self).teardown()
def setUpClass(cls): super(BlobExpireTest, cls).setUpClass() cls.db = TemporaryFilesystemBlobDB()
def setUpClass(cls): # intentional call to super super setUpClass super(TestBlobMixinWithMigratingDbBeforeCopyToNew, cls).setUpClass() cls.db = PutInOldCopyToNewBlobDB(cls.db, TemporaryFilesystemBlobDB())
def setUpClass(cls): super().setUpClass() cls.db = TemporaryFilesystemBlobDB()
class HqTestSuiteRunner(CouchDbKitTestSuiteRunner): """ A test suite runner for Hq. On top of the couchdb testrunner, also apply all our monkeypatches to the settings. To use this, change the settings.py file to read: TEST_RUNNER = 'Hq.testrunner.HqTestSuiteRunner' """ dbs = [] def setup_test_environment(self, **kwargs): self._assert_only_test_databases_accessed() # monkey patch TEST_APPS into INSTALLED_APPS # so that tests are run for them # without having to explicitly have them in INSTALLED_APPS # weird list/tuple type issues, so force everything to tuples settings.INSTALLED_APPS = (tuple(settings.INSTALLED_APPS) + tuple(settings.TEST_APPS)) settings.CELERY_ALWAYS_EAGER = True # keep a copy of the original PILLOWTOPS setting around in case other tests want it. settings._PILLOWTOPS = settings.PILLOWTOPS settings.PILLOWTOPS = {} super(HqTestSuiteRunner, self).setup_test_environment(**kwargs) def setup_databases(self, **kwargs): from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() self._assert_is_a_test_db(settings.COUCH_DATABASE_NAME) return super(HqTestSuiteRunner, self).setup_databases(**kwargs) def teardown_databases(self, old_config, **kwargs): self.blob_db.close() for db_uri in settings.EXTRA_COUCHDB_DATABASES.values(): db = Database(db_uri) self._assert_is_a_test_db(db_uri) self._delete_db_if_exists(db) super(HqTestSuiteRunner, self).teardown_databases(old_config, **kwargs) def _assert_only_test_databases_accessed(self): original_init = Database.__init__ self_ = self def asserting_init(self, uri, create=False, server=None, **params): original_init(self, uri, create=create, server=server, **params) try: self_._assert_is_a_test_db(self.dbname) except AssertionError: db = self def request(self, *args, **kwargs): self_._assert_is_a_test_db(db.dbname) self.res.request = request Database.__init__ = asserting_init @classmethod def get_test_db_name(cls, db_uri): cls._assert_is_a_test_db(db_uri) return db_uri @staticmethod def _assert_is_a_test_db(db_uri): dbname = urlparse(db_uri).path assert dbname.lstrip('/').startswith('test_'), db_uri assert not dbname.endswith('_test'), db_uri @staticmethod def _delete_db_if_exists(db): try: db.server.delete_db(db.dbname) except ResourceNotFound: pass def get_all_test_labels(self): return [self._strip(app) for app in settings.INSTALLED_APPS if app not in settings.APPS_TO_EXCLUDE_FROM_TESTS and not app.startswith('django.')] def run_tests(self, test_labels, extra_tests=None, **kwargs): test_labels = test_labels or self.get_all_test_labels() return super(HqTestSuiteRunner, self).run_tests( test_labels, extra_tests, **kwargs ) def _strip(self, entry): app_config = AppConfig.create(entry) return app_config.label
class HqdbContext(DatabaseContext): """Database setup/teardown In addition to the normal django database setup/teardown, also setup/teardown couch databases. Database setup/teardown may be skipped, depending on the presence and value of an environment variable (`REUSE_DB`). Typical usage is `REUSE_DB=1` which means skip database setup and migrations if possible and do not teardown databases after running tests. If connection fails for any test database in `settings.DATABASES` all databases will be re-created and migrated. When using REUSE_DB=1, you may also want to provide a value for the --reusedb option, either reset, flush, migrate, or teardown. ./manage.py test --help will give you a description of these. """ def __init__(self, tests, runner): reuse_db = (CmdLineParametersPlugin.get('reusedb') or string_to_boolean(os.environ.get("REUSE_DB") or "0")) self.reuse_db = reuse_db self.skip_setup_for_reuse_db = reuse_db and reuse_db != "reset" self.skip_teardown_for_reuse_db = reuse_db and reuse_db != "teardown" super(HqdbContext, self).__init__(tests, runner) def should_skip_test_setup(self): return CmdLineParametersPlugin.get('collect_only') def setup(self): if self.should_skip_test_setup(): return from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() if self.skip_setup_for_reuse_db and self._databases_ok(): if self.reuse_db == "migrate": call_command('migrate_multi', interactive=False) if self.reuse_db == "flush": flush_databases() return # skip remaining setup if self.reuse_db == "reset": self.delete_couch_databases() sys.__stdout__.write("\n") # newline for creating database message if self.reuse_db: sys.__stdout__.write("REUSE_DB={} ".format(self.reuse_db)) super(HqdbContext, self).setup() def _databases_ok(self): from django.db import connections old_names = [] for connection in connections.all(): db = connection.settings_dict assert db["NAME"].startswith(TEST_DATABASE_PREFIX), db["NAME"] try: connection.ensure_connection() except OperationalError: return False old_names.append((connection, db["NAME"], True)) self.old_names = old_names, [] return True def delete_couch_databases(self): for db in get_all_test_dbs(): try: db.server.delete_db(db.dbname) log.info("deleted database %s", db.dbname) except ResourceNotFound: log.info("database %s not found! it was probably already deleted.", db.dbname) def teardown(self): if self.should_skip_test_setup(): return self.blob_db.close() if self.skip_teardown_for_reuse_db: return self.delete_couch_databases() # HACK clean up leaked database connections from corehq.sql_db.connections import connection_manager connection_manager.dispose_all() super(HqdbContext, self).teardown()
def setUp(self): super(GDPRScrubUserFromFormsCouchTests, self).setUp() self.db = TemporaryFilesystemBlobDB()
def setUpClass(cls): super(TestMigratingBlobDB, cls).setUpClass() assert isinstance(cls.db, TemporaryS3BlobDB), cls.db cls.s3db = cls.db cls.fsdb = TemporaryFilesystemBlobDB() cls.db = TemporaryMigratingBlobDB(cls.s3db, cls.fsdb)
def setUpClass(cls): super(TestMetaDB, cls).setUpClass() cls.db = TemporaryFilesystemBlobDB()
def setUpClass(cls): super(BaseTestCase, cls).setUpClass() cls.db = TemporaryFilesystemBlobDB()
class HqdbContext(DatabaseContext): """Database setup/teardown In addition to the normal django database setup/teardown, also setup/teardown couch databases. Database setup/teardown may be skipped, depending on the presence and value of an environment variable (`REUSE_DB`). Typical usage is `REUSE_DB=1` which means skip database setup and migrations if possible and do not teardown databases after running tests. If connection fails for any test database in `settings.DATABASES` all databases will be re-created and migrated. When using REUSE_DB=1, you may also want to provide a value for the --reusedb option, either reset, flush, migrate, or teardown. ./manage.py test --help will give you a description of these. """ def __init__(self, tests, runner): reuse_db = (CmdLineParametersPlugin.get('reusedb') or string_to_boolean(os.environ.get("REUSE_DB") or "0")) self.reuse_db = reuse_db self.skip_setup_for_reuse_db = reuse_db and reuse_db != "reset" self.skip_teardown_for_reuse_db = reuse_db and reuse_db != "teardown" super(HqdbContext, self).__init__(tests, runner) def should_skip_test_setup(self): return CmdLineParametersPlugin.get('collect_only') def setup(self): if self.should_skip_test_setup(): return from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() self.old_names = self._get_databases() if self.skip_setup_for_reuse_db and self._databases_ok(): if self.reuse_db == "migrate": call_command('migrate_multi', interactive=False) if self.reuse_db == "flush": flush_databases() return # skip remaining setup if self.reuse_db == "reset": self.reset_databases() print("", file=sys.__stdout__) # newline for creating database message if self.reuse_db: print("REUSE_DB={} ".format(self.reuse_db), file=sys.__stdout__, end="") if self.skip_setup_for_reuse_db: # pass this on to the Django runner to avoid creating databases # that already exist self.runner.keepdb = True super(HqdbContext, self).setup() def reset_databases(self): self.delete_couch_databases() # tear down all databases together to avoid dependency issues teardown = [] for connection, db_name, is_first in self.old_names: try: connection.ensure_connection() teardown.append((connection, db_name, is_first)) except OperationalError: pass # ignore databases that don't exist self.runner.teardown_databases(reversed(teardown)) def _databases_ok(self): for connection, db_name, _ in self.old_names: db = connection.settings_dict assert db["NAME"].startswith(TEST_DATABASE_PREFIX), db["NAME"] try: connection.ensure_connection() except OperationalError as e: print(str(e), file=sys.__stderr__) return False return True def _get_databases(self): from django.db import connections old_names = [] test_databases, mirrored_aliases = get_unique_databases_and_mirrors() assert not mirrored_aliases, "DB mirrors not supported" for signature, (db_name, aliases) in test_databases.items(): alias = list(aliases)[0] connection = connections[alias] old_names.append((connection, db_name, True)) return old_names def delete_couch_databases(self): for db in get_all_test_dbs(): try: db.server.delete_db(db.dbname) log.info("deleted database %s", db.dbname) except ResourceNotFound: log.info("database %s not found! it was probably already deleted.", db.dbname) def teardown(self): if self.should_skip_test_setup(): return self.blob_db.close() if self.skip_teardown_for_reuse_db: return self.delete_couch_databases() # HACK clean up leaked database connections from corehq.sql_db.connections import connection_manager connection_manager.dispose_all() # in case this was set before we want to remove it now self.runner.keepdb = False # tear down in reverse order self.old_names = reversed(self.old_names) super(HqdbContext, self).teardown()
class HqdbContext(DatabaseContext): """Database context with couchdb setup/teardown In addition to the normal django database setup/teardown, also setup/teardown couchdb databases. This is mostly copied from ``couchdbkit.ext.django.testrunner.CouchDbKitTestSuiteRunner`` """ @classmethod def verify_test_db(cls, app, uri): if '/test_' not in uri: raise ValueError("not a test db url: app=%s url=%r" % (app, uri)) return app def should_skip_test_setup(self): # FRAGILE look in sys.argv; can't get nose config from here return "--collect-only" in sys.argv def setup(self): from django.conf import settings if self.should_skip_test_setup(): return from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() # get/verify list of apps with databases to be deleted on teardown databases = getattr(settings, "COUCHDB_DATABASES", []) if isinstance(databases, (list, tuple)): # Convert old style to new style databases = {app_name: uri for app_name, uri in databases} self.apps = [self.verify_test_db(*item) for item in databases.items()] sys.__stdout__.write("\n") # newline for creating database message super(HqdbContext, self).setup() def teardown(self): if self.should_skip_test_setup(): return self.blob_db.close() # delete couch databases deleted_databases = [] skipcount = 0 for app in self.apps: app_label = app.split('.')[-1] db = loading.get_db(app_label) if db.dbname in deleted_databases: skipcount += 1 continue try: db.server.delete_db(db.dbname) deleted_databases.append(db.dbname) log.info("deleted database %s for %s", db.dbname, app_label) except ResourceNotFound: log.info("database %s not found for %s! it was probably already deleted.", db.dbname, app_label) if skipcount: log.info("skipped deleting %s app databases that were already deleted", skipcount) # HACK clean up leaked database connections from corehq.sql_db.connections import connection_manager connection_manager.dispose_all() super(HqdbContext, self).teardown()
def setUpClass(cls): super(TestBlobMixinWithMigratingDbBeforeCopyToNew, cls).setUpClass() cls.db = PutInOldBlobDB(cls.db, TemporaryFilesystemBlobDB())
def setUpClass(cls): super(TestBlobDownload, cls).setUpClass() cls.db = TemporaryFilesystemBlobDB()
def setUpClass(cls): super(TestAtomicBlobs, cls).setUpClass() cls.db = TemporaryFilesystemBlobDB()
def setUp(self): super(DeleteAttachmentsFSDBTests, self).setUp() self.db = TemporaryFilesystemBlobDB()
class HqdbContext(DatabaseContext): """Database setup/teardown In addition to the normal django database setup/teardown, also setup/teardown couch databases. Database setup/teardown may be skipped, depending on the presence and value of an environment variable (`REUSE_DB`). Typical usage is `REUSE_DB=1` which means skip database setup and migrations if possible and do not teardown databases after running tests. If connection fails for any test database in `settings.DATABASES` all databases will be re-created and migrated. When using REUSE_DB=1, you may also want to provide a value for the --reusedb option, either reset, flush, migrate, or teardown. ./manage.py test --help will give you a description of these. """ def __init__(self, tests, runner): reuse_db = (CmdLineParametersPlugin.get('reusedb') or string_to_boolean(os.environ.get("REUSE_DB") or "0")) self.reuse_db = reuse_db self.skip_setup_for_reuse_db = reuse_db and reuse_db != "reset" self.skip_teardown_for_reuse_db = reuse_db and reuse_db != "teardown" super(HqdbContext, self).__init__(tests, runner) def should_skip_test_setup(self): return CmdLineParametersPlugin.get('collect_only') @timelimit(480) def setup(self): if self.should_skip_test_setup(): return from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() self.old_names = self._get_databases() if self.skip_setup_for_reuse_db and self._databases_ok(): if self.reuse_db == "migrate": call_command('migrate_multi', interactive=False) if self.reuse_db == "flush": flush_databases() return # skip remaining setup if self.reuse_db == "reset": self.reset_databases() print("", file=sys.__stdout__) # newline for creating database message if self.reuse_db: print("REUSE_DB={} ".format(self.reuse_db), file=sys.__stdout__, end="") if self.skip_setup_for_reuse_db: # pass this on to the Django runner to avoid creating databases # that already exist self.runner.keepdb = True super(HqdbContext, self).setup() def reset_databases(self): self.delete_couch_databases() # tear down all databases together to avoid dependency issues teardown = [] for connection, db_name, is_first in self.old_names: try: connection.ensure_connection() teardown.append((connection, db_name, is_first)) except OperationalError: pass # ignore databases that don't exist self.runner.teardown_databases(reversed(teardown)) def _databases_ok(self): for connection, db_name, _ in self.old_names: db = connection.settings_dict assert db["NAME"].startswith(TEST_DATABASE_PREFIX), db["NAME"] try: connection.ensure_connection() except OperationalError as e: print(str(e), file=sys.__stderr__) return False return True def _get_databases(self): from django.db import connections old_names = [] test_databases, mirrored_aliases = get_unique_databases_and_mirrors() assert not mirrored_aliases, "DB mirrors not supported" for signature, (db_name, aliases) in test_databases.items(): alias = list(aliases)[0] connection = connections[alias] old_names.append((connection, db_name, True)) return old_names def delete_couch_databases(self): for db in get_all_test_dbs(): try: db.server.delete_db(db.dbname) log.info("deleted database %s", db.dbname) except ResourceNotFound: log.info( "database %s not found! it was probably already deleted.", db.dbname) def teardown(self): if self.should_skip_test_setup(): return self.blob_db.close() if self.skip_teardown_for_reuse_db: return self.delete_couch_databases() # HACK clean up leaked database connections from corehq.sql_db.connections import connection_manager connection_manager.dispose_all() # in case this was set before we want to remove it now self.runner.keepdb = False # tear down in reverse order self.old_names = reversed(self.old_names) super(HqdbContext, self).teardown()
def setup_databases(self, **kwargs): from corehq.blobs.tests.util import TemporaryFilesystemBlobDB self.blob_db = TemporaryFilesystemBlobDB() self._assert_is_a_test_db(settings.COUCH_DATABASE_NAME) return super(HqTestSuiteRunner, self).setup_databases(**kwargs)
def setUpClass(cls): cls.db = TemporaryFilesystemBlobDB()