def test_save_basic_export_to_blobdb(self): index = ['single'] saved_export = SavedBasicExport(configuration=_mk_config(index=index)) saved_export.save() saved_export.set_payload("content") name = saved_export.get_attachment_name() self.assertTrue(saved_export.has_file()) self.assertIn(name, saved_export.external_blobs) self.assertEqual(saved_export.size, 7) with saved_export.get_payload(stream=True) as fh: self.assertEqual(fh.read(), "content")
def test_save_basic_export_to_blobdb(self): index = ['single'] saved_export = SavedBasicExport(configuration=_mk_config(index=index)) saved_export.save() saved_export.set_payload("content") name = saved_export.get_attachment_name() self.assertTrue(saved_export.has_file()) self.assertIn(name, saved_export.external_blobs) self.assertEqual(saved_export.size, 7) with saved_export.get_payload(stream=True) as fh: self.assertEqual(fh.read(), b"content")
def test_migrate_happy_path(self): saved = SavedBasicExport(configuration=_mk_config()) saved.save() payload = b'binary data not valid utf-8 \xe4\x94' name = saved.get_attachment_name() super(BlobMixin, saved).put_attachment(payload, name) saved.save() self.do_migration([saved]) exp = SavedBasicExport.get(saved._id) self.assertEqual(exp.get_payload(), payload)
def test_migrate_with_concurrent_modification(self): # setup data saved = SavedBasicExport(configuration=_mk_config()) saved.save() name = saved.get_attachment_name() new_payload = 'something new' old_payload = 'something old' super(BlobMixin, saved).put_attachment(old_payload, name) super(BlobMixin, saved).put_attachment(old_payload, "other") saved.save() # verify: attachments are in couch self.assertEqual(len(saved._attachments), 2) self.assertEqual(len(saved.external_blobs), 0) modified = [] print_status = mod.print_status # setup concurrent modification def modify_doc_and_print_status(num, total): if not modified: # do concurrent modification doc = SavedBasicExport.get(saved._id) doc.set_payload(new_payload) doc.save() modified.append(True) print_status(num, total) # hook print_status() call to simulate concurrent modification with replattr(mod, "print_status", modify_doc_and_print_status): # do migration migrated, skipped = mod.MIGRATIONS[self.slug].migrate() self.assertGreaterEqual(skipped, 1) # verify: migration state not set when docs are skipped with self.assertRaises(mod.BlobMigrationState.DoesNotExist): mod.BlobMigrationState.objects.get(slug=self.slug) # verify: attachments were not migrated exp = SavedBasicExport.get(saved._id) self.assertEqual(len(exp._attachments), 1, exp._attachments) self.assertEqual(len(exp.external_blobs), 1, exp.external_blobs) self.assertEqual(exp.get_payload(), new_payload) self.assertEqual(exp.fetch_attachment("other"), old_payload)
def test_migrate_with_concurrent_modification(self): saved = SavedBasicExport(configuration=_mk_config()) saved.save() name = saved.get_attachment_name() new_payload = 'something new' old_payload = 'something old' super(BlobMixin, saved).put_attachment(old_payload, name) super(BlobMixin, saved).put_attachment(old_payload, "other") saved.save() self.assertEqual(len(saved._attachments), 2) def modify(doc): doc = SavedBasicExport.get(doc._id) doc.set_payload(new_payload) doc.save() self.do_failed_migration({saved: (1, 1)}, modify) exp = SavedBasicExport.get(saved._id) self.assertEqual(exp.get_payload(), new_payload) self.assertEqual(exp.fetch_attachment("other"), old_payload)
def test_migrate_saved_exports(self): # setup data saved = SavedBasicExport(configuration=_mk_config()) saved.save() payload = 'something small and simple' name = saved.get_attachment_name() super(BlobMixin, saved).put_attachment(payload, name) saved.save() # verify: attachment is in couch and migration not complete self.assertEqual(len(saved._attachments), 1) self.assertEqual(len(saved.external_blobs), 0) with tempdir() as tmp, replattr(SavedBasicExport, "migrating_blobs_from_couch", True): filename = join(tmp, "file.txt") # do migration migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename) self.assertGreaterEqual(migrated, 1) # verify: migration state recorded mod.BlobMigrationState.objects.get(slug=self.slug) # verify: migrated data was written to the file with open(filename) as fh: lines = list(fh) doc = {d["_id"]: d for d in (json.loads(x) for x in lines)}[saved._id] self.assertEqual(doc["_rev"], saved._rev) self.assertEqual(len(lines), migrated, lines) # verify: attachment was moved to blob db exp = SavedBasicExport.get(saved._id) self.assertNotEqual(exp._rev, saved._rev) self.assertEqual(len(exp.blobs), 1, repr(exp.blobs)) self.assertFalse(exp._attachments, exp._attachments) self.assertEqual(len(exp.external_blobs), 1) self.assertEqual(exp.get_payload(), payload)