def test_migrate_backend(self): # verify: attachment is in couch and migration not complete with maybe_not_found(): s3_blobs = sum(1 for b in self.db.new_db._s3_bucket().objects.all()) self.assertEqual(s3_blobs, 0) with tempdir() as tmp: filename = join(tmp, "file.txt") # do migration migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename) self.assertGreaterEqual(migrated, self.test_size) # verify: migration state recorded mod.BlobMigrationState.objects.get(slug=self.slug) # verify: migrated data was written to the file with open(filename) as fh: lines = list(fh) ids = {d._id for d in self.migrate_docs} migrated = {d["_id"] for d in (json.loads(x) for x in lines)} self.assertEqual(len(ids.intersection(migrated)), self.test_size) # verify: attachment was copied to new blob db for doc in self.migrate_docs: exp = SavedBasicExport.get(doc._id) self.assertEqual(exp._rev, doc._rev) # rev should not change self.assertTrue(doc.blobs) bucket = doc._blobdb_bucket() for meta in doc.blobs.values(): content = self.db.new_db.get(meta.id, bucket) self.assertEqual(len(content.read()), meta.content_length)
def download_saved_export(request, export_id): export = SavedBasicExport.get(export_id) attach = export._attachments[export.configuration.filename] response = HttpResponse(mimetype=attach["content_type"]) response.write(export.fetch_attachment(export.configuration.filename)) response['Content-Disposition'] = 'attachment; filename=%s' % export.configuration.filename return response
def modify_doc_and_print_status(num, total): if not modified: # do concurrent modification doc = SavedBasicExport.get(saved._id) doc.set_payload(new_payload) doc.save() modified.append(True) print_status(num, total)
def test_migrate_happy_path(self): saved = SavedBasicExport(configuration=_mk_config()) saved.save() payload = b'binary data not valid utf-8 \xe4\x94' name = saved.get_attachment_name() super(BlobMixin, saved).put_attachment(payload, name) saved.save() self.do_migration([saved]) exp = SavedBasicExport.get(saved._id) self.assertEqual(exp.get_payload(), payload)
def download_saved_export(request, export_id): export = SavedBasicExport.get(export_id) attach = export._attachments[export.get_attachment_name()] response = HttpResponse(mimetype=attach["content_type"]) response.write(export.get_payload()) # ht: http://stackoverflow.com/questions/1207457/convert-unicode-to-string-in-python-containing-extra-symbols normalized_filename = unicodedata.normalize( 'NFKD', unicode(export.configuration.filename), ).encode('ascii','ignore') response['Content-Disposition'] = 'attachment; filename=%s' % normalized_filename return response
def download_saved_export(request, export_id): export = SavedBasicExport.get(export_id) content_type = Format.from_format(export.configuration.format).mimetype payload = export.get_payload(stream=True) response = StreamingHttpResponse(FileWrapper(payload), content_type=content_type) if export.configuration.format != 'html': # ht: http://stackoverflow.com/questions/1207457/convert-unicode-to-string-in-python-containing-extra-symbols normalized_filename = unicodedata.normalize( 'NFKD', unicode(export.configuration.filename), ).encode('ascii', 'ignore') response['Content-Disposition'] = 'attachment; filename="%s"' % normalized_filename return response
def download_saved_export(request, export_id): export = SavedBasicExport.get(export_id) attach = export._attachments[export.get_attachment_name()] response = HttpResponse(mimetype=attach["content_type"]) response.write(export.get_payload()) # ht: http://stackoverflow.com/questions/1207457/convert-unicode-to-string-in-python-containing-extra-symbols normalized_filename = unicodedata.normalize( 'NFKD', unicode(export.configuration.filename), ).encode('ascii', 'ignore') response[ 'Content-Disposition'] = 'attachment; filename=%s' % normalized_filename return response
def test_migrate_with_concurrent_modification(self): # setup data saved = SavedBasicExport(configuration=_mk_config()) saved.save() name = saved.get_attachment_name() new_payload = 'something new' old_payload = 'something old' super(BlobMixin, saved).put_attachment(old_payload, name) super(BlobMixin, saved).put_attachment(old_payload, "other") saved.save() # verify: attachments are in couch self.assertEqual(len(saved._attachments), 2) self.assertEqual(len(saved.external_blobs), 0) modified = [] print_status = mod.print_status # setup concurrent modification def modify_doc_and_print_status(num, total): if not modified: # do concurrent modification doc = SavedBasicExport.get(saved._id) doc.set_payload(new_payload) doc.save() modified.append(True) print_status(num, total) # hook print_status() call to simulate concurrent modification with replattr(mod, "print_status", modify_doc_and_print_status): # do migration migrated, skipped = mod.MIGRATIONS[self.slug].migrate() self.assertGreaterEqual(skipped, 1) # verify: migration state not set when docs are skipped with self.assertRaises(mod.BlobMigrationState.DoesNotExist): mod.BlobMigrationState.objects.get(slug=self.slug) # verify: attachments were not migrated exp = SavedBasicExport.get(saved._id) self.assertEqual(len(exp._attachments), 1, exp._attachments) self.assertEqual(len(exp.external_blobs), 1, exp.external_blobs) self.assertEqual(exp.get_payload(), new_payload) self.assertEqual(exp.fetch_attachment("other"), old_payload)
def test_migrate_with_concurrent_modification(self): saved = SavedBasicExport(configuration=_mk_config()) saved.save() name = saved.get_attachment_name() new_payload = 'something new' old_payload = 'something old' super(BlobMixin, saved).put_attachment(old_payload, name) super(BlobMixin, saved).put_attachment(old_payload, "other") saved.save() self.assertEqual(len(saved._attachments), 2) def modify(doc): doc = SavedBasicExport.get(doc._id) doc.set_payload(new_payload) doc.save() self.do_failed_migration({saved: (1, 1)}, modify) exp = SavedBasicExport.get(saved._id) self.assertEqual(exp.get_payload(), new_payload) self.assertEqual(exp.fetch_attachment("other"), old_payload)
def test_migrate_saved_exports(self): # setup data saved = SavedBasicExport(configuration=_mk_config()) saved.save() payload = 'something small and simple' name = saved.get_attachment_name() super(BlobMixin, saved).put_attachment(payload, name) saved.save() # verify: attachment is in couch and migration not complete self.assertEqual(len(saved._attachments), 1) self.assertEqual(len(saved.external_blobs), 0) with tempdir() as tmp, replattr(SavedBasicExport, "migrating_blobs_from_couch", True): filename = join(tmp, "file.txt") # do migration migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename) self.assertGreaterEqual(migrated, 1) # verify: migration state recorded mod.BlobMigrationState.objects.get(slug=self.slug) # verify: migrated data was written to the file with open(filename) as fh: lines = list(fh) doc = {d["_id"]: d for d in (json.loads(x) for x in lines)}[saved._id] self.assertEqual(doc["_rev"], saved._rev) self.assertEqual(len(lines), migrated, lines) # verify: attachment was moved to blob db exp = SavedBasicExport.get(saved._id) self.assertNotEqual(exp._rev, saved._rev) self.assertEqual(len(exp.blobs), 1, repr(exp.blobs)) self.assertFalse(exp._attachments, exp._attachments) self.assertEqual(len(exp.external_blobs), 1) self.assertEqual(exp.get_payload(), payload)
def hq_download_saved_export(req, domain, export_id): export = SavedBasicExport.get(export_id) # quasi-security hack: the first key of the index is always assumed # to be the domain assert domain == export.configuration.index[0] return couchexport_views.download_saved_export(req, export_id)
def modify(doc): doc = SavedBasicExport.get(doc._id) doc.set_payload(new_payload) doc.save()