def couch_blob_helper(doc, *args, **kw): obj = BlobHelper(doc, *args, **kw) get_domain = DOMAIN_MAP.get(obj.doc_type) if get_domain is not None: assert not hasattr(obj, "domain"), obj obj.domain = get_domain(doc) assert hasattr(obj, "domain"), obj.doc_type return obj
def couch_blob_helper(doc, *args, **kw): obj = BlobHelper(doc, *args, **kw) get_domain = DOMAIN_MAP.get(obj.doc_type) if get_domain is not None: assert not hasattr(obj, "domain"), obj obj.domain = get_domain(doc) elif not hasattr(obj, "domain"): obj.domain = None # will trigger "unknown-domain" error return obj
def _prepare_doc(self, doc): if self.load_attachments: obj = BlobHelper(doc, self.couchdb) doc["_attachments"] = { name: { "content_type": meta["content_type"], "content": obj.fetch_attachment(name), } for name, meta in doc["_attachments"].items() }
def _do_migration(self, doc): obj = BlobHelper(doc, self.couchdb) bucket = obj._blobdb_bucket() assert obj.external_blobs and obj.external_blobs == obj.blobs, doc for name, meta in obj.blobs.iteritems(): self.total_blobs += 1 try: content = self.db.old_db.get(meta.id, bucket) except NotFound: self.not_found += 1 else: with content: self.db.copy_blob(content, meta.info, bucket) return True
def migrate(self, doc, couchdb): obj = BlobHelper(doc, couchdb) bucket = obj._blobdb_bucket() assert obj.external_blobs and obj.external_blobs == obj.blobs, doc for name, meta in obj.blobs.iteritems(): self.total_blobs += 1 try: content = self.db.old_db.get(meta.id, bucket) except NotFound: self.not_found += 1 else: with content: self.db.copy_blob(content, meta.info, bucket) return True
def process_doc(self, doc): obj = BlobHelper(doc, self.couchdb) bucket = obj._blobdb_bucket() assert obj.external_blobs and obj.external_blobs == obj.blobs, doc from_db = get_blob_db() for name, meta in obj.blobs.iteritems(): self.total_blobs += 1 try: content = from_db.get(meta.id, bucket) except NotFound: self.not_found += 1 else: with content: self.db.copy_blob(content, meta.info, bucket) return True
def _do_migration(self, doc): attachments = doc.pop("_attachments") external_blobs = doc.setdefault("external_blobs", {}) obj = BlobHelper(doc, self.couchdb) try: with obj.atomic_blobs(): for name, data in list(attachments.iteritems()): if name in external_blobs: continue # skip attachment already in blob db obj.put_attachment(name=name, **data) except ResourceConflict: # Do not migrate document if `atomic_blobs()` fails. # This is an unlikely state, but could happen if the # document is (externally) modified between when the # migration fetches and processes the document. return False return True
def migrate(self, doc, couchdb): attachments = doc.pop("_attachments") external_blobs = doc.setdefault("external_blobs", {}) obj = BlobHelper(doc, couchdb) try: with obj.atomic_blobs(): for name, data in list(attachments.iteritems()): if name in external_blobs: continue # skip attachment already in blob db obj.put_attachment(name=name, **data) except ResourceConflict: # Do not migrate document if `atomic_blobs()` fails. # This is an unlikely state, but could happen if the # document is (externally) modified between when the # migration fetches and processes the document. return False return True
def __init__(self, doc, database, exclude_attachments=False): self._attachments = {} self.attachments = {} self.database = database _attachments = doc.get("_attachments", None) or {} _attachments.update(doc.get("external_blobs", None) or {}) if _attachments: if not exclude_attachments: self._attachments = _attachments obj = BlobHelper(doc, database, None) self.attachments = {k: obj.fetch_attachment(k) for k in _attachments} if doc.get("_attachments"): doc["_attachments"] = {} if "external_blobs" in doc: doc["external_blobs"] = {} self.doc = doc del self.doc['_rev']
def save(transform, database): # this is a fancy save method because we do some special casing # with the attachments and with deleted documents def save(): try: database.save_doc(transform.doc, force_update=True) except ResourceNotFound: # this is likely a document that was deleted locally that # you later want to copy back over there is a wacky hack # that you can use to handle this rev = get_deleted_doc_rev(database, transform.doc['_id']) transform.doc['_rev'] = rev database.save_doc(transform.doc) if transform.attachments: obj = BlobHelper(transform.doc, database) with obj.atomic_blobs(save): for name, attach in transform.attachments.items(): content_type = transform._attachments[name]["content_type"] obj.put_attachment(attach, name, content_type=content_type) else: save()
def return_iterator(): yield "<restoredata>" for result in res['hits']['hits']: data_row = result['fields'] # if data_row['script_case_id'] not in active_patients: # continue try: xml_str = (BlobHelper(data_row, db) .fetch_attachment('form.xml') .replace("<?xml version=\'1.0\' ?>", '') .replace("<?xml version='1.0' encoding='UTF-8' ?>", '')) yield xml_str except Exception, ex: logging.error("for downloader: error fetching attachment: %s" % ex)
def broken_suite_files(build): db = Application.get_db() error = None try: suite = BlobHelper(build, db).fetch_attachment('files/suite.xml') except ResourceNotFound: error = 'build has no attachment files/suite.xml' else: try: validate_suite(suite) except SuiteValidationError as error: pass if error: yield '%s\t%s\t%s\t%s\t%s\n' % ( build.get('built_on'), build.get('domain'), build['_id'], build.get('copy_of'), error, )
def _get_submission_xml(xform, db): xml = BlobHelper(xform, db).fetch_attachment('form.xml') if isinstance(xml, unicode): xml = xml.encode('utf-8') return xml
def _get_submission_xml(xform, db): xml = BlobHelper(xform, db, CODES.form_xml).fetch_attachment('form.xml') if isinstance(xml, six.text_type): xml = xml.encode('utf-8') return xml
def migrate(slug, doc_type_map, doc_migrator_class, filename=None, reset=False, max_retry=2): """Migrate blobs :param slug: Migration name. :param doc_type_map: Dict of `doc_type_name: model_class` pairs. :param doc_migrator_class: A `BaseDocMigrator` subclass used to migrate documents. :param filename: File path for intermediate storage of migration data. :param reset: Reset existing migration state (if any), causing all documents to be reconsidered for migration, if this is true. This does not reset the django migration flag. flag, which is set when the migration completes successfully. :param max_retry: Number of times to retry migrating a document before giving up. :returns: A tuple `(<num migrated>, <num skipped>)` """ couchdb = next(iter(doc_type_map.values())).get_db() assert all(m.get_db() is couchdb for m in doc_type_map.values()), \ "documents must live in same couch db: %s" % repr(doc_type_map) dirpath = None if filename is None: dirpath = mkdtemp() filename = os.path.join(dirpath, "export.txt") def encode_content(data): if isinstance(data, unicode): data = data.encode("utf-8") return b64encode(data) total = sum(get_doc_count_by_type(couchdb, doc_type) for doc_type in doc_type_map) print("Migrating {} documents: {}...".format( total, ", ".join(sorted(doc_type_map)) )) migrated = 0 skipped = 0 visited = 0 start = datetime.now() doc_migrator = doc_migrator_class() load_attachments = doc_migrator.load_attachments blobs_key = doc_migrator.blobs_key iter_key = slug + "-blob-migration" docs_by_type = ResumableDocsByTypeIterator(couchdb, doc_type_map, iter_key) if reset: docs_by_type.discard_state() with open(filename, 'wb') as f: for doc in docs_by_type: visited += 1 if doc.get(blobs_key): if load_attachments: obj = BlobHelper(doc, couchdb) doc["_attachments"] = { name: { "content_type": meta["content_type"], "content": obj.fetch_attachment(name), } for name, meta in doc["_attachments"].items() } # make copy with encoded attachments for JSON dump backup_doc = dict(doc) backup_doc["_attachments"] = { name: { "content_type": meta["content_type"], "content": encode_content(meta["content"]), } for name, meta in doc["_attachments"].items() } else: backup_doc = doc f.write('{}\n'.format(json.dumps(backup_doc))) f.flush() ok = doc_migrator.migrate(doc, couchdb) if ok: migrated += 1 else: try: docs_by_type.retry(doc, max_retry) except TooManyRetries: print("Skip: {doc_type} {_id}".format(**doc)) skipped += 1 if (migrated + skipped) % 100 == 0: elapsed = datetime.now() - start remaining = elapsed / visited * total print("Migrated {}/{} of {} documents in {} ({} remaining)" .format(migrated, visited, total, elapsed, remaining)) doc_migrator.after_migration() if dirpath is not None: os.remove(filename) os.rmdir(dirpath) print("Migrated {}/{} of {} documents ({} previously migrated, {} had no attachments)." .format( migrated, visited, total, total - visited, visited - (migrated + skipped) )) if skipped: print(MIGRATIONS_SKIPPED_WARNING.format(skipped)) else: BlobMigrationState.objects.get_or_create(slug=slug)[0].save() return migrated, skipped
def _get_submission_xml(xform, db): return BlobHelper(xform, db, CODES.form_xml).fetch_attachment('form.xml')