def _send_form_to_es(self,
domain=None,
completion_time=None,
received_on=None,
attachment_dict=None,
**metadata_kwargs):
attachment_dict = attachment_dict or {}
metadata = TestFormMetadata(
domain=domain or self.domain,
time_end=completion_time or datetime.utcnow(),
received_on=received_on or datetime.utcnow(),
)
for attr, value in metadata_kwargs.items():
setattr(metadata, attr, value)
form_pair = make_es_ready_form(metadata)
if attachment_dict:
form_pair.wrapped_form.external_blobs = {
name: BlobMetaRef(**meta)
for name, meta in attachment_dict.items()
}
form_pair.json_form['external_blobs'] = attachment_dict
es_form = transform_xform_for_elasticsearch(form_pair.json_form)
send_to_elasticsearch('forms', es_form)
self.es.indices.refresh(XFORM_INDEX_INFO.index)
return form_pair
def blobs(self):
from corehq.blobs.mixin import BlobMetaRef
blobs = {}
if self._attachments:
blobs.update({
name: BlobMetaRef(
content_length=info.get("length", None),
content_type=info.get("content_type", None),
) for name, info in self._attachments.items()
})
if self.external_blobs:
blobs.update({
name: BlobMetaRef.wrap(info)
for name, info in self.external_blobs.items()
})
return blobs
def xml_to_form(domain, xml_meta, case_id, all_metas):
form_id = xml_meta.parent_id
with xml_meta.open() as fh:
xml = fh.read()
form_data = convert_xform_to_json(xml)
form = FormProcessorCouch.new_xform(form_data)
form.domain = domain
form.received_on = get_received_on(case_id, form_id)
for meta in all_metas:
form.external_blobs[meta.name] = BlobMetaRef(
key=meta.key,
blobmeta_id=meta.id,
content_type=meta.content_type,
content_length=meta.content_length,
)
return form
def blobs(self):
from corehq.blobs.mixin import BlobMetaRef
blobs = {}
if self._attachments:
blobs.update({
name: BlobMetaRef(
content_length=info.get("length", None),
content_type=info.get("content_type", None),
) for name, info in six.iteritems(self._attachments)
})
if self.external_blobs:
blobs.update({
name: BlobMetaRef.wrap(info)
for name, info in six.iteritems(self.external_blobs)
})
return blobs
def check_blob(rec, old_db, new_db, migrate=False):
key = rec["blob_key"]
if new_db.exists(key=key):
return "new"
if old_db.exists(key=key):
if migrate:
with old_db.get(key=key,
type_code=CODES.maybe_compressed) as content:
new_db.copy_blob(content, key=key)
action = "Migrated from"
else:
action = "Found in"
print("{} old db: {}".format(action, key))
return "old"
doc_type = BLOB_MIXIN_MODELS.get(rec["type_code"])
if doc_type is not None:
couchdb = doc_type.get_db()
doc_id = rec["parent_id"]
try:
doc = couchdb.get(doc_id)
except ResourceNotFound:
print("Not referenced: {} doc not found".format(json.dumps(rec)))
return "noref"
for name, info in doc.get("external_blobs", {}).items():
data = BlobMetaRef._normalize_json(couchdb.dbname, doc_id, info)
if data["key"] == key:
print("Missing: {} blob info: {}: {}".format(
json.dumps(rec),
repr(name),
info,
))
return "lost"
print("Not referenced: {}".format(json.dumps(rec)))
return "noref"
print("Missing: {}".format(json.dumps(rec)))
return "lost"
def check_blob(rec, old_db, new_db, migrate=False):
key = rec["blob_key"]
if new_db.exists(key=key):
return "new"
if old_db.exists(key=key):
if migrate:
with old_db.get(key=key) as content:
new_db.copy_blob(content, key=key)
action = "Migrated from"
else:
action = "Found in"
print("{} old db: {}".format(action, key))
return "old"
doc_type = BLOB_MIXIN_MODELS.get(rec["type_code"])
if doc_type is not None:
couchdb = doc_type.get_db()
doc_id = rec["parent_id"]
try:
doc = couchdb.get(doc_id)
except ResourceNotFound:
print("Not referenced: {} doc not found".format(json.dumps(rec)))
return "noref"
for name, info in doc.get("external_blobs", {}).items():
data = BlobMetaRef._normalize_json(couchdb.dbname, doc_id, info)
if data["key"] == key:
print("Missing: {} blob info: {}: {}".format(
json.dumps(rec),
repr(name),
info,
))
return "lost"
print("Not referenced: {}".format(json.dumps(rec)))
return "noref"
print("Missing: {}".format(json.dumps(rec)))
return "lost"
def __init__(self, obj, key, domain, reindexer):
self.obj = obj
self.domain = domain
self.blobs = {"": BlobMetaRef(key=key, **reindexer.blob_kwargs(obj))}
self.external_blobs = self.blobs