def new_meta(**kw):
kw.setdefault("domain", "test")
kw.setdefault("parent_id", "test")
kw.setdefault("type_code", blobs.CODES.tempfile)
if kw["type_code"] == blobs.CODES.form_xml:
kw.setdefault("compressed_length", -1)
return BlobMeta(**kw)
def _attachment_sizes(self):
if not should_use_sql_backend(self.domain):
self.stdout.write('\nAttachment stats only available for SQL domains\n')
return
db_name = get_db_aliases_for_partitioned_query()[0] # just query one shard DB
with BlobMeta.get_cursor_for_partition_db(db_name, readonly=True) as cursor:
cursor.execute("""
SELECT
meta.content_type,
width_bucket(content_length, 0, 2900000, 10) AS bucket,
min(content_length) as bucket_min, max(content_length) AS bucket_max,
count(content_length) AS freq
FROM blobs_blobmeta meta INNER JOIN form_processor_xforminstancesql
ON meta.parent_id = form_processor_xforminstancesql.form_id
WHERE content_length IS NOT NULL AND form_processor_xforminstancesql.domain = %s
GROUP BY content_type, bucket
ORDER BY content_type, bucket
""", [self.domain])
self.stdout.write('Form attachment sizes (bytes)')
self._print_table(
['Content Type', 'Count', 'Bucket range', 'Bucket (1-10)'],
[
(row.content_type, row.freq, '[%s-%s]' % (row.bucket_min, row.bucket_max), row.bucket)
for row in fetchall_as_namedtuple(cursor)
]
)
def migrate(self, doc):
if not doc.get("external_blobs"):
return True
type_code = self.get_type_code(doc)
obj = self.blob_helper(doc, self.couchdb, type_code)
domain = obj.domain
if domain is None:
self.error(
obj, {
"error": "unknown-domain",
"doc_type": obj.doc_type,
"doc_id": obj._id,
})
domain = UNKNOWN_DOMAIN
if getattr(obj, "_attachments", None):
self.error(
obj, {
"error": "ignored-couch-attachments",
"doc_type": obj.doc_type,
"doc_id": obj._id,
"domain": obj.domain,
"attachments": obj._attachments,
})
with BlobMeta.get_cursor_for_partition_value(doc['_id']) as cursor:
for name, meta in obj.external_blobs.items():
if meta.blobmeta_id is not None:
# blobmeta already saved
continue
cursor.execute("""
INSERT INTO blobs_blobmeta (
domain,
type_code,
parent_id,
name,
key,
content_type,
content_length,
created_on
) VALUES (%s, %s, %s, %s, %s, %s, %s, CLOCK_TIMESTAMP())
ON CONFLICT (key) DO NOTHING
""",
params=[
domain,
type_code,
doc["_id"],
name,
meta.key,
meta.content_type,
meta.content_length or 0,
])
self.total_blobs += 1
return True
def test_extract_form_attachment_info(self):
image_1_name = "1234.jpg"
image_2_name = "5678.jpg"
form = {
"name": "foo",
"color": "bar",
"image_1": image_1_name,
"my_group": {
"image_2": image_2_name
}
}
attachments = {
image_1_name: {
"content_type": "image/jpeg",
"content_length": 1024,
},
image_2_name: {
"content_type": "image/jpeg",
"content_length": 2048,
},
"form.xml": {
"content_type": "text/xml",
"content_length": 2048,
}
}
with mock.patch.object(XFormInstanceSQL,
'form_data') as form_data_mock:
form_data_mock.__get__ = mock.MagicMock(return_value=form)
couch_xform = XFormInstance(
received_on=datetime.datetime.now(),
form=form,
)
for name, meta in attachments.items():
couch_xform.deferred_put_attachment("content", name, **meta)
sql_xform = XFormInstanceSQL(received_on=datetime.datetime.now())
sql_xform.attachments_list = [
BlobMeta(name=name, **meta)
for name, meta in attachments.items()
]
for xform in (couch_xform, sql_xform):
print(type(xform).__name__)
form_info = _extract_form_attachment_info(
xform, {"my_group-image_2", "image_1"})
attachments = {a['name']: a for a in form_info['attachments']}
self.assertTrue(image_1_name in attachments)
self.assertTrue(image_2_name in attachments)
self.assertEqual(attachments[image_1_name]['question_id'],
"image_1")
self.assertEqual(attachments[image_2_name]['question_id'],
"my_group-image_2")
def new_meta(**kw):
kw.setdefault("domain", "test")
kw.setdefault("parent_id", "test")
kw.setdefault("type_code", blobs.CODES.form_xml)
return BlobMeta(**kw)