def get_db_for_partitioned_model(model, hints):
from corehq.sql_db.util import get_db_alias_for_partitioned_doc, get_db_aliases_for_partitioned_query
if not hints:
raise Exception(
f'Routing for partitioned models requires a hint. Use one of {ALL_HINTS}'
)
if len(set(hints) & ALL_HINTS) > 1:
raise Exception(
f'Unable to perform routing, multiple hints provided: {hints}')
if HINT_INSTANCE in hints:
partition_value = getattr(hints[HINT_INSTANCE], 'partition_value',
None)
if partition_value is not None:
return get_db_alias_for_partitioned_doc(partition_value)
if hints.get(HINT_PLPROXY):
return plproxy_config.proxy_db
if HINT_USING in hints:
db = hints[HINT_USING]
assert db in get_db_aliases_for_partitioned_query()
return db
if HINT_PARTITION_VALUE in hints:
return get_db_alias_for_partitioned_doc(hints[HINT_PARTITION_VALUE])
raise Exception(
f'Unable to route query for {model}. No matching hints. Use one of {ALL_HINTS}'
)
def test_same_dbalias_util(self):
from corehq.sql_db.util import get_db_alias_for_partitioned_doc, new_id_in_same_dbalias
for i in range(10):
# test multiple times to test a wider probability
f1_id = six.text_type(uuid4())
old_db_alias = get_db_alias_for_partitioned_doc(f1_id)
f2_id = new_id_in_same_dbalias(f1_id)
new_db_alias = get_db_alias_for_partitioned_doc(f2_id)
self.assertEqual(new_db_alias, old_db_alias)
def test_same_dbalias_util(self):
from corehq.sql_db.util import get_db_alias_for_partitioned_doc, new_id_in_same_dbalias
for i in range(10):
# test multiple times to test a wider probability
f1_id = str(uuid4())
old_db_alias = get_db_alias_for_partitioned_doc(f1_id)
f2_id = new_id_in_same_dbalias(f1_id)
new_db_alias = get_db_alias_for_partitioned_doc(f2_id)
self.assertEqual(new_db_alias, old_db_alias)
def reparent(self, old_parent_id, new_parent_id):
"""Reassign blobs' parent
Both `old_parent_id` and `new_parent_id` must map to the same
database partition.
"""
dbname = get_db_alias_for_partitioned_doc(old_parent_id)
new_db = get_db_alias_for_partitioned_doc(new_parent_id)
assert dbname == new_db, (
"Cannot reparent to new partition: %s -> %s" %
(old_parent_id, new_parent_id))
query = BlobMeta.objects.partitioned_query(old_parent_id)
query.filter(parent_id=old_parent_id).update(parent_id=new_parent_id)
def new_id_in_different_dbalias(partition_value):
"""
Returns a new partition value from a different db alias than
the given partition value does
"""
if not settings.USE_PARTITIONED_DATABASE:
raise SkipTest("cannot get different db alias for non-sharded db")
old_db_name = get_db_alias_for_partitioned_doc(partition_value)
new_db_name = old_db_name
while old_db_name == new_db_name:
# todo; guard against infinite loop
new_partition_value = str(uuid.uuid4())
new_db_name = get_db_alias_for_partitioned_doc(new_partition_value)
return new_partition_value
def reparent(self, old_parent_id, new_parent_id):
"""Reassign blobs' parent
Both `old_parent_id` and `new_parent_id` must map to the same
database partition.
"""
dbname = get_db_alias_for_partitioned_doc(old_parent_id)
new_db = get_db_alias_for_partitioned_doc(new_parent_id)
assert dbname == new_db, ("Cannot reparent to new partition: %s -> %s" %
(old_parent_id, new_parent_id))
with connections[dbname].cursor() as cursor:
cursor.execute(
"UPDATE blobs_blobmeta SET parent_id = %s WHERE parent_id = %s",
[new_parent_id, old_parent_id],
)
def test_soft_delete(self):
meta = TestFormMetadata(domain=DOMAIN)
get_simple_wrapped_form('f1', metadata=meta)
f2 = get_simple_wrapped_form('f2', metadata=meta)
f2.archive()
get_simple_wrapped_form('f3', metadata=meta)
accessors = FormAccessors(DOMAIN)
# delete
num = accessors.soft_delete_forms(['f1', 'f2'], deletion_id='123')
self.assertEqual(num, 2)
for form_id in ['f1', 'f2']:
form = accessors.get_form(form_id)
self.assertTrue(form.is_deleted)
self.assertEqual(form.deletion_id, '123')
form = accessors.get_form('f3')
self.assertFalse(form.is_deleted)
for form_id in ['f1', 'f2']:
form = FormAccessors(DOMAIN).get_form(form_id)
if isinstance(form, PartitionedModel):
form.delete(using=get_db_alias_for_partitioned_doc(form.form_id))
else:
form.delete()
def test_blob_expires(self):
now = datetime(2017, 1, 1)
shard = get_db_alias_for_partitioned_doc(self.args["parent_id"])
manager = BlobMeta.objects.using(shard)
pre_expire_count = manager.count()
with capture_log_output(mod.__name__) as logs:
with patch('corehq.blobs.metadata._utcnow', return_value=now):
self.db.put(BytesIO(b'content'), timeout=60, **self.args)
self.assertIsNotNone(self.db.get(key=self.key))
with patch('corehq.blobs.tasks._utcnow',
return_value=now + timedelta(minutes=61)):
bytes_deleted = delete_expired_blobs()
self.assertEqual(bytes_deleted, len('content'))
with self.assertRaises(NotFound):
self.db.get(key=self.key)
self.assertEqual(manager.all().count(), pre_expire_count)
self.assertRegexpMatches(
logs.get_output(),
r"deleted expired blobs: .+'blob-identifier'",
)
def _delete_data_files(domain_name):
db = get_db_alias_for_partitioned_doc(domain_name)
get_blob_db().bulk_delete(metas=list(
BlobMeta.objects.using(db).filter(
parent_id=domain_name,
type_code=CODES.data_file,
)))
def test_resume_migration(self):
with tempdir() as tmp:
filename = join(tmp, "file.txt")
migrator = MIGRATIONS[self.slug]
migrated1, skipped = migrator.migrate(filename)
self.assertGreaterEqual(migrated1, self.test_size)
self.assertFalse(skipped)
# discard state to simulate interrupted migration
for mig in migrator.iter_migrators():
mig.get_document_provider().get_document_iterator(1).discard_state()
# resumed migration: all docs already migrated, so BlobMeta records
# exist, but should not cause errors on attempting to insert them
migrated2, skipped = MIGRATIONS[self.slug].migrate(filename)
self.assertEqual(migrated1, migrated2)
self.assertFalse(skipped)
mod.BlobMigrationState.objects.get(slug=self.slug)
parent_ids = chain(
(doc.id for doc in self.couch_docs),
(doc.parent_id for doc in self.sql_docs),
)
# should have one blob per parent
for parent_id in parent_ids:
db = get_db_alias_for_partitioned_doc(parent_id)
metas = list(BlobMeta.objects.using(db).filter(parent_id=parent_id))
self.assertEqual(len(metas), 1, metas)
def update_form(self, form, publish_changes=True):
from ..change_publishers import publish_form_saved
assert form.is_saved(), "this method doesn't support creating unsaved forms"
assert not form.has_unsaved_attachments(), 'Adding attachments to saved form not supported'
assert not form.has_tracked_models_to_delete(), 'Deleting other models not supported by this method'
assert not form.has_tracked_models_to_update(), 'Updating other models not supported by this method'
assert not form.has_tracked_models_to_create(BlobMeta), \
'Adding new attachments not supported by this method'
new_operations = form.get_tracked_models_to_create(XFormOperation)
db_name = form.db
if form.orig_id:
old_db_name = get_db_alias_for_partitioned_doc(form.orig_id)
assert old_db_name == db_name, "this method doesn't support moving the form to new db"
with transaction.atomic(using=db_name):
if form.form_id_updated():
operations = form.original_operations + new_operations
form.save()
get_blob_db().metadb.reparent(form.orig_id, form.form_id)
for model in operations:
model.form = form
model.save()
else:
with transaction.atomic(db_name):
form.save()
for operation in new_operations:
operation.form = form
operation.save()
if publish_changes:
publish_form_saved(form)
def get(self, **kw):
"""Get metadata for a single blob
All arguments must be passed as keyword arguments.
:param parent_id: `BlobMeta.parent_id`
:param type_code: `BlobMeta.type_code`
:param name: `BlobMeta.name`
:param key: `BlobMeta.key`
:raises: `BlobMeta.DoesNotExist` if the metadata is not found.
:returns: A `BlobMeta` object.
"""
keywords = set(kw)
if 'key' in keywords and keywords != {'key', 'parent_id'}:
kw.pop('key', None)
if 'parent_id' not in keywords:
raise TypeError("Missing argument 'parent_id'")
else:
kw.pop('parent_id')
raise TypeError("Unexpected arguments: {}".format(", ".join(kw)))
elif 'key' not in keywords and keywords != {"parent_id", "type_code", "name"}:
# arg check until on Python 3 -> PEP 3102: required keyword args
kw.pop("parent_id", None)
kw.pop("type_code", None)
kw.pop("name", None)
if not kw:
raise TypeError("Missing argument 'name' and/or 'parent_id'")
raise TypeError("Unexpected arguments: {}".format(", ".join(kw)))
dbname = get_db_alias_for_partitioned_doc(kw["parent_id"])
meta = BlobMeta.objects.using(dbname).filter(**kw).first()
if meta is None:
raise BlobMeta.DoesNotExist(repr(kw))
return meta
def delete_blobmeta(meta):
if meta.id > 0:
meta.delete()
else:
db = get_db_alias_for_partitioned_doc(meta.parent_id)
DeprecatedXFormAttachmentSQL.objects.using(db).filter(
id=-meta.id).delete()
def test_resume_migration(self):
with tempdir() as tmp:
filename = join(tmp, "file.txt")
migrator = MIGRATIONS[self.slug]
migrated1, skipped = migrator.migrate(filename)
self.assertGreaterEqual(migrated1, self.test_size)
self.assertFalse(skipped)
# discard state to simulate interrupted migration
for mig in migrator.iter_migrators():
mig.get_document_provider().get_document_iterator(
1).discard_state()
# resumed migration: all docs already migrated, so BlobMeta records
# exist, but should not cause errors on attempting to insert them
migrated2, skipped = MIGRATIONS[self.slug].migrate(filename)
self.assertEqual(migrated1, migrated2)
self.assertFalse(skipped)
mod.BlobMigrationState.objects.get(slug=self.slug)
parent_ids = chain(
(doc.id for doc in self.couch_docs),
(doc.parent_id for doc in self.sql_docs),
)
# should have one blob per parent
for parent_id in parent_ids:
db = get_db_alias_for_partitioned_doc(parent_id)
metas = list(
BlobMeta.objects.using(db).filter(parent_id=parent_id))
self.assertEqual(len(metas), 1, metas)
def test_resume_migration(self):
class IncompleteDPC(migrate.DocumentProcessorController):
def _processing_complete(self):
self.document_iterator.discard_state()
with tempdir() as tmp:
filename = join(tmp, "file.txt")
with replattr(migrate, "DocumentProcessorController", IncompleteDPC):
# interrupted migration
migrated1, skipped = MIGRATIONS[self.slug].migrate(filename)
self.assertGreaterEqual(migrated1, self.test_size)
self.assertFalse(skipped)
# resumed migration: all docs already migrated, so BlobMeta records
# exist, but should not cause errors on attempting to insert them
migrated2, skipped = MIGRATIONS[self.slug].migrate(filename)
self.assertEqual(migrated1, migrated2)
self.assertFalse(skipped)
mod.BlobMigrationState.objects.get(slug=self.slug)
parent_ids = chain(
(doc.id for doc in self.couch_docs),
(doc.parent_id for doc in self.sql_docs),
)
# should have one blob per parent
for parent_id in parent_ids:
db = get_db_alias_for_partitioned_doc(parent_id)
metas = list(BlobMeta.objects.using(db).filter(parent_id=parent_id))
self.assertEqual(len(metas), 1, metas)
def test_migrate_backend(self):
with tempdir() as tmp:
filename = join(tmp, "file.txt")
# do migration
migrated, skipped = MIGRATIONS[self.slug].migrate(filename)
self.assertGreaterEqual(migrated, self.test_size)
# verify: migration state recorded
mod.BlobMigrationState.objects.get(slug=self.slug)
# verify: metadata was saved in BlobMeta table
for doc in self.couch_docs:
obj = doc.class_.get(doc.id)
self.assertEqual(obj._rev,
doc.data["_rev"]) # rev should not change
self.assertEqual(set(obj.blobs), set(doc.data["external_blobs"]))
db = get_db_alias_for_partitioned_doc(obj._id)
metas = {
meta.name: meta
for meta in BlobMeta.objects.using(db).filter(parent_id=doc.id)
}
for name, meta in obj.blobs.items():
blobmeta = metas[name]
dbname = doc.class_.get_db().dbname
key = "%s/%s/%s" % (dbname, obj._id, doc.doc_type.lower())
self.assertEqual(blobmeta.key, key, doc)
self.assertEqual(blobmeta.domain, doc.domain, doc)
self.assertEqual(blobmeta.content_type, meta.content_type, doc)
self.assertEqual(blobmeta.content_length, meta.content_length,
doc)
for doc in self.sql_docs:
db = get_db_alias_for_partitioned_doc(doc.parent_id)
blobmeta = BlobMeta.objects.using(db).get(
parent_id=doc.parent_id,
type_code=doc.type_code,
name="",
)
self.assertEqual(blobmeta.domain, doc.domain, doc)
self.assertEqual(blobmeta.content_type, doc.content_type, doc)
self.assertEqual(blobmeta.content_length, doc.content_length, doc)
def migrate(self, doc):
if not doc.get("external_blobs"):
return True
type_code = self.get_type_code(doc)
obj = self.blob_helper(doc, self.couchdb, type_code)
db = get_db_alias_for_partitioned_doc(doc["_id"])
domain = obj.domain
if domain is None:
self.error(
obj, {
"error": "unknown-domain",
"doc_type": obj.doc_type,
"doc_id": obj._id,
})
domain = UNKNOWN_DOMAIN
if getattr(obj, "_attachments", None):
self.error(
obj, {
"error": "ignored-couch-attachments",
"doc_type": obj.doc_type,
"doc_id": obj._id,
"domain": obj.domain,
"attachments": obj._attachments,
})
with connections[db].cursor() as cursor:
for name, meta in six.iteritems(obj.external_blobs):
if meta.blobmeta_id is not None:
# blobmeta already saved
continue
cursor.execute("""
INSERT INTO blobs_blobmeta (
domain,
type_code,
parent_id,
name,
key,
content_type,
content_length,
created_on
) VALUES (%s, %s, %s, %s, %s, %s, %s, CLOCK_TIMESTAMP())
ON CONFLICT (key) DO NOTHING
""",
params=[
domain,
type_code,
doc["_id"],
name,
meta.key,
meta.content_type,
meta.content_length or 0,
])
self.total_blobs += 1
return True
def test_save_empty_properties(self):
meta = new_meta()
self.assertEqual(meta.properties, {})
self.db.put(BytesIO(b"content"), meta=meta)
self.assertEqual(get_meta(meta).properties, {})
dbname = get_db_alias_for_partitioned_doc(meta.parent_id)
with connections[dbname].cursor() as cursor:
cursor.execute(
"SELECT id, properties FROM blobs_blobmeta WHERE id = %s",
[meta.id],
)
self.assertEqual(cursor.fetchall(), [(meta.id, None)])
def get_for_parent(self, parent_id, type_code=None):
"""Get a list of `BlobMeta` objects for the given parent
:param parent_id: `BlobMeta.parent_id`
:param type_code: `BlobMeta.type_code` (optional).
:returns: A list of `BlobMeta` objects.
"""
dbname = get_db_alias_for_partitioned_doc(parent_id)
kw = {"parent_id": parent_id}
if type_code is not None:
kw["type_code"] = type_code
return list(BlobMeta.objects.using(dbname).filter(**kw))
def test_save_empty_properties(self):
meta = new_meta()
self.assertEqual(meta.properties, {})
self.db.put(BytesIO(b"content"), meta=meta)
self.assertEqual(get_meta(meta).properties, {})
dbname = get_db_alias_for_partitioned_doc(meta.parent_id)
with connections[dbname].cursor() as cursor:
cursor.execute(
"SELECT id, properties FROM blobs_blobmeta WHERE id = %s",
[meta.id],
)
self.assertEqual(cursor.fetchall(), [(meta.id, None)])
def test_migrate_backend(self):
with tempdir() as tmp:
filename = join(tmp, "file.txt")
# do migration
migrated, skipped = MIGRATIONS[self.slug].migrate(filename)
self.assertGreaterEqual(migrated, self.test_size)
# verify: migration state recorded
mod.BlobMigrationState.objects.get(slug=self.slug)
# verify: metadata was saved in BlobMeta table
for doc in self.couch_docs:
obj = doc.class_.get(doc.id)
self.assertEqual(obj._rev, doc.data["_rev"]) # rev should not change
self.assertEqual(set(obj.blobs), set(doc.data["external_blobs"]))
db = get_db_alias_for_partitioned_doc(obj._id)
metas = {meta.name: meta
for meta in BlobMeta.objects.using(db).filter(parent_id=doc.id)}
for name, meta in obj.blobs.items():
blobmeta = metas[name]
dbname = doc.class_.get_db().dbname
key = "%s/%s/%s" % (dbname, obj._id, doc.doc_type.lower())
self.assertEqual(blobmeta.key, key, doc)
self.assertEqual(blobmeta.domain, doc.domain, doc)
self.assertEqual(blobmeta.content_type, meta.content_type, doc)
self.assertEqual(blobmeta.content_length, meta.content_length, doc)
for doc in self.sql_docs:
db = get_db_alias_for_partitioned_doc(doc.parent_id)
blobmeta = BlobMeta.objects.using(db).get(
parent_id=doc.parent_id,
type_code=doc.type_code,
name="",
)
self.assertEqual(blobmeta.domain, doc.domain, doc)
self.assertEqual(blobmeta.content_type, doc.content_type, doc)
self.assertEqual(blobmeta.content_length, doc.content_length, doc)
def migrate(self, doc):
if not doc.get("external_blobs"):
return True
type_code = self.get_type_code(doc)
obj = self.blob_helper(doc, self.couchdb, type_code)
db = get_db_alias_for_partitioned_doc(doc["_id"])
domain = obj.domain
if domain is None:
self.error(obj, {
"error": "unknown-domain",
"doc_type": obj.doc_type,
"doc_id": obj._id,
})
domain = UNKNOWN_DOMAIN
if getattr(obj, "_attachments", None):
self.error(obj, {
"error": "ignored-couch-attachments",
"doc_type": obj.doc_type,
"doc_id": obj._id,
"domain": obj.domain,
"attachments": obj._attachments,
})
with connections[db].cursor() as cursor:
for name, meta in six.iteritems(obj.external_blobs):
if meta.blobmeta_id is not None:
# blobmeta already saved
continue
cursor.execute("""
INSERT INTO blobs_blobmeta (
domain,
type_code,
parent_id,
name,
key,
content_type,
content_length,
created_on
) VALUES (%s, %s, %s, %s, %s, %s, %s, CLOCK_TIMESTAMP())
ON CONFLICT (key) DO NOTHING
""", params=[
domain,
type_code,
doc["_id"],
name,
meta.key,
meta.content_type,
meta.content_length or 0,
])
self.total_blobs += 1
return True
def test_blob_does_not_expire(self):
now = datetime(2017, 1, 1)
shard = get_db_alias_for_partitioned_doc(self.args["parent_id"])
manager = BlobMeta.objects.using(shard)
pre_expire_count = manager.all().count()
with patch('corehq.blobs.metadata._utcnow', return_value=now):
self.db.put(BytesIO(b'content'), timeout=60, **self.args)
self.assertIsNotNone(self.db.get(key=self.key))
with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=30)):
delete_expired_blobs()
self.assertIsNotNone(self.db.get(key=self.key))
self.assertEqual(manager.all().count(), pre_expire_count + 1)
def test_blob_does_not_expire(self):
now = datetime(2017, 1, 1)
shard = get_db_alias_for_partitioned_doc(self.args["parent_id"])
manager = BlobMeta.objects.using(shard)
pre_expire_count = manager.all().count()
with patch('corehq.blobs.metadata._utcnow', return_value=now):
self.db.put(BytesIO(b'content'), timeout=60, **self.args)
self.assertIsNotNone(self.db.get(key=self.key))
with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=30)):
delete_expired_blobs()
self.assertIsNotNone(self.db.get(key=self.key))
self.assertEqual(manager.all().count(), pre_expire_count + 1)
def test_get_attachment_by_name(self):
form = create_form_for_test(DOMAIN)
form_xml = get_simple_form_xml(form.form_id)
form_db = get_db_alias_for_partitioned_doc(form.form_id)
with self.assertRaises(AttachmentNotFound):
FormAccessorSQL.get_attachment_by_name(form.form_id, 'not_a_form.xml')
with self.assertNumQueries(1, using=form_db):
attachment_meta = FormAccessorSQL.get_attachment_by_name(form.form_id, 'form.xml')
self.assertEqual(form.form_id, attachment_meta.parent_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
with attachment_meta.open() as content:
self.assertEqual(form_xml, content.read().decode('utf-8'))
def test_get_attachment_by_name(self):
form = create_form_for_test(DOMAIN)
form_xml = get_simple_form_xml(form.form_id)
form_db = get_db_alias_for_partitioned_doc(form.form_id)
get_attachment = XFormInstance.objects.get_attachment_by_name
with self.assertRaises(AttachmentNotFound):
get_attachment(form.form_id, 'not_a_form.xml')
with self.assertNumQueries(1, using=form_db):
attachment_meta = get_attachment(form.form_id, 'form.xml')
self.assertEqual(form.form_id, attachment_meta.parent_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
with attachment_meta.open() as content:
self.assertEqual(form_xml, content.read().decode('utf-8'))
def iter_keys(parent_id, name, code):
args = {
"parent_id": parent_id,
"type_code": (CODES.form_xml if "badcode" in action else code),
"name": name,
"key": parent_id + "-" + name,
"content_length": 2,
}
if "dup" not in action:
args["created_on"] = RECEIVED_ON
meta = new_meta(**args)
yield parent_id, code, name
deprecated = "deprecated" in action
if deprecated:
form_id = get_new_id(parent_id)
yield form_id, code, name
else:
form_id = parent_id
attach(meta, form_id, orig_id=(parent_id if deprecated else None))
meta_count = 1
if action != "normal":
meta = self.db.put(BytesIO(b"cx"), meta=meta)
meta_count += 1
if "old" in action:
assert deprecated, action
attach(meta, parent_id, deprecated_form_id=form_id)
if "dup" in action:
meta_count += 1
else:
assert "badcode" not in action, action
meta.delete()
if "x3" in action:
meta_count += 1
third_id = get_new_id(form_id)
attach(meta, third_id, deprecated_form_id=form_id)
yield third_id, code, name
db = get_db_alias_for_partitioned_doc(parent_id)
metas = (
list(BlobMeta.objects.using(db).filter(key=meta.key)) +
list(get_form_attachment_blob_metas_by_key(meta.key, db))
)
assert len(metas) == meta_count, (metas, action, meta_count)
def test_get_with_attachments(self):
form = create_form_for_test(DOMAIN)
form = FormAccessorSQL.get_form(form.form_id) # refetch to clear cached attachments
form_db = get_db_alias_for_partitioned_doc(form.form_id)
with self.assertNumQueries(1, using=form_db):
form.get_attachment_meta('form.xml')
with self.assertNumQueries(1, using=form_db):
form.get_attachment_meta('form.xml')
with self.assertNumQueries(2, using=form_db):
form = FormAccessorSQL.get_with_attachments(form.form_id)
self._check_simple_form(form)
with self.assertNumQueries(0, using=form_db):
attachment_meta = form.get_attachment_meta('form.xml')
self.assertEqual(form.form_id, attachment_meta.parent_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
def attach(meta, form_id, **kw):
# put metadata into xformattachmentsql table
db = get_db_alias_for_partitioned_doc(form_id)
if meta.name == "form.xml":
XFormInstanceSQL(
domain=meta.domain,
form_id=form_id,
received_on=RECEIVED_ON,
xmlns="testing",
**kw
).save(using=db)
DeprecatedXFormAttachmentSQL(
form_id=form_id,
attachment_id=uuid4().hex,
blob_id=meta.key,
blob_bucket="",
name=meta.name,
content_length=meta.content_length,
md5='wrong',
).save(using=db)
def test_get_with_attachments(self):
form = create_form_for_test(DOMAIN)
form = FormAccessorSQL.get_form(
form.form_id) # refetch to clear cached attachments
form_db = get_db_alias_for_partitioned_doc(form.form_id)
with self.assertNumQueries(1, using=form_db):
form.get_attachment_meta('form.xml')
with self.assertNumQueries(1, using=form_db):
form.get_attachment_meta('form.xml')
with self.assertNumQueries(2, using=form_db):
form = FormAccessorSQL.get_with_attachments(form.form_id)
self._check_simple_form(form)
with self.assertNumQueries(0, using=form_db):
attachment_meta = form.get_attachment_meta('form.xml')
self.assertEqual(form.form_id, attachment_meta.parent_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
def get(self, **kw):
"""Get metadata for a single blob
All arguments must be passed as keyword arguments.
:param parent_id: `BlobMeta.parent_id`
:param type_code: `BlobMeta.type_code`
:param name: `BlobMeta.name`
:raises: `BlobMeta.DoesNotExist` if the metadata is not found.
:returns: A `BlobMeta` object.
"""
if set(kw) != {"parent_id", "type_code", "name"}:
# arg check until on Python 3 -> PEP 3102: required keyword args
kw.pop("parent_id", None)
kw.pop("type_code", None)
kw.pop("name", None)
if not kw:
raise TypeError("Missing argument 'name' and/or 'parent_id'")
raise TypeError("Unexpected arguments: {}".format(", ".join(kw)))
dbname = get_db_alias_for_partitioned_doc(kw["parent_id"])
return BlobMeta.objects.using(dbname).get(**kw)
def move_datafile_to_blobmeta(apps, schema_editor):
DataFile = apps.get_model('export', 'DataFile')
BlobMeta = apps.get_model('blobs', 'BlobMeta')
# At time of writing there are only 91 DataFile rows on prod, 1 on icds-new
# this may need to be changed if envs exist having many many more
#
# '_default' is the bucket name from the old blob db API.
for datafile in DataFile.objects.all():
db = get_db_alias_for_partitioned_doc(datafile.domain)
BlobMeta(
domain=datafile.domain,
parent_id=datafile.domain,
type_code=CODES.data_file,
key="_default/" + datafile.blob_id,
properties={
"description": datafile.description
},
content_type=datafile.content_type,
content_length=datafile.content_length,
expires_on=datafile.delete_after,
).save(using=db)
def _do_migration(self, doc):
if not doc.get("external_blobs"):
return True
type_code = self.get_type_code(doc)
obj = self.blob_helper(doc, self.couchdb, type_code)
db = get_db_alias_for_partitioned_doc(doc["_id"])
with connections[db].cursor() as cursor:
for name, meta in six.iteritems(obj.blobs):
if meta.blobmeta_id is not None:
# blobmeta already saved
continue
if obj.domain is None:
print("Unknown domain: {!r}".format(obj))
cursor.execute("""
INSERT INTO blobs_blobmeta (
domain,
type_code,
parent_id,
name,
key,
content_type,
content_length,
created_on
) VALUES (%s, %s, %s, %s, %s, %s, %s, CLOCK_TIMESTAMP())
ON CONFLICT (key) DO NOTHING
""",
params=[
(UNKNOWN_DOMAIN
if obj.domain is None else obj.domain),
type_code,
doc["_id"],
name,
meta.key,
meta.content_type,
meta.content_length or 0,
])
self.total_blobs += 1
return True
def test_blob_expires(self):
now = datetime(2017, 1, 1)
shard = get_db_alias_for_partitioned_doc(self.args["parent_id"])
manager = BlobMeta.objects.using(shard)
pre_expire_count = manager.count()
with capture_log_output(mod.__name__) as logs:
with patch('corehq.blobs.metadata._utcnow', return_value=now):
self.db.put(BytesIO(b'content'), timeout=60, **self.args)
self.assertIsNotNone(self.db.get(key=self.key))
with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=61)):
bytes_deleted = delete_expired_blobs()
self.assertEqual(bytes_deleted, len('content'))
with self.assertRaises(NotFound):
self.db.get(key=self.key)
self.assertEqual(manager.all().count(), pre_expire_count)
self.assertRegexpMatches(
logs.get_output(),
r"deleted expired blobs: .+'blob-identifier'",
)
def case_exists(case_id):
from corehq.sql_db.util import get_db_alias_for_partitioned_doc
db = get_db_alias_for_partitioned_doc(case_id)
return CommCareCaseSQL.objects.using(db).filter(case_id=case_id).exists()
def test_get_case_with_empty_id(self):
db = get_db_alias_for_partitioned_doc('')
with self.assertNumQueries(0,
using=db), self.assertRaises(CaseNotFound):
CommCareCase.objects.get_case('')
def get_meta(meta):
"""Fetch a new copy of the given metadata from the database"""
db = get_db_alias_for_partitioned_doc(meta.parent_id)
return BlobMeta.objects.using(db).get(id=meta.id)
def load(self, key):
parent_id, doc_id = key.rsplit(" ", 1)
dbname = get_db_alias_for_partitioned_doc(parent_id)
obj = self.model_class.objects.using(dbname).get(id=int(doc_id))
return self.doc_to_json(obj)
def get_case_timed_schedule_instances_for_schedule_id(case_id, schedule_id):
from corehq.messaging.scheduling.scheduling_partitioned.models import CaseTimedScheduleInstance
db_name = get_db_alias_for_partitioned_doc(case_id)
return CaseTimedScheduleInstance.objects.using(db_name).filter(
case_id=case_id, timed_schedule_id=schedule_id)
def _delete_data_files(domain_name):
db = get_db_alias_for_partitioned_doc(domain_name)
get_blob_db().bulk_delete(metas=list(BlobMeta.objects.using(db).filter(
parent_id=domain_name,
type_code=CODES.data_file,
)))
def load(self, key):
parent_id, doc_id = key.rsplit(" ", 1)
dbname = get_db_alias_for_partitioned_doc(parent_id)
obj = self.model_class.objects.using(dbname).get(id=int(doc_id))
return self.doc_to_json(obj)
def get_db(partition_value):
from corehq.sql_db.util import get_db_alias_for_partitioned_doc
return get_db_alias_for_partitioned_doc(partition_value)
def get_db(partition_value):
from corehq.sql_db.util import get_db_alias_for_partitioned_doc
return get_db_alias_for_partitioned_doc(partition_value)
