def test_hard_delete_forms_and_attachments(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = sorted(form.form_id for form in forms)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms(
'other_domain', [other_form.form_id]))
attachments = sorted(get_blob_db().metadb.get_for_parents(form_ids),
key=lambda meta: meta.parent_id)
self.assertEqual(3, len(attachments))
deleted = FormAccessorSQL.hard_delete_forms(
DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
for attachment in attachments[1:]:
with self.assertRaises(BlobNotFound):
attachment.open()
with attachments[0].open() as content:
self.assertIsNotNone(content.read())
other_form = FormAccessorSQL.get_form(other_form.form_id)
self.assertIsNotNone(other_form.get_xml())
def test_hard_delete_forms_and_attachments(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = [form.form_id for form in forms]
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id]))
attachments = list(FormAccessorSQL.get_attachments_for_forms(form_ids, ordered=True))
self.assertEqual(3, len(attachments))
deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
for attachment in attachments[1:]:
with self.assertRaises(AttachmentNotFound):
attachment.read_content()
self.assertIsNotNone(attachments[0].read_content())
other_form = FormAccessorSQL.get_form(other_form.form_id)
self.assertIsNotNone(other_form.get_xml())
def test_hard_delete_forms_and_attachments(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = sorted(form.form_id for form in forms)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id]))
attachments = sorted(
get_blob_db().metadb.get_for_parents(form_ids),
key=lambda meta: meta.parent_id
)
self.assertEqual(3, len(attachments))
deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
for attachment in attachments[1:]:
with self.assertRaises(BlobNotFound):
attachment.open()
with attachments[0].open() as content:
self.assertIsNotNone(content.read())
other_form = FormAccessorSQL.get_form(other_form.form_id)
self.assertIsNotNone(other_form.get_xml())
def test_hard_delete_forms_and_attachments(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = [form.form_id for form in forms]
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id]))
attachments = list(FormAccessorSQL.get_attachments_for_forms(form_ids, ordered=True))
self.assertEqual(3, len(attachments))
deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
for attachment in attachments[1:]:
with self.assertRaises(AttachmentNotFound):
attachment.read_content()
self.assertIsNotNone(attachments[0].read_content())
other_form = FormAccessorSQL.get_form(other_form.form_id)
self.assertIsNotNone(other_form.get_xml())
def test_hard_delete_forms(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = [form.form_id for form in forms]
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id]))
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
def test_hard_delete_forms(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = [form.form_id for form in forms]
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id]))
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
def test_get_forms(self):
form1 = create_form_for_test(DOMAIN)
form2 = create_form_for_test(DOMAIN)
forms = FormAccessorSQL.get_forms(['missing_form'])
self.assertEqual(0, len(forms))
forms = FormAccessorSQL.get_forms([form1.form_id])
self.assertEqual(1, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
forms = FormAccessorSQL.get_forms([form1.form_id, form2.form_id], ordered=True)
self.assertEqual(2, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
self.assertEqual(form2.form_id, forms[1].form_id)
def test_get_forms(self):
form1 = create_form_for_test(DOMAIN)
form2 = create_form_for_test(DOMAIN)
forms = FormAccessorSQL.get_forms(['missing_form'])
self.assertEqual(0, len(forms))
forms = FormAccessorSQL.get_forms([form1.form_id])
self.assertEqual(1, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
forms = FormAccessorSQL.get_forms([form1.form_id, form2.form_id], ordered=True)
self.assertEqual(2, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
self.assertEqual(form2.form_id, forms[1].form_id)
def iter_patch_form_diffs(domain, *, kind=None, doc_ids=None, by_kind=None):
if kind:
if by_kind:
raise ValueError("cannot query 'kind' and 'by_kind' together")
if kind not in ["forms", "cases"]:
raise ValueError(f"kind must be 'forms' or 'cases'; got {kind}")
if not doc_ids:
raise ValueError(f"please specify doc ids: --select={kind}:id,...")
by_kind = {kind: doc_ids}
if by_kind:
if by_kind.keys() - {"forms", "cases"}:
kinds = list(by_kind)
raise ValueError(f"valid kinds 'forms' and 'cases'; got {kinds}")
form_ids = by_kind.get("forms", [])
case_ids = by_kind.get("cases", [])
if case_ids:
# may be inefficient for cases with many forms
for case in CaseAccessorSQL.get_cases(case_ids):
form_ids.extend(case.xform_ids)
forms = (f for f in FormAccessorSQL.get_forms(form_ids)
if f.xmlns == PatchForm.xmlns)
else:
# based on iter_form_ids_by_xmlns
q_expr = Q(domain=domain, xmlns=PatchForm.xmlns)
forms = paginate_query_across_partitioned_databases(
XFormInstanceSQL, q_expr, load_source='couch_to_sql_migration')
for form in forms:
yield from iter_doc_diffs(form)
def iter_all_changes(self, start_from=None):
if not self.domains:
return
for form_id_chunk in self._iter_form_id_chunks():
for form in FormAccessorSQL.get_forms(form_id_chunk):
yield Change(
id=form.form_id,
sequence_id=None,
document=form.to_json(),
deleted=False,
metadata=change_meta_from_sql_form(form),
document_store=None,
)
def iter_all_changes(self, start_from=None):
if not self.domains:
return
for form_id_chunk in self._iter_form_id_chunks():
for form in FormAccessorSQL.get_forms(form_id_chunk):
yield Change(
id=form.form_id,
sequence_id=None,
document=form.to_json(),
deleted=False,
metadata=change_meta_from_sql_form(form),
document_store=None,
)
def get_sql_docs(ids):
return {f.form_id: f for f in FormAccessorSQL.get_forms(ids)}
def get_sql_forms(form_id, **kw):
return FormAccessorSQL.get_forms(form_id, **kw)
def get_deleted_form_ids(form_ids):
forms = FormAccessorSQL.get_forms(form_ids)
return {f.form_id for f in forms if f.is_deleted}
