def test_hard_delete_forms_and_attachments(self): forms = [create_form_for_test(DOMAIN) for i in range(3)] form_ids = sorted(form.form_id for form in forms) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(3, len(forms)) other_form = create_form_for_test('other_domain') self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms( 'other_domain', [other_form.form_id])) attachments = sorted(get_blob_db().metadb.get_for_parents(form_ids), key=lambda meta: meta.parent_id) self.assertEqual(3, len(attachments)) deleted = FormAccessorSQL.hard_delete_forms( DOMAIN, form_ids[1:] + [other_form.form_id]) self.assertEqual(2, deleted) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(1, len(forms)) self.assertEqual(form_ids[0], forms[0].form_id) for attachment in attachments[1:]: with self.assertRaises(BlobNotFound): attachment.open() with attachments[0].open() as content: self.assertIsNotNone(content.read()) other_form = FormAccessorSQL.get_form(other_form.form_id) self.assertIsNotNone(other_form.get_xml())
def test_hard_delete_forms_and_attachments(self): forms = [create_form_for_test(DOMAIN) for i in range(3)] form_ids = [form.form_id for form in forms] forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(3, len(forms)) other_form = create_form_for_test('other_domain') self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id])) attachments = list(FormAccessorSQL.get_attachments_for_forms(form_ids, ordered=True)) self.assertEqual(3, len(attachments)) deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id]) self.assertEqual(2, deleted) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(1, len(forms)) self.assertEqual(form_ids[0], forms[0].form_id) for attachment in attachments[1:]: with self.assertRaises(AttachmentNotFound): attachment.read_content() self.assertIsNotNone(attachments[0].read_content()) other_form = FormAccessorSQL.get_form(other_form.form_id) self.assertIsNotNone(other_form.get_xml())
def test_hard_delete_forms_and_attachments(self): forms = [create_form_for_test(DOMAIN) for i in range(3)] form_ids = sorted(form.form_id for form in forms) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(3, len(forms)) other_form = create_form_for_test('other_domain') self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id])) attachments = sorted( get_blob_db().metadb.get_for_parents(form_ids), key=lambda meta: meta.parent_id ) self.assertEqual(3, len(attachments)) deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id]) self.assertEqual(2, deleted) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(1, len(forms)) self.assertEqual(form_ids[0], forms[0].form_id) for attachment in attachments[1:]: with self.assertRaises(BlobNotFound): attachment.open() with attachments[0].open() as content: self.assertIsNotNone(content.read()) other_form = FormAccessorSQL.get_form(other_form.form_id) self.assertIsNotNone(other_form.get_xml())
def test_hard_delete_forms_and_attachments(self): forms = [create_form_for_test(DOMAIN) for i in range(3)] form_ids = [form.form_id for form in forms] forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(3, len(forms)) other_form = create_form_for_test('other_domain') self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id])) attachments = list(FormAccessorSQL.get_attachments_for_forms(form_ids, ordered=True)) self.assertEqual(3, len(attachments)) deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id]) self.assertEqual(2, deleted) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(1, len(forms)) self.assertEqual(form_ids[0], forms[0].form_id) for attachment in attachments[1:]: with self.assertRaises(AttachmentNotFound): attachment.read_content() self.assertIsNotNone(attachments[0].read_content()) other_form = FormAccessorSQL.get_form(other_form.form_id) self.assertIsNotNone(other_form.get_xml())
def test_hard_delete_forms(self): forms = [create_form_for_test(DOMAIN) for i in range(3)] form_ids = [form.form_id for form in forms] other_form = create_form_for_test('other_domain') self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id])) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(3, len(forms)) deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id]) self.assertEqual(2, deleted) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(1, len(forms)) self.assertEqual(form_ids[0], forms[0].form_id)
def test_hard_delete_forms(self): forms = [create_form_for_test(DOMAIN) for i in range(3)] form_ids = [form.form_id for form in forms] other_form = create_form_for_test('other_domain') self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id])) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(3, len(forms)) deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id]) self.assertEqual(2, deleted) forms = FormAccessorSQL.get_forms(form_ids) self.assertEqual(1, len(forms)) self.assertEqual(form_ids[0], forms[0].form_id)
def test_get_forms(self): form1 = create_form_for_test(DOMAIN) form2 = create_form_for_test(DOMAIN) forms = FormAccessorSQL.get_forms(['missing_form']) self.assertEqual(0, len(forms)) forms = FormAccessorSQL.get_forms([form1.form_id]) self.assertEqual(1, len(forms)) self.assertEqual(form1.form_id, forms[0].form_id) forms = FormAccessorSQL.get_forms([form1.form_id, form2.form_id], ordered=True) self.assertEqual(2, len(forms)) self.assertEqual(form1.form_id, forms[0].form_id) self.assertEqual(form2.form_id, forms[1].form_id)
def test_get_forms(self): form1 = create_form_for_test(DOMAIN) form2 = create_form_for_test(DOMAIN) forms = FormAccessorSQL.get_forms(['missing_form']) self.assertEqual(0, len(forms)) forms = FormAccessorSQL.get_forms([form1.form_id]) self.assertEqual(1, len(forms)) self.assertEqual(form1.form_id, forms[0].form_id) forms = FormAccessorSQL.get_forms([form1.form_id, form2.form_id], ordered=True) self.assertEqual(2, len(forms)) self.assertEqual(form1.form_id, forms[0].form_id) self.assertEqual(form2.form_id, forms[1].form_id)
def iter_patch_form_diffs(domain, *, kind=None, doc_ids=None, by_kind=None): if kind: if by_kind: raise ValueError("cannot query 'kind' and 'by_kind' together") if kind not in ["forms", "cases"]: raise ValueError(f"kind must be 'forms' or 'cases'; got {kind}") if not doc_ids: raise ValueError(f"please specify doc ids: --select={kind}:id,...") by_kind = {kind: doc_ids} if by_kind: if by_kind.keys() - {"forms", "cases"}: kinds = list(by_kind) raise ValueError(f"valid kinds 'forms' and 'cases'; got {kinds}") form_ids = by_kind.get("forms", []) case_ids = by_kind.get("cases", []) if case_ids: # may be inefficient for cases with many forms for case in CaseAccessorSQL.get_cases(case_ids): form_ids.extend(case.xform_ids) forms = (f for f in FormAccessorSQL.get_forms(form_ids) if f.xmlns == PatchForm.xmlns) else: # based on iter_form_ids_by_xmlns q_expr = Q(domain=domain, xmlns=PatchForm.xmlns) forms = paginate_query_across_partitioned_databases( XFormInstanceSQL, q_expr, load_source='couch_to_sql_migration') for form in forms: yield from iter_doc_diffs(form)
def iter_all_changes(self, start_from=None): if not self.domains: return for form_id_chunk in self._iter_form_id_chunks(): for form in FormAccessorSQL.get_forms(form_id_chunk): yield Change( id=form.form_id, sequence_id=None, document=form.to_json(), deleted=False, metadata=change_meta_from_sql_form(form), document_store=None, )
def iter_all_changes(self, start_from=None): if not self.domains: return for form_id_chunk in self._iter_form_id_chunks(): for form in FormAccessorSQL.get_forms(form_id_chunk): yield Change( id=form.form_id, sequence_id=None, document=form.to_json(), deleted=False, metadata=change_meta_from_sql_form(form), document_store=None, )
def get_sql_docs(ids): return {f.form_id: f for f in FormAccessorSQL.get_forms(ids)}
def get_sql_forms(form_id, **kw): return FormAccessorSQL.get_forms(form_id, **kw)
def get_deleted_form_ids(form_ids): forms = FormAccessorSQL.get_forms(form_ids) return {f.form_id for f in forms if f.is_deleted}