def test_hard_delete_forms_and_attachments(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = sorted(form.form_id for form in forms)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id]))
attachments = sorted(
get_blob_db().metadb.get_for_parents(form_ids),
key=lambda meta: meta.parent_id
)
self.assertEqual(3, len(attachments))
deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
for attachment in attachments[1:]:
with self.assertRaises(BlobNotFound):
attachment.open()
with attachments[0].open() as content:
self.assertIsNotNone(content.read())
other_form = FormAccessorSQL.get_form(other_form.form_id)
self.assertIsNotNone(other_form.get_xml())
def test_get_forms_received_since(self):
# since this test depends on the global form list just wipe everything
FormProcessorTestUtils.delete_all_sql_forms()
form1 = create_form_for_test(DOMAIN)
form2 = create_form_for_test(DOMAIN)
middle = datetime.utcnow()
time.sleep(.01)
form3 = create_form_for_test(DOMAIN)
form4 = create_form_for_test(DOMAIN)
time.sleep(.01)
end = datetime.utcnow()
forms_back = list(FormAccessorSQL.get_all_forms_received_since())
self.assertEqual(4, len(forms_back))
self.assertEqual(set(form.form_id for form in forms_back),
set([form1.form_id, form2.form_id, form3.form_id, form4.form_id]))
forms_back = list(FormAccessorSQL.get_all_forms_received_since(middle))
self.assertEqual(2, len(forms_back))
self.assertEqual(set(form.form_id for form in forms_back),
set([form3.form_id, form4.form_id]))
self.assertEqual(0, len(list(FormAccessorSQL.get_all_forms_received_since(end))))
self.assertEqual(1, len(list(FormAccessorSQL.get_forms_received_since(limit=1))))
def test_save_new_form(self):
unsaved_form = create_form_for_test(DOMAIN, save=False)
FormAccessorSQL.save_new_form(unsaved_form)
self.assertTrue(unsaved_form.is_saved())
attachments = FormAccessorSQL.get_attachments(unsaved_form.form_id)
self.assertEqual(1, len(attachments))
def test_save_form_deprecated(self):
existing_form, new_form = _simulate_form_edit()
FormAccessorSQL.save_deprecated_form(existing_form)
FormAccessorSQL.save_new_form(new_form)
self._validate_deprecation(existing_form, new_form)
def unarchive(self, user_id=None):
if not self.is_archived:
return
from corehq.form_processor.backends.sql.dbaccessors import FormAccessorSQL
FormAccessorSQL.unarchive_form(self, user_id=user_id)
xform_unarchived.send(sender="form_processor", xform=self)
def _validate_deprecation(self, existing_form, new_form):
saved_new_form = FormAccessorSQL.get_form(new_form.form_id)
deprecated_form = FormAccessorSQL.get_form(existing_form.form_id)
self.assertEqual(deprecated_form.form_id, saved_new_form.deprecated_form_id)
self.assertTrue(deprecated_form.is_deprecated)
self.assertNotEqual(saved_new_form.form_id, deprecated_form.form_id)
self.assertEqual(saved_new_form.form_id, deprecated_form.orig_id)
def test_save_form_deprecated(self):
existing_form, new_form = _simulate_form_edit()
FormAccessorSQL.update_form(existing_form, publish_changes=False)
FormAccessorSQL.save_new_form(new_form)
self._validate_deprecation(existing_form, new_form)
def test_form_with_id_exists(self):
form = create_form_for_test(DOMAIN)
self.assertFalse(FormAccessorSQL.form_exists('not a form'))
self.assertFalse(FormAccessorSQL.form_exists(form.form_id, 'wrong domain'))
self.assertTrue(FormAccessorSQL.form_exists(form.form_id))
self.assertTrue(FormAccessorSQL.form_exists(form.form_id, DOMAIN))
def hard_delete_case_and_forms(cls, domain, case, xforms):
form_ids = [xform.form_id for xform in xforms]
FormAccessorSQL.hard_delete_forms(domain, form_ids)
CaseAccessorSQL.hard_delete_cases(domain, [case.case_id])
for form in xforms:
form.state |= XFormInstanceSQL.DELETED
publish_form_saved(form)
case.deleted = True
publish_case_saved(case)
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(domain, status))
db = get_diff_db(domain)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(
doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
form_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "XFormInstance-Deleted", XFormInstance.get_db())
)
form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status(
"XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase", (0, 0))
has_diffs |= self._print_status(
'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "CommCareCase-Deleted", XFormInstance.get_db())
)
case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase-Deleted", (0, 0))
has_diffs |= self._print_status(
'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(
key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only
)
if diffs_only and not has_diffs:
print(shell_green("No differences found between old and new docs!"))
return has_diffs
def _simulate_form_edit():
existing_form = create_form_for_test(DOMAIN, save=False)
FormAccessorSQL.save_new_form(existing_form)
existing_form = FormAccessorSQL.get_form(existing_form.form_id)
new_form = create_form_for_test(DOMAIN, save=False)
new_form.form_id = existing_form.form_id
existing_form, new_form = apply_deprecation(existing_form, new_form)
assert existing_form.form_id != new_form.form_id
return existing_form, new_form
def test_hard_delete_forms(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = [form.form_id for form in forms]
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id]))
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
def test_get_attachment_by_name(self):
form = create_form_for_test(DOMAIN)
form_xml = get_simple_form_xml(form.form_id)
with self.assertRaises(AttachmentNotFound):
FormAccessorSQL.get_attachment_by_name(form.form_id, 'not_a_form.xml')
with self.assertNumQueries(1, using=db_for_read_write(XFormAttachmentSQL)):
attachment_meta = FormAccessorSQL.get_attachment_by_name(form.form_id, 'form.xml')
self.assertEqual(form.form_id, attachment_meta.form_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
self.assertEqual(form_xml, attachment_meta.read_content())
def save_processed_models(cls, processed_forms, cases=None, stock_updates=None):
with transaction.atomic():
logging.debug('Beginning atomic commit\n')
# Save deprecated form first to avoid ID conflicts
if processed_forms.deprecated:
FormAccessorSQL.save_deprecated_form(processed_forms.deprecated)
FormAccessorSQL.save_new_form(processed_forms.submitted)
if cases:
for case in cases:
CaseAccessorSQL.save_case(case)
for stock_update in stock_updates or []:
stock_update.commit()
cls._publish_changes(processed_forms, cases)
def test_get_forms(self):
form1 = create_form_for_test(DOMAIN)
form2 = create_form_for_test(DOMAIN)
forms = FormAccessorSQL.get_forms(['missing_form'])
self.assertEqual(0, len(forms))
forms = FormAccessorSQL.get_forms([form1.form_id])
self.assertEqual(1, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
forms = FormAccessorSQL.get_forms([form1.form_id, form2.form_id], ordered=True)
self.assertEqual(2, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
self.assertEqual(form2.form_id, forms[1].form_id)
def test_hard_delete_forms(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = [form.form_id for form in forms]
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms(
'other_domain', [other_form.form_id]))
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
deleted = FormAccessorSQL.hard_delete_forms(
DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
def test_save_form_db_error(self):
form = create_form_for_test(DOMAIN)
dup_form = create_form_for_test(DOMAIN, save=False)
dup_form.form_id = form.form_id
try:
FormAccessorSQL.save_new_form(dup_form)
except Exception:
dup_form.form_id = uuid.uuid4().hex
FormAccessorSQL.save_new_form(dup_form)
else:
self.fail("saving dup form didn't raise an exception")
attachments = FormAccessorSQL.get_attachments(dup_form.form_id)
self.assertEqual(1, len(attachments))
def blow_away_migration(domain, state_dir):
assert not should_use_sql_backend(domain)
delete_state_db(domain, state_dir)
log.info("deleting forms...")
for form_ids in iter_chunks(XFormInstanceSQL, "form_id", domain):
FormAccessorSQL.hard_delete_forms(domain,
form_ids,
delete_attachments=False)
log.info("deleting cases...")
for case_ids in iter_chunks(CommCareCaseSQL, "case_id", domain):
CaseAccessorSQL.hard_delete_cases(domain, case_ids)
log.info("blew away migration for domain %s\n", domain)
def test_update_form_problem_and_state(self):
form = create_form_for_test(DOMAIN)
self.assertEqual(XFormInstanceSQL.NORMAL, form.state)
original_domain = form.domain
problem = 'Houston, we have a problem'
form.state = XFormInstanceSQL.ERROR
form.problem = problem
form.domain = 'new domain' # shouldn't get saved
FormAccessorSQL.update_form_problem_and_state(form)
saved_form = FormAccessorSQL.get_form(form.form_id)
self.assertEqual(XFormInstanceSQL.ERROR, saved_form.state)
self.assertEqual(problem, saved_form.problem)
self.assertEqual(original_domain, saved_form.domain)
def test_get_forms(self):
form1 = create_form_for_test(DOMAIN)
form2 = create_form_for_test(DOMAIN)
forms = FormAccessorSQL.get_forms(['missing_form'])
self.assertEqual(0, len(forms))
forms = FormAccessorSQL.get_forms([form1.form_id])
self.assertEqual(1, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
forms = FormAccessorSQL.get_forms([form1.form_id, form2.form_id], ordered=True)
self.assertEqual(2, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
self.assertEqual(form2.form_id, forms[1].form_id)
def test_save_form_db_error(self):
form = create_form_for_test(DOMAIN)
dup_form = create_form_for_test(DOMAIN, save=False)
dup_form.form_id = form.form_id
try:
FormAccessorSQL.save_new_form(dup_form)
except Exception:
dup_form.form_id = uuid.uuid4().hex
FormAccessorSQL.save_new_form(dup_form)
else:
self.fail("saving dup form didn't raise an exception")
attachments = FormAccessorSQL.get_attachments(dup_form.form_id)
self.assertEqual(1, len(attachments))
def test_update_form(self):
form = create_form_for_test(DOMAIN)
form.user_id = 'user2'
operation_date = datetime.utcnow()
form.track_create(XFormOperationSQL(
user_id='user2',
date=operation_date,
operation=XFormOperationSQL.EDIT
))
FormAccessorSQL.update_form(form)
saved_form = FormAccessorSQL.get_form(form.form_id)
self.assertEqual('user2', saved_form.user_id)
self.assertEqual(1, len(saved_form.history))
self.assertEqual(operation_date, saved_form.history[0].date)
def test_update_form_problem_and_state(self):
form = create_form_for_test(DOMAIN)
self.assertEqual(XFormInstanceSQL.NORMAL, form.state)
original_domain = form.domain
problem = 'Houston, we have a problem'
form.state = XFormInstanceSQL.ERROR
form.problem = problem
form.domain = 'new domain' # shouldn't get saved
FormAccessorSQL.update_form_problem_and_state(form)
saved_form = FormAccessorSQL.get_form(form.form_id)
self.assertEqual(XFormInstanceSQL.ERROR, saved_form.state)
self.assertEqual(problem, saved_form.problem)
self.assertEqual(original_domain, saved_form.domain)
def get_diff_stats(domain, state_dir, strict=True):
db = open_state_db(domain, state_dir)
diff_stats = db.get_diff_stats()
stats = {}
def _update_stats(doc_type, couch_count, sql_count):
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
if diff_count or couch_count != sql_count:
stats[doc_type] = (couch_count, sql_count, diff_count,
num_docs_with_diffs)
for doc_type in doc_types():
form_ids_in_couch = len(set(get_form_ids_by_type(domain, doc_type)))
form_ids_in_sql = len(
set(
FormAccessorSQL.get_form_ids_in_domain_by_type(
domain, doc_type)))
_update_stats(doc_type, form_ids_in_couch, form_ids_in_sql)
form_ids_in_couch = len(
set(
get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted",
XFormInstance.get_db())))
form_ids_in_sql = len(
set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain)))
_update_stats("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql)
case_ids_in_couch = len(set(get_case_ids_in_domain(domain)))
case_ids_in_sql = len(set(CaseAccessorSQL.get_case_ids_in_domain(domain)))
_update_stats("CommCareCase", case_ids_in_couch, case_ids_in_sql)
if strict:
# only care about these in strict mode
case_ids_in_couch = len(
set(
get_doc_ids_in_domain_by_type(domain, "CommCareCase-Deleted",
XFormInstance.get_db())))
case_ids_in_sql = len(
set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)))
_update_stats("CommCareCase-Deleted", case_ids_in_couch,
case_ids_in_sql)
if diff_stats:
for key in diff_stats.keys():
_update_stats(key, 0, 0)
return stats
def test_get_forms_with_attachments_meta(self):
attachment_file = open('./corehq/ex-submodules/casexml/apps/case/tests/data/attachments/fruity.jpg', 'rb')
attachments = {
'pic.jpg': UploadedFile(attachment_file, 'pic.jpg', content_type='image/jpeg')
}
form_with_pic = create_form_for_test(DOMAIN, attachments=attachments)
plain_form = create_form_for_test(DOMAIN)
forms = FormAccessorSQL.get_forms_with_attachments_meta(
[form_with_pic.form_id, plain_form.form_id], ordered=True
)
self.assertEqual(2, len(forms))
form = forms[0]
self.assertEqual(form_with_pic.form_id, form.form_id)
with self.assertNumQueries(0, using=db_for_read_write(XFormAttachmentSQL)):
expected = {
'form.xml': 'text/xml',
'pic.jpg': 'image/jpeg',
}
attachments = form.get_attachments()
self.assertEqual(2, len(attachments))
self.assertEqual(expected, {att.name: att.content_type for att in attachments})
with self.assertNumQueries(0, using=db_for_read_write(XFormAttachmentSQL)):
expected = {
'form.xml': 'text/xml',
}
attachments = forms[1].get_attachments()
self.assertEqual(1, len(attachments))
self.assertEqual(expected, {att.name: att.content_type for att in attachments})
def test_get_attachment_by_name(self):
form = create_form_for_test(DOMAIN)
form_xml = get_simple_form_xml(form.form_id)
form_db = get_db_alias_for_partitioned_doc(form.form_id)
with self.assertRaises(AttachmentNotFound):
FormAccessorSQL.get_attachment_by_name(form.form_id, 'not_a_form.xml')
with self.assertNumQueries(1, using=form_db):
attachment_meta = FormAccessorSQL.get_attachment_by_name(form.form_id, 'form.xml')
self.assertEqual(form.form_id, attachment_meta.parent_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
with attachment_meta.open() as content:
self.assertEqual(form_xml, content.read().decode('utf-8'))
def test_get_forms_by_last_modified(self):
start = datetime(2016, 1, 1)
end = datetime(2018, 1, 1)
form1 = create_form_for_test(DOMAIN, received_on=datetime(2017, 1, 1))
create_form_for_test(DOMAIN, received_on=datetime(2015, 1, 1))
# Test that it gets all states
form2 = create_form_for_test(
DOMAIN,
state=XFormInstanceSQL.ARCHIVED,
received_on=datetime(2017, 1, 1)
)
# Test that other date fields are properly fetched
form3 = create_form_for_test(
DOMAIN,
received_on=datetime(2015, 1, 1),
edited_on=datetime(2017, 1, 1),
)
forms = list(FormAccessorSQL.iter_forms_by_last_modified(start, end))
self.assertEqual(3, len(forms))
self.assertEqual(
{form1.form_id, form2.form_id, form3.form_id},
{form.form_id for form in forms},
)
def reprocess_unfinished_stub(stub, save=True):
if stub.saved:
# ignore for now
logger.info("Ignoring 'saved' stub: %s", stub.xform_id)
return
if not should_use_sql_backend(stub.domain):
# ignore for couch domains
logger.info('Removing stub from non SQL domain: %s', stub.xform_id)
save and stub.delete()
return
form_id = stub.xform_id
try:
form = FormAccessorSQL.get_form(form_id)
except XFormNotFound:
# form doesn't exist which means the failure probably happend during saving so
# let mobile handle re-submitting it
logger.error('Form not found: %s', form_id)
save and stub.delete()
return
if form.is_deleted:
save and stub.delete()
if form.is_normal:
result = _reprocess_form(form, save)
save and stub.delete()
return result
def iter_patch_form_diffs(domain, *, kind=None, doc_ids=None, by_kind=None):
if kind:
if by_kind:
raise ValueError("cannot query 'kind' and 'by_kind' together")
if kind not in ["forms", "cases"]:
raise ValueError(f"kind must be 'forms' or 'cases'; got {kind}")
if not doc_ids:
raise ValueError(f"please specify doc ids: --select={kind}:id,...")
by_kind = {kind: doc_ids}
if by_kind:
if by_kind.keys() - {"forms", "cases"}:
kinds = list(by_kind)
raise ValueError(f"valid kinds 'forms' and 'cases'; got {kinds}")
form_ids = by_kind.get("forms", [])
case_ids = by_kind.get("cases", [])
if case_ids:
# may be inefficient for cases with many forms
for case in CaseAccessorSQL.get_cases(case_ids):
form_ids.extend(case.xform_ids)
forms = (f for f in FormAccessorSQL.get_forms(form_ids)
if f.xmlns == PatchForm.xmlns)
else:
# based on iter_form_ids_by_xmlns
q_expr = Q(domain=domain, xmlns=PatchForm.xmlns)
forms = paginate_query_across_partitioned_databases(
XFormInstanceSQL, q_expr, load_source='couch_to_sql_migration')
for form in forms:
yield from iter_doc_diffs(form)
def get_case_transactions(case_id, updated_xforms=None):
transactions = CaseAccessorSQL.get_transactions_for_case_rebuild(case_id)
form_ids = {tx.form_id for tx in transactions}
updated_xforms_map = {
xform.form_id: xform for xform in updated_xforms if not xform.is_deprecated
} if updated_xforms else {}
form_ids_to_fetch = list(form_ids - set(updated_xforms_map.keys()))
xform_map = {form.form_id: form for form in FormAccessorSQL.get_forms_with_attachments_meta(form_ids_to_fetch)}
def get_form(form_id):
if form_id in updated_xforms_map:
return updated_xforms_map[form_id]
try:
return xform_map[form_id]
except KeyError:
raise XFormNotFound
for transaction in transactions:
if transaction.form_id:
try:
transaction.cached_form = get_form(transaction.form_id)
except XFormNotFound:
logging.error('Form not found during rebuild: %s', transaction.form_id)
return transactions
def test_get_forms_by_last_modified(self):
start = datetime(2016, 1, 1)
end = datetime(2018, 1, 1)
form1 = create_form_for_test(DOMAIN, received_on=datetime(2017, 1, 1))
create_form_for_test(DOMAIN, received_on=datetime(2015, 1, 1))
# Test that it gets all states
form2 = create_form_for_test(
DOMAIN,
state=XFormInstanceSQL.ARCHIVED,
received_on=datetime(2017, 1, 1)
)
# Test that other date fields are properly fetched
form3 = create_form_for_test(
DOMAIN,
received_on=datetime(2015, 1, 1),
edited_on=datetime(2017, 1, 1),
)
forms = list(FormAccessorSQL.iter_forms_by_last_modified(start, end))
self.assertEqual(3, len(forms))
self.assertEqual(
{form1.form_id, form2.form_id, form3.form_id},
{form.form_id for form in forms},
)
def test_get_forms_with_attachments_meta(self):
attachment_file = open('./corehq/ex-submodules/casexml/apps/case/tests/data/attachments/fruity.jpg', 'rb')
attachments = {
'pic.jpg': UploadedFile(attachment_file, 'pic.jpg', content_type='image/jpeg')
}
form_with_pic = create_form_for_test(DOMAIN, attachments=attachments)
plain_form = create_form_for_test(DOMAIN)
forms = FormAccessorSQL.get_forms_with_attachments_meta(
[form_with_pic.form_id, plain_form.form_id], ordered=True
)
self.assertEqual(2, len(forms))
form = forms[0]
self.assertEqual(form_with_pic.form_id, form.form_id)
with self.assertNumQueries(0, using=db_for_read_write(XFormAttachmentSQL)):
expected = {
'form.xml': 'text/xml',
'pic.jpg': 'image/jpeg',
}
attachments = form.get_attachments()
self.assertEqual(2, len(attachments))
self.assertEqual(expected, {att.name: att.content_type for att in attachments})
with self.assertNumQueries(0, using=db_for_read_write(XFormAttachmentSQL)):
expected = {
'form.xml': 'text/xml',
}
attachments = forms[1].get_attachments()
self.assertEqual(1, len(attachments))
self.assertEqual(expected, {att.name: att.content_type for att in attachments})
def _get_form_ids(self, domain):
if should_use_sql_backend(domain):
problem_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(
domain, 'XFormError')
else:
problem_ids = get_form_ids_by_type(domain, 'XFormError')
return problem_ids
def get_forms_by_last_modified(start_datetime, end_datetime):
'''
Returns all form ids that have been modified within a time range. The start date is
exclusive while the end date is inclusive (start_datetime, end_datetime].
'''
return FormAccessorSQL.iter_forms_by_last_modified(start_datetime,
end_datetime)
def save_processed_models(cls, processed_forms, cases=None, stock_result=None):
with transaction.atomic():
logging.debug('Beginning atomic commit\n')
# Save deprecated form first to avoid ID conflicts
if processed_forms.deprecated:
FormAccessorSQL.save_deprecated_form(processed_forms.deprecated)
FormAccessorSQL.save_new_form(processed_forms.submitted)
if cases:
for case in cases:
CaseAccessorSQL.save_case(case)
if stock_result:
ledgers_to_save = stock_result.models_to_save
LedgerAccessorSQL.save_ledger_values(ledgers_to_save, processed_forms.deprecated)
cls._publish_changes(processed_forms, cases, stock_result)
def test_save_form_db_error(self):
form = create_form_for_test(DOMAIN)
dup_form = create_form_for_test(DOMAIN, save=False)
dup_form.form_id = form.form_id
try:
with transaction.atomic(dup_form.db):
# prevent rolled back transaction from rolling back the test's transaction
FormAccessorSQL.save_new_form(dup_form)
except Exception:
dup_form.form_id = uuid.uuid4().hex
FormAccessorSQL.save_new_form(dup_form)
else:
self.fail("saving dup form didn't raise an exception")
attachments = FormAccessorSQL.get_attachments(dup_form.form_id)
self.assertEqual(1, len(attachments))
def form(self):
from corehq.form_processor.backends.sql.dbaccessors import FormAccessorSQL
if not self.form_id:
return None
form = getattr(self, 'cached_form', None)
if not form:
self.cached_form = FormAccessorSQL.get_form(self.form_id)
return self.cached_form
def form(self):
from corehq.form_processor.backends.sql.dbaccessors import FormAccessorSQL
if not self.form_id:
return None
form = getattr(self, 'cached_form', None)
if not form:
self.cached_form = FormAccessorSQL.get_form(self.form_id)
return self.cached_form
def save_processed_models(cls, processed_forms, cases=None, stock_result=None):
with transaction.atomic():
logging.debug('Beginning atomic commit\n')
# Save deprecated form first to avoid ID conflicts
if processed_forms.deprecated:
FormAccessorSQL.save_deprecated_form(processed_forms.deprecated)
FormAccessorSQL.save_new_form(processed_forms.submitted)
if cases:
for case in cases:
CaseAccessorSQL.save_case(case)
if stock_result:
ledgers_to_save = stock_result.models_to_save
LedgerAccessorSQL.save_ledger_values(ledgers_to_save, processed_forms.deprecated)
cls._publish_changes(processed_forms, cases, stock_result)
def test_get_attachment_by_name(self):
form = create_form_for_test(DOMAIN)
form_xml = get_simple_form_xml(form.form_id)
with self.assertRaises(AttachmentNotFound):
FormAccessorSQL.get_attachment_by_name(form.form_id,
'not_a_form.xml')
with self.assertNumQueries(
1, using=db_for_read_write(XFormAttachmentSQL)):
attachment_meta = FormAccessorSQL.get_attachment_by_name(
form.form_id, 'form.xml')
self.assertEqual(form.form_id, attachment_meta.form_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
self.assertEqual(form_xml, attachment_meta.read_content())
def save_processed_models(cls,
processed_forms,
cases=None,
stock_result=None,
publish_to_kafka=True):
db_names = {processed_forms.submitted.db}
if processed_forms.deprecated:
db_names |= {processed_forms.deprecated.db}
if cases:
db_names |= {case.db for case in cases}
if stock_result:
db_names |= {
ledger_value.db
for ledger_value in stock_result.models_to_save
}
with ExitStack() as stack:
for db_name in db_names:
stack.enter_context(transaction.atomic(db_name))
# Save deprecated form first to avoid ID conflicts
if processed_forms.deprecated:
FormAccessorSQL.update_form(processed_forms.deprecated,
publish_changes=False)
FormAccessorSQL.save_new_form(processed_forms.submitted)
if cases:
for case in cases:
CaseAccessorSQL.save_case(case)
if stock_result:
ledgers_to_save = stock_result.models_to_save
LedgerAccessorSQL.save_ledger_values(ledgers_to_save,
stock_result)
if cases:
sort_submissions = toggles.SORT_OUT_OF_ORDER_FORM_SUBMISSIONS_SQL.enabled(
processed_forms.submitted.domain, toggles.NAMESPACE_DOMAIN)
if sort_submissions:
for case in cases:
if SqlCaseUpdateStrategy(
case).reconcile_transactions_if_necessary():
CaseAccessorSQL.save_case(case)
def undo_form_edits(form_tuples, logger):
cases_to_rebuild = defaultdict(set)
ledgers_to_rebuild = defaultdict(set)
operation_date = datetime.utcnow()
for live_form, deprecated_form in form_tuples:
# undo corehq.form_processor.parsers.form.apply_deprecation
case_cache = CaseDbCacheSQL(live_form.domain,
load_src="undo_form_edits")
live_case_updates = get_case_updates(live_form)
deprecated_case_updates = get_case_updates(deprecated_form)
case_cache.populate(
set(cu.id for cu in live_case_updates)
| set(cu.id for cu in deprecated_case_updates))
deprecated_form.form_id = new_id_in_same_dbalias(
deprecated_form.form_id)
deprecated_form.state = XFormInstanceSQL.NORMAL
deprecated_form.orig_id = None
deprecated_form.edited_on = None
live_form.deprecated_form_id = None
live_form.received_on = live_form.edited_on
live_form.edited_on = None
affected_cases, affected_ledgers = update_case_transactions_for_form(
case_cache, live_case_updates, deprecated_case_updates, live_form,
deprecated_form)
for form in (live_form, deprecated_form):
form.track_create(
XFormOperationSQL(user_id='system',
operation=XFormOperationSQL.UUID_DATA_FIX,
date=operation_date))
FormAccessorSQL.update_form(form)
logger.log('Form edit undone: {}, {}({})'.format(
live_form.form_id, deprecated_form.form_id,
deprecated_form.original_form_id))
cases_to_rebuild[live_form.domain].update(affected_cases)
ledgers_to_rebuild[live_form.domain].update(affected_ledgers)
logger.log('Cases to rebuild: {}'.format(','.join(affected_cases)))
logger.log('Ledgers to rebuild: {}'.format(','.join(
[l.as_id() for l in affected_ledgers])))
return cases_to_rebuild, ledgers_to_rebuild
def test_get_forms_by_type(self):
form1 = create_form_for_test(DOMAIN)
form2 = create_form_for_test(DOMAIN)
# basic check
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormInstance', 5)
self.assertEqual(2, len(forms))
self.assertEqual({form1.form_id, form2.form_id}, {f.form_id for f in forms})
# check reverse ordering
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormInstance', 5, recent_first=True)
self.assertEqual(2, len(forms))
self.assertEqual([form2.form_id, form1.form_id], [f.form_id for f in forms])
# check limit
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormInstance', 1)
self.assertEqual(1, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
# change state of form1
FormAccessorSQL.archive_form(form1, 'user1')
# check filtering by state
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormArchived', 2)
self.assertEqual(1, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormInstance', 2)
self.assertEqual(1, len(forms))
self.assertEqual(form2.form_id, forms[0].form_id)
def test_hard_delete_forms_none_to_delete(self):
for domain_name in [self.domain.name, self.domain2.name]:
create_form_for_test(domain_name)
create_form_for_test(domain_name, state=XFormInstanceSQL.ARCHIVED)
self.assertEqual(
len(FormAccessors(domain_name).get_all_form_ids_in_domain()),
1)
self.domain.delete()
self.assertEqual(
len(FormAccessors(self.domain.name).get_all_form_ids_in_domain()),
0)
self.assertEqual(
len(FormAccessors(self.domain2.name).get_all_form_ids_in_domain()),
1)
self.assertEqual(
len(
FormAccessorSQL.get_deleted_form_ids_in_domain(
self.domain.name)), 2)
self.assertEqual(
len(
FormAccessorSQL.get_deleted_form_ids_in_domain(
self.domain2.name)), 0)
call_command('hard_delete_forms_and_cases_in_domain',
self.domain2.name,
noinput=True)
self.assertEqual(
len(FormAccessors(self.domain.name).get_all_form_ids_in_domain()),
0)
self.assertEqual(
len(FormAccessors(self.domain2.name).get_all_form_ids_in_domain()),
1)
self.assertEqual(
len(
FormAccessorSQL.get_deleted_form_ids_in_domain(
self.domain.name)), 2)
self.assertEqual(
len(
FormAccessorSQL.get_deleted_form_ids_in_domain(
self.domain2.name)), 0)
def test_get_forms_by_type(self):
form1 = create_form_for_test(DOMAIN)
form2 = create_form_for_test(DOMAIN)
# basic check
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormInstance', 5)
self.assertEqual(2, len(forms))
self.assertEqual({form1.form_id, form2.form_id}, {f.form_id for f in forms})
# check reverse ordering
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormInstance', 5, recent_first=True)
self.assertEqual(2, len(forms))
self.assertEqual([form2.form_id, form1.form_id], [f.form_id for f in forms])
# check limit
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormInstance', 1)
self.assertEqual(1, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
# change state of form1
FormAccessorSQL.archive_form(form1, 'user1')
# check filtering by state
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormArchived', 2)
self.assertEqual(1, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormInstance', 2)
self.assertEqual(1, len(forms))
self.assertEqual(form2.form_id, forms[0].form_id)
def diff_form_state(form_id, *, in_couch=False):
in_couch = in_couch or FormAccessorCouch.form_exists(form_id)
in_sql = FormAccessorSQL.form_exists(form_id)
couch_miss = "missing"
if not in_couch and get_blob_db().metadb.get_for_parent(form_id):
couch_miss = "missing, blob present"
old = {"form_state": "present" if in_couch else couch_miss}
new = {"form_state": "present" if in_sql else "missing"}
return old, new
def diff_form_state(form_id, *, in_couch=False):
in_couch = in_couch or FormAccessorCouch.form_exists(form_id)
in_sql = FormAccessorSQL.form_exists(form_id)
couch_miss = "missing"
if not in_couch and get_blob_db().metadb.get_for_parent(form_id):
couch_miss = MISSING_BLOB_PRESENT
old = {"form_state": FORM_PRESENT if in_couch else couch_miss}
new = {"form_state": FORM_PRESENT if in_sql else "missing"}
return old, new
def test_get_attachment_by_name(self):
form = create_form_for_test(DOMAIN)
form_xml = get_simple_form_xml(form.form_id)
form_db = get_db_alias_for_partitioned_doc(form.form_id)
with self.assertRaises(AttachmentNotFound):
FormAccessorSQL.get_attachment_by_name(form.form_id,
'not_a_form.xml')
with self.assertNumQueries(1, using=form_db):
attachment_meta = FormAccessorSQL.get_attachment_by_name(
form.form_id, 'form.xml')
self.assertEqual(form.form_id, attachment_meta.parent_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
with attachment_meta.open() as content:
self.assertEqual(form_xml, content.read().decode('utf-8'))
def save_processed_models(cls, processed_forms, cases=None, stock_result=None, publish_to_kafka=True):
db_names = {processed_forms.submitted.db}
if processed_forms.deprecated:
db_names |= {processed_forms.deprecated.db}
if cases:
db_names |= {case.db for case in cases}
if stock_result:
db_names |= {
ledger_value.db for ledger_value in stock_result.models_to_save
}
with ExitStack() as stack:
for db_name in db_names:
stack.enter_context(transaction.atomic(db_name))
# Save deprecated form first to avoid ID conflicts
if processed_forms.deprecated:
FormAccessorSQL.update_form(processed_forms.deprecated, publish_changes=False)
FormAccessorSQL.save_new_form(processed_forms.submitted)
if cases:
for case in cases:
CaseAccessorSQL.save_case(case)
if stock_result:
ledgers_to_save = stock_result.models_to_save
LedgerAccessorSQL.save_ledger_values(ledgers_to_save, stock_result)
if cases:
sort_submissions = toggles.SORT_OUT_OF_ORDER_FORM_SUBMISSIONS_SQL.enabled(
processed_forms.submitted.domain, toggles.NAMESPACE_DOMAIN)
if sort_submissions:
for case in cases:
if SqlCaseUpdateStrategy(case).reconcile_transactions_if_necessary():
CaseAccessorSQL.save_case(case)
if publish_to_kafka:
try:
cls.publish_changes_to_kafka(processed_forms, cases, stock_result)
except Exception as e:
raise KafkaPublishingError(e)
def undo_form_edits(form_tuples, logger):
cases_to_rebuild = defaultdict(set)
ledgers_to_rebuild = defaultdict(set)
operation_date = datetime.utcnow()
for live_form, deprecated_form in form_tuples:
# undo corehq.form_processor.parsers.form.apply_deprecation
case_cache = CaseDbCacheSQL(live_form.domain)
live_case_updates = get_case_updates(live_form)
deprecated_case_updates = get_case_updates(deprecated_form)
case_cache.populate(
set(cu.id for cu in live_case_updates) | set(cu.id for cu in deprecated_case_updates)
)
deprecated_form.form_id = new_id_in_same_dbalias(deprecated_form.form_id)
deprecated_form.state = XFormInstanceSQL.NORMAL
deprecated_form.orig_id = None
deprecated_form.edited_on = None
live_form.deprecated_form_id = None
live_form.received_on = live_form.edited_on
live_form.edited_on = None
affected_cases, affected_ledgers = update_case_transactions_for_form(
case_cache, live_case_updates, deprecated_case_updates, live_form, deprecated_form
)
for form in (live_form, deprecated_form):
form.track_create(XFormOperationSQL(
user_id='system',
operation=XFormOperationSQL.UUID_DATA_FIX,
date=operation_date)
)
FormAccessorSQL.update_form(form)
logger.log('Form edit undone: {}, {}({})'.format(
live_form.form_id, deprecated_form.form_id, deprecated_form.original_form_id
))
cases_to_rebuild[live_form.domain].update(affected_cases)
ledgers_to_rebuild[live_form.domain].update(affected_ledgers)
logger.log('Cases to rebuild: {}'.format(','.join(affected_cases)))
logger.log('Ledgers to rebuild: {}'.format(','.join([l.as_id() for l in affected_ledgers])))
return cases_to_rebuild, ledgers_to_rebuild
def save_processed_models(cls,
processed_forms,
cases=None,
stock_result=None,
publish_to_kafka=True):
db_names = {processed_forms.submitted.db}
if processed_forms.deprecated:
db_names |= {processed_forms.deprecated.db}
if cases:
db_names |= {case.db for case in cases}
if stock_result:
db_names |= {
ledger_value.db
for ledger_value in stock_result.models_to_save
}
with ExitStack() as stack:
for db_name in db_names:
stack.enter_context(transaction.atomic(db_name))
# Save deprecated form first to avoid ID conflicts
if processed_forms.deprecated:
FormAccessorSQL.update_form(processed_forms.deprecated,
publish_changes=False)
FormAccessorSQL.save_new_form(processed_forms.submitted)
if cases:
for case in cases:
CaseAccessorSQL.save_case(case)
if stock_result:
ledgers_to_save = stock_result.models_to_save
LedgerAccessorSQL.save_ledger_values(ledgers_to_save,
stock_result)
if publish_to_kafka:
try:
cls.publish_changes_to_kafka(processed_forms, cases,
stock_result)
except Exception as e:
raise KafkaPublishingError(e)
def test_get_with_attachments(self):
form = create_form_for_test(DOMAIN)
form = FormAccessorSQL.get_form(form.form_id) # refetch to clear cached attachments
form_db = get_db_alias_for_partitioned_doc(form.form_id)
with self.assertNumQueries(1, using=form_db):
form.get_attachment_meta('form.xml')
with self.assertNumQueries(1, using=form_db):
form.get_attachment_meta('form.xml')
with self.assertNumQueries(2, using=form_db):
form = FormAccessorSQL.get_with_attachments(form.form_id)
self._check_simple_form(form)
with self.assertNumQueries(0, using=form_db):
attachment_meta = form.get_attachment_meta('form.xml')
self.assertEqual(form.form_id, attachment_meta.parent_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
def test_hard_delete_forms_none_to_delete(self):
for domain_name in [self.domain.name, self.domain2.name]:
create_form_for_test(domain_name)
self.assertEqual(len(FormAccessors(domain_name).get_all_form_ids_in_domain()), 1)
self.domain.delete()
self.assertEqual(len(FormAccessors(self.domain.name).get_all_form_ids_in_domain()), 0)
self.assertEqual(len(FormAccessors(self.domain2.name).get_all_form_ids_in_domain()), 1)
self.assertEqual(len(FormAccessorSQL.get_deleted_form_ids_in_domain(self.domain.name)), 1)
self.assertEqual(len(FormAccessorSQL.get_deleted_form_ids_in_domain(self.domain2.name)), 0)
call_command('hard_delete_forms_and_cases_in_domain', self.domain2.name, noinput=True)
self.assertEqual(len(FormAccessors(self.domain.name).get_all_form_ids_in_domain()), 0)
self.assertEqual(len(FormAccessors(self.domain2.name).get_all_form_ids_in_domain()), 1)
self.assertEqual(len(FormAccessorSQL.get_deleted_form_ids_in_domain(self.domain.name)), 1)
self.assertEqual(len(FormAccessorSQL.get_deleted_form_ids_in_domain(self.domain2.name)), 0)
def test_deleted_form_migration(self):
form = create_and_save_a_form(self.domain_name)
FormAccessors(self.domain.name).soft_delete_forms(
[form.form_id], datetime.utcnow(), 'test-deletion'
)
self.assertEqual(1, len(get_doc_ids_in_domain_by_type(
self.domain_name, "XFormInstance-Deleted", XFormInstance.get_db())
))
self._do_migration_and_assert_flags(self.domain_name)
self.assertEqual(1, len(FormAccessorSQL.get_deleted_form_ids_in_domain(self.domain_name)))
self._compare_diffs([])
def test_get_with_attachments(self):
form = create_form_for_test(DOMAIN)
form = FormAccessorSQL.get_form(
form.form_id) # refetch to clear cached attachments
form_db = get_db_alias_for_partitioned_doc(form.form_id)
with self.assertNumQueries(1, using=form_db):
form.get_attachment_meta('form.xml')
with self.assertNumQueries(1, using=form_db):
form.get_attachment_meta('form.xml')
with self.assertNumQueries(2, using=form_db):
form = FormAccessorSQL.get_with_attachments(form.form_id)
self._check_simple_form(form)
with self.assertNumQueries(0, using=form_db):
attachment_meta = form.get_attachment_meta('form.xml')
self.assertEqual(form.form_id, attachment_meta.parent_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
def test_serialize_attachments(self):
form_id = uuid.uuid4().hex
form_xml = get_simple_form_xml(form_id)
submit_form_locally(form_xml, domain=self.domain)
form = FormAccessorSQL().get_form(form_id)
with self.assertNumQueries(1, using=form.db):
# 1 query to fetch the form.xml attachment. The rest are lazy
form_json = form.to_json(include_attachments=True)
form_xml = form.get_attachment_meta('form.xml')
with self.assertNumQueries(1, using=form.db):
# lazy evaluation of attachments list
self.assertEqual(form_json['external_blobs']['form.xml']['id'],
str(form_xml.key))
# this query goes through pl_proxy
with self.assertNumQueries(1, using=form.db):
# lazy evaluation of history
self.assertEqual(0, len(form_json['history']))
def test_deleted_form_migration(self):
form = create_and_save_a_form(self.domain_name)
FormAccessors(self.domain.name).soft_delete_forms(
[form.form_id], datetime.utcnow(), 'test-deletion'
)
self.assertEqual(1, len(get_doc_ids_in_domain_by_type(
self.domain_name, "XFormInstance-Deleted", XFormInstance.get_db())
))
self._do_migration_and_assert_flags(self.domain_name)
self.assertEqual(1, len(FormAccessorSQL.get_deleted_form_ids_in_domain(self.domain_name)))
self._compare_diffs([])
def test_archive_unarchive_form(self):
case_id = uuid.uuid4().hex
form = create_form_for_test(DOMAIN, case_id=case_id)
self.assertEqual(XFormInstanceSQL.NORMAL, form.state)
self.assertEqual(0, len(form.history))
transactions = CaseAccessorSQL.get_transactions(case_id)
self.assertEqual(1, len(transactions))
self.assertFalse(transactions[0].revoked)
FormAccessorSQL.archive_form(form, 'user1')
form = FormAccessorSQL.get_form(form.form_id)
self.assertEqual(XFormInstanceSQL.ARCHIVED, form.state)
operations = form.history
self.assertEqual(1, len(operations))
self.assertEqual(form.form_id, operations[0].form_id)
self.assertEqual('user1', operations[0].user_id)
transactions = CaseAccessorSQL.get_transactions(case_id)
self.assertEqual(1, len(transactions))
self.assertTrue(transactions[0].revoked)
FormAccessorSQL.unarchive_form(form, 'user2')
form = FormAccessorSQL.get_form(form.form_id)
self.assertEqual(XFormInstanceSQL.NORMAL, form.state)
operations = form.history
self.assertEqual(2, len(operations))
self.assertEqual(form.form_id, operations[1].form_id)
self.assertEqual('user2', operations[1].user_id)
transactions = CaseAccessorSQL.get_transactions(case_id)
self.assertEqual(1, len(transactions))
self.assertFalse(transactions[0].revoked)
def test_hard_delete_forms_and_attachments(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = [form.form_id for form in forms]
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id]))
attachments = list(FormAccessorSQL.get_attachments_for_forms(form_ids, ordered=True))
self.assertEqual(3, len(attachments))
deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
for attachment in attachments[1:]:
with self.assertRaises(AttachmentNotFound):
attachment.read_content()
self.assertIsNotNone(attachments[0].read_content())
other_form = FormAccessorSQL.get_form(other_form.form_id)
self.assertIsNotNone(other_form.get_xml())
def test_get_form_operations(self):
form = create_form_for_test(DOMAIN)
operations = FormAccessorSQL.get_form_operations('missing_form')
self.assertEqual([], operations)
operations = FormAccessorSQL.get_form_operations(form.form_id)
self.assertEqual([], operations)
# don't call form.archive to avoid sending the signals
self.archive_form(form, user_id='user1')
self.unarchive_form(form, user_id='user2')
operations = FormAccessorSQL.get_form_operations(form.form_id)
self.assertEqual(2, len(operations))
self.assertEqual('user1', operations[0].user_id)
self.assertEqual(XFormOperationSQL.ARCHIVE, operations[0].operation)
self.assertIsNotNone(operations[0].date)
self.assertEqual('user2', operations[1].user_id)
self.assertEqual(XFormOperationSQL.UNARCHIVE, operations[1].operation)
self.assertIsNotNone(operations[1].date)
self.assertGreater(operations[1].date, operations[0].date)
