def test_get_forms_with_attachments_meta(self):
attachment_file = open('./corehq/ex-submodules/casexml/apps/case/tests/data/attachments/fruity.jpg', 'rb')
attachments = {
'pic.jpg': UploadedFile(attachment_file, 'pic.jpg', content_type='image/jpeg')
}
form_with_pic = create_form_for_test(DOMAIN, attachments=attachments)
plain_form = create_form_for_test(DOMAIN)
forms = FormAccessorSQL.get_forms_with_attachments_meta(
[form_with_pic.form_id, plain_form.form_id], ordered=True
)
self.assertEqual(2, len(forms))
form = forms[0]
self.assertEqual(form_with_pic.form_id, form.form_id)
with self.assertNumQueries(0, using=db_for_read_write(XFormAttachmentSQL)):
expected = {
'form.xml': 'text/xml',
'pic.jpg': 'image/jpeg',
}
attachments = form.get_attachments()
self.assertEqual(2, len(attachments))
self.assertEqual(expected, {att.name: att.content_type for att in attachments})
with self.assertNumQueries(0, using=db_for_read_write(XFormAttachmentSQL)):
expected = {
'form.xml': 'text/xml',
}
attachments = forms[1].get_attachments()
self.assertEqual(1, len(attachments))
self.assertEqual(expected, {att.name: att.content_type for att in attachments})
def test_hard_delete_forms_and_attachments(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = sorted(form.form_id for form in forms)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id]))
attachments = sorted(
get_blob_db().metadb.get_for_parents(form_ids),
key=lambda meta: meta.parent_id
)
self.assertEqual(3, len(attachments))
deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
for attachment in attachments[1:]:
with self.assertRaises(BlobNotFound):
attachment.open()
with attachments[0].open() as content:
self.assertIsNotNone(content.read())
other_form = FormAccessorSQL.get_form(other_form.form_id)
self.assertIsNotNone(other_form.get_xml())
def test_hard_delete_forms_and_attachments(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = [form.form_id for form in forms]
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id]))
attachments = list(FormAccessorSQL.get_attachments_for_forms(form_ids, ordered=True))
self.assertEqual(3, len(attachments))
deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
for attachment in attachments[1:]:
with self.assertRaises(AttachmentNotFound):
attachment.read_content()
self.assertIsNotNone(attachments[0].read_content())
other_form = FormAccessorSQL.get_form(other_form.form_id)
self.assertIsNotNone(other_form.get_xml())
def test_get_forms_by_type(self):
form1 = create_form_for_test(DOMAIN)
form2 = create_form_for_test(DOMAIN)
# basic check
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormInstance', 5)
self.assertEqual(2, len(forms))
self.assertEqual({form1.form_id, form2.form_id}, {f.form_id for f in forms})
# check reverse ordering
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormInstance', 5, recent_first=True)
self.assertEqual(2, len(forms))
self.assertEqual([form2.form_id, form1.form_id], [f.form_id for f in forms])
# check limit
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormInstance', 1)
self.assertEqual(1, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
# change state of form1
FormAccessorSQL.archive_form(form1, 'user1')
# check filtering by state
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormArchived', 2)
self.assertEqual(1, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormInstance', 2)
self.assertEqual(1, len(forms))
self.assertEqual(form2.form_id, forms[0].form_id)
def test_get_forms_by_last_modified(self):
start = datetime(2016, 1, 1)
end = datetime(2018, 1, 1)
form1 = create_form_for_test(DOMAIN, received_on=datetime(2017, 1, 1))
create_form_for_test(DOMAIN, received_on=datetime(2015, 1, 1))
# Test that it gets all states
form2 = create_form_for_test(
DOMAIN,
state=XFormInstanceSQL.ARCHIVED,
received_on=datetime(2017, 1, 1)
)
# Test that other date fields are properly fetched
form3 = create_form_for_test(
DOMAIN,
received_on=datetime(2015, 1, 1),
edited_on=datetime(2017, 1, 1),
)
forms = list(FormAccessorSQL.iter_forms_by_last_modified(start, end))
self.assertEqual(3, len(forms))
self.assertEqual(
{form1.form_id, form2.form_id, form3.form_id},
{form.form_id for form in forms},
)
def test_get_forms_received_since(self):
# since this test depends on the global form list just wipe everything
FormProcessorTestUtils.delete_all_sql_forms()
form1 = create_form_for_test(DOMAIN)
form2 = create_form_for_test(DOMAIN)
middle = datetime.utcnow()
time.sleep(.01)
form3 = create_form_for_test(DOMAIN)
form4 = create_form_for_test(DOMAIN)
time.sleep(.01)
end = datetime.utcnow()
forms_back = list(FormAccessorSQL.get_all_forms_received_since())
self.assertEqual(4, len(forms_back))
self.assertEqual(set(form.form_id for form in forms_back),
set([form1.form_id, form2.form_id, form3.form_id, form4.form_id]))
forms_back = list(FormAccessorSQL.get_all_forms_received_since(middle))
self.assertEqual(2, len(forms_back))
self.assertEqual(set(form.form_id for form in forms_back),
set([form3.form_id, form4.form_id]))
self.assertEqual(0, len(list(FormAccessorSQL.get_all_forms_received_since(end))))
self.assertEqual(1, len(list(FormAccessorSQL.get_forms_received_since(limit=1))))
def _simulate_form_edit():
existing_form = create_form_for_test(DOMAIN, save=False)
FormAccessorSQL.save_new_form(existing_form)
existing_form = FormAccessorSQL.get_form(existing_form.form_id)
new_form = create_form_for_test(DOMAIN, save=False)
new_form.form_id = existing_form.form_id
existing_form, new_form = apply_deprecation(existing_form, new_form)
assert existing_form.form_id != new_form.form_id
return existing_form, new_form
def test_hard_delete_forms(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = [form.form_id for form in forms]
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id]))
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
def test_get_forms(self):
form1 = create_form_for_test(DOMAIN)
form2 = create_form_for_test(DOMAIN)
forms = FormAccessorSQL.get_forms(['missing_form'])
self.assertEqual(0, len(forms))
forms = FormAccessorSQL.get_forms([form1.form_id])
self.assertEqual(1, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
forms = FormAccessorSQL.get_forms([form1.form_id, form2.form_id], ordered=True)
self.assertEqual(2, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
self.assertEqual(form2.form_id, forms[1].form_id)
def test_save_form_db_error(self):
form = create_form_for_test(DOMAIN)
dup_form = create_form_for_test(DOMAIN, save=False)
dup_form.form_id = form.form_id
try:
FormAccessorSQL.save_new_form(dup_form)
except Exception:
dup_form.form_id = uuid.uuid4().hex
FormAccessorSQL.save_new_form(dup_form)
else:
self.fail("saving dup form didn't raise an exception")
attachments = FormAccessorSQL.get_attachments(dup_form.form_id)
self.assertEqual(1, len(attachments))
def test_form_with_id_exists(self):
form = create_form_for_test(DOMAIN)
self.assertFalse(FormAccessorSQL.form_exists('not a form'))
self.assertFalse(FormAccessorSQL.form_exists(form.form_id, 'wrong domain'))
self.assertTrue(FormAccessorSQL.form_exists(form.form_id))
self.assertTrue(FormAccessorSQL.form_exists(form.form_id, DOMAIN))
def test_archive_unarchive_form(self):
case_id = uuid.uuid4().hex
form = create_form_for_test(DOMAIN, case_id=case_id)
self.assertEqual(XFormInstanceSQL.NORMAL, form.state)
self.assertEqual(0, len(form.history))
transactions = CaseAccessorSQL.get_transactions(case_id)
self.assertEqual(1, len(transactions))
self.assertFalse(transactions[0].revoked)
FormAccessorSQL.archive_form(form, 'user1')
form = FormAccessorSQL.get_form(form.form_id)
self.assertEqual(XFormInstanceSQL.ARCHIVED, form.state)
operations = form.history
self.assertEqual(1, len(operations))
self.assertEqual(form.form_id, operations[0].form_id)
self.assertEqual('user1', operations[0].user_id)
transactions = CaseAccessorSQL.get_transactions(case_id)
self.assertEqual(1, len(transactions))
self.assertTrue(transactions[0].revoked)
FormAccessorSQL.unarchive_form(form, 'user2')
form = FormAccessorSQL.get_form(form.form_id)
self.assertEqual(XFormInstanceSQL.NORMAL, form.state)
operations = form.history
self.assertEqual(2, len(operations))
self.assertEqual(form.form_id, operations[1].form_id)
self.assertEqual('user2', operations[1].user_id)
transactions = CaseAccessorSQL.get_transactions(case_id)
self.assertEqual(1, len(transactions))
self.assertFalse(transactions[0].revoked)
def test_save_new_form(self):
unsaved_form = create_form_for_test(DOMAIN, save=False)
FormAccessorSQL.save_new_form(unsaved_form)
self.assertTrue(unsaved_form.is_saved())
attachments = FormAccessorSQL.get_attachments(unsaved_form.form_id)
self.assertEqual(1, len(attachments))
def test_python_hashing_gives_correct_db(self):
# Rudimentary test to ensure that python sharding matches SQL sharding
num_forms = 100
form_ids = [create_form_for_test(DOMAIN).form_id for i in range(num_forms)]
dbs_for_docs = ShardAccessor.get_database_for_docs(form_ids)
for form_id, db_alias in dbs_for_docs.items():
XFormInstanceSQL.objects.using(db_alias).get(form_id=form_id)
def test_dump_load_form(self):
expected_object_counts = Counter({
XFormInstanceSQL: 2,
BlobMeta: 2
})
pre_forms = [
create_form_for_test(self.domain_name),
create_form_for_test(self.domain_name)
]
self._dump_and_load(expected_object_counts)
form_ids = self.form_accessors.get_all_form_ids_in_domain('XFormInstance')
self.assertEqual(set(form_ids), set(form.form_id for form in pre_forms))
for pre_form in pre_forms:
post_form = self.form_accessors.get_form(pre_form.form_id)
self.assertDictEqual(pre_form.to_json(), post_form.to_json())
def setUpClass(cls):
super().setUpClass()
cls.domain = uuid.uuid4().hex
cls.case_ids = [
uuid.uuid4().hex for i in range(4)
]
with drop_connected_signals(case_post_save), drop_connected_signals(sql_case_post_save):
for case_id in cls.case_ids:
create_form_for_test(cls.domain, case_id)
cls.es = get_es_new()
cls.es_interface = ElasticsearchInterface(cls.es)
cls.index = TEST_INDEX_INFO.index
cls.es_alias = TEST_INDEX_INFO.alias
with trap_extra_setup(ConnectionError):
ensure_index_deleted(cls.index)
initialize_index_and_mapping(cls.es, TEST_INDEX_INFO)
def test_dump_laod_form(self):
expected_object_counts = Counter({
XFormInstanceSQL: 2,
XFormAttachmentSQL: 2
})
pre_forms = [
create_form_for_test(self.domain_name),
create_form_for_test(self.domain_name)
]
self._dump_and_load(expected_object_counts)
form_ids = self.form_accessors.get_all_form_ids_in_domain('XFormInstance')
self.assertEqual(set(form_ids), set(form.form_id for form in pre_forms))
for pre_form in pre_forms:
post_form = self.form_accessors.get_form(pre_form.form_id)
self.assertDictEqual(pre_form.to_json(), post_form.to_json())
def test_hard_delete_forms_none_to_delete(self):
for domain_name in [self.domain.name, self.domain2.name]:
create_form_for_test(domain_name)
self.assertEqual(
len(FormAccessors(domain_name).get_all_form_ids_in_domain()),
1)
self.domain.delete()
self.assertEqual(
len(FormAccessors(self.domain.name).get_all_form_ids_in_domain()),
0)
self.assertEqual(
len(FormAccessors(self.domain2.name).get_all_form_ids_in_domain()),
1)
self.assertEqual(
len(
FormAccessorSQL.get_deleted_form_ids_in_domain(
self.domain.name)), 1)
self.assertEqual(
len(
FormAccessorSQL.get_deleted_form_ids_in_domain(
self.domain2.name)), 0)
call_command('hard_delete_forms_and_cases_in_domain',
self.domain2.name,
noinput=True)
self.assertEqual(
len(FormAccessors(self.domain.name).get_all_form_ids_in_domain()),
0)
self.assertEqual(
len(FormAccessors(self.domain2.name).get_all_form_ids_in_domain()),
1)
self.assertEqual(
len(
FormAccessorSQL.get_deleted_form_ids_in_domain(
self.domain.name)), 1)
self.assertEqual(
len(
FormAccessorSQL.get_deleted_form_ids_in_domain(
self.domain2.name)), 0)
def setUpClass(cls):
super(FormFactIntegrationTest, cls).setUpClass()
delete_all_docs_by_doc_type(Domain.get_db(), ['Domain', 'Domain-Deleted'])
delete_all_docs_by_doc_type(CommCareUser.get_db(), ['CommCareUser', 'WebUser'])
cls.domain_records = [
Domain(name=cls.domain, hr_name='One', creating_user_id='abc', is_active=True),
]
for domain in cls.domain_records:
domain.save()
cls.user_records = [
# TODO: Handle WebUsers who have multiple domains
# WebUser.create(
# cls.domain,
# 'web-user',
# '***',
# date_joined=datetime.utcnow(),
# first_name='A',
# last_name='B',
# email='*****@*****.**',
# is_active=True,
# is_staff=False,
# is_superuser=True,
# ),
CommCareUser.create(
cls.domain,
'commcare-user',
'***',
date_joined=datetime.utcnow(),
email='*****@*****.**',
is_active=True,
is_staff=True,
is_superuser=False,
),
]
cls.form_records = [
create_form_for_test(cls.domain, user_id=cls.user_records[0]._id),
create_form_for_test(cls.domain, user_id=cls.user_records[0]._id),
create_form_for_test(cls.domain, user_id=cls.user_records[0]._id),
]
cls.batch = create_batch(cls.slug)
def test_python_hashing_gives_correct_db(self):
# Rudimentary test to ensure that python sharding matches SQL sharding
num_forms = 100
form_ids = [
create_form_for_test(DOMAIN).form_id for i in range(num_forms)
]
dbs_for_docs = ShardAccessor.get_database_for_docs(form_ids)
for form_id, db_alias in dbs_for_docs.items():
XFormInstanceSQL.objects.using(db_alias).get(form_id=form_id)
def test_hard_delete_forms_none_to_delete(self):
for domain_name in [self.domain.name, self.domain2.name]:
create_form_for_test(domain_name)
self.assertEqual(len(FormAccessors(domain_name).get_all_form_ids_in_domain()), 1)
self.domain.delete()
self.assertEqual(len(FormAccessors(self.domain.name).get_all_form_ids_in_domain()), 0)
self.assertEqual(len(FormAccessors(self.domain2.name).get_all_form_ids_in_domain()), 1)
self.assertEqual(len(FormAccessorSQL.get_deleted_form_ids_in_domain(self.domain.name)), 1)
self.assertEqual(len(FormAccessorSQL.get_deleted_form_ids_in_domain(self.domain2.name)), 0)
call_command('hard_delete_forms_and_cases_in_domain', self.domain2.name, noinput=True)
self.assertEqual(len(FormAccessors(self.domain.name).get_all_form_ids_in_domain()), 0)
self.assertEqual(len(FormAccessors(self.domain2.name).get_all_form_ids_in_domain()), 1)
self.assertEqual(len(FormAccessorSQL.get_deleted_form_ids_in_domain(self.domain.name)), 1)
self.assertEqual(len(FormAccessorSQL.get_deleted_form_ids_in_domain(self.domain2.name)), 0)
def test_write_blob_bucket(self):
form = create_form_for_test(DOMAIN)
attachments = FormAccessorSQL.get_attachments(form.form_id)
self.assertEqual(1, len(attachments))
self.assertEqual(None, attachments[0].blob_bucket)
FormAccessorSQL.write_blob_bucket(attachments[0], 'new-bucket')
attachments = FormAccessorSQL.get_attachments(form.form_id)
self.assertEqual(1, len(attachments))
self.assertEqual('new-bucket', attachments[0].blob_bucket)
def test_get_form_ids_in_domain(self):
form1 = create_form_for_test(DOMAIN)
form2 = create_form_for_test(DOMAIN)
create_form_for_test('bad-domain')
# basic check
form_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(DOMAIN, 'XFormInstance')
self.assertEqual(2, len(form_ids))
self.assertEqual({form1.form_id, form2.form_id}, set(form_ids))
# change state of form1
FormAccessorSQL.archive_form(form1, 'user1')
# check filtering by state
form_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(DOMAIN, 'XFormArchived')
self.assertEqual(1, len(form_ids))
self.assertEqual(form1.form_id, form_ids[0])
form_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(DOMAIN, 'XFormInstance')
self.assertEqual(1, len(form_ids))
self.assertEqual(form2.form_id, form_ids[0])
def test_get_form_ids_in_domain(self):
form1 = create_form_for_test(DOMAIN)
form2 = create_form_for_test(DOMAIN)
create_form_for_test('bad-domain')
# basic check
form_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(DOMAIN, 'XFormInstance')
self.assertEqual(2, len(form_ids))
self.assertEqual({form1.form_id, form2.form_id}, set(form_ids))
# change state of form1
FormAccessorSQL.archive_form(form1, 'user1')
# check filtering by state
form_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(DOMAIN, 'XFormArchived')
self.assertEqual(1, len(form_ids))
self.assertEqual(form1.form_id, form_ids[0])
form_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(DOMAIN, 'XFormInstance')
self.assertEqual(1, len(form_ids))
self.assertEqual(form2.form_id, form_ids[0])
def test_get_forms_with_attachments_meta(self):
attachment_file = open(
'./corehq/ex-submodules/casexml/apps/case/tests/data/attachments/fruity.jpg',
'rb')
attachments = {
'pic.jpg':
UploadedFile(attachment_file, 'pic.jpg', content_type='image/jpeg')
}
form_with_pic = create_form_for_test(DOMAIN, attachments=attachments)
plain_form = create_form_for_test(DOMAIN)
forms = FormAccessorSQL.get_forms_with_attachments_meta(
[form_with_pic.form_id, plain_form.form_id], ordered=True)
self.assertEqual(2, len(forms))
form = forms[0]
self.assertEqual(form_with_pic.form_id, form.form_id)
with self.assertNumQueries(
0, using=db_for_read_write(XFormAttachmentSQL)):
expected = {
'form.xml': 'text/xml',
'pic.jpg': 'image/jpeg',
}
attachments = form.get_attachments()
self.assertEqual(2, len(attachments))
self.assertEqual(
expected, {att.name: att.content_type
for att in attachments})
with self.assertNumQueries(
0, using=db_for_read_write(XFormAttachmentSQL)):
expected = {
'form.xml': 'text/xml',
}
attachments = forms[1].get_attachments()
self.assertEqual(1, len(attachments))
self.assertEqual(
expected, {att.name: att.content_type
for att in attachments})
def test_form_deletion(self):
form_states = [state_tuple[0] for state_tuple in XFormInstanceSQL.STATES]
for domain_name in [self.domain.name, self.domain2.name]:
for form_state in form_states:
create_form_for_test(domain_name, state=form_state)
for doc_type in doc_type_to_state:
self.assertEqual(
len(FormAccessors(domain_name).get_all_form_ids_in_domain(doc_type=doc_type)),
1
)
self.domain.delete()
for doc_type in doc_type_to_state:
self.assertEqual(
len(FormAccessors(self.domain.name).get_all_form_ids_in_domain(doc_type=doc_type)),
0
)
self.assertEqual(
len(FormAccessors(self.domain2.name).get_all_form_ids_in_domain(doc_type=doc_type)),
1
)
def test_get_attachment_by_name(self):
form = create_form_for_test(DOMAIN)
form_xml = get_simple_form_xml(form.form_id)
with self.assertRaises(AttachmentNotFound):
FormAccessorSQL.get_attachment_by_name(form.form_id, 'not_a_form.xml')
with self.assertNumQueries(1, using=db_for_read_write(XFormAttachmentSQL)):
attachment_meta = FormAccessorSQL.get_attachment_by_name(form.form_id, 'form.xml')
self.assertEqual(form.form_id, attachment_meta.form_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
self.assertEqual(form_xml, attachment_meta.read_content())
def test_form_deletion(self):
form_states = [state_tuple[0] for state_tuple in XFormInstanceSQL.STATES]
for domain_name in [self.domain.name, self.domain2.name]:
for form_state in form_states:
create_form_for_test(domain_name, state=form_state)
for doc_type in doc_type_to_state:
self.assertEqual(
len(FormAccessors(domain_name).get_all_form_ids_in_domain(doc_type=doc_type)),
1
)
self.domain.delete()
for doc_type in doc_type_to_state:
self.assertEqual(
len(FormAccessors(self.domain.name).get_all_form_ids_in_domain(doc_type=doc_type)),
0
)
self.assertEqual(
len(FormAccessors(self.domain2.name).get_all_form_ids_in_domain(doc_type=doc_type)),
1
)
def test_update_form(self):
form = create_form_for_test(DOMAIN)
form.user_id = 'user2'
operation_date = datetime.utcnow()
form.track_create(
XFormOperationSQL(user_id='user2',
date=operation_date,
operation=XFormOperationSQL.EDIT))
FormAccessorSQL.update_form(form)
saved_form = FormAccessorSQL.get_form(form.form_id)
self.assertEqual('user2', saved_form.user_id)
self.assertEqual(1, len(saved_form.history))
self.assertEqual(operation_date, saved_form.history[0].date)
def test_get_attachment_by_name(self):
form = create_form_for_test(DOMAIN)
form_xml = get_simple_form_xml(form.form_id)
with self.assertRaises(AttachmentNotFound):
FormAccessorSQL.get_attachment_by_name(form.form_id, 'not_a_form.xml')
with self.assertNumQueries(1, using=db_for_read_write(XFormAttachmentSQL)):
attachment_meta = FormAccessorSQL.get_attachment_by_name(form.form_id, 'form.xml')
self.assertEqual(form.form_id, attachment_meta.form_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
self.assertEqual(form_xml, attachment_meta.read_content())
def test_update_form(self):
form = create_form_for_test(DOMAIN)
form.user_id = 'user2'
operation_date = datetime.utcnow()
form.track_create(XFormOperationSQL(
user_id='user2',
date=operation_date,
operation=XFormOperationSQL.EDIT
))
FormAccessorSQL.update_form(form)
saved_form = FormAccessorSQL.get_form(form.form_id)
self.assertEqual('user2', saved_form.user_id)
self.assertEqual(1, len(saved_form.history))
self.assertEqual(operation_date, saved_form.history[0].date)
def test_get_with_attachments(self):
form = create_form_for_test(DOMAIN)
with self.assertNumQueries(1, using=db_for_read_write(XFormAttachmentSQL)):
form.get_attachment_meta('form.xml')
with self.assertNumQueries(2, using=db_for_read_write(XFormAttachmentSQL)):
form = FormAccessorSQL.get_with_attachments(form.form_id)
self._check_simple_form(form)
with self.assertNumQueries(0, using=db_for_read_write(XFormAttachmentSQL)):
attachment_meta = form.get_attachment_meta('form.xml')
self.assertEqual(form.form_id, attachment_meta.form_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
def test_objects_distributed_to_all_dbs(self):
"""
Rudimentary test to ensure that not all cases / forms get saved to the same DB.
"""
num_forms = 20
for i in range(num_forms):
create_form_for_test(DOMAIN, case_id=uuid4().hex)
forms_per_db = {}
cases_per_db = {}
for db in partition_config.get_form_processing_dbs():
forms_per_db[db] = XFormInstanceSQL.objects.using(db).filter(domain=DOMAIN).count()
cases_per_db[db] = CommCareCaseSQL.objects.using(db).filter(domain=DOMAIN).count()
self.assertEqual(num_forms, sum(forms_per_db.values()), forms_per_db)
self.assertEqual(num_forms, sum(cases_per_db.values()), cases_per_db)
self.assertTrue(
all(num_forms_in_db < num_forms for num_forms_in_db in forms_per_db.values()),
forms_per_db
)
self.assertTrue(
all(num_cases_in_db < num_forms for num_cases_in_db in cases_per_db.values()),
cases_per_db
)
def test_objects_distributed_to_all_dbs(self):
"""
Rudimentary test to ensure that not all cases / forms get saved to the same DB.
"""
num_forms = 20
for i in range(num_forms):
create_form_for_test(DOMAIN, case_id=uuid4().hex)
forms_per_db = {}
cases_per_db = {}
for db in partition_config.get_form_processing_dbs():
forms_per_db[db] = XFormInstanceSQL.objects.using(db).filter(domain=DOMAIN).count()
cases_per_db[db] = CommCareCaseSQL.objects.using(db).filter(domain=DOMAIN).count()
self.assertEqual(num_forms, sum(forms_per_db.values()), forms_per_db)
self.assertEqual(num_forms, sum(cases_per_db.values()), cases_per_db)
self.assertTrue(
all(num_forms_in_db < num_forms for num_forms_in_db in forms_per_db.values()),
forms_per_db
)
self.assertTrue(
all(num_cases_in_db < num_forms for num_cases_in_db in cases_per_db.values()),
cases_per_db
)
def test_get_with_attachments(self):
form = create_form_for_test(DOMAIN)
with self.assertNumQueries(1, using=db_for_read_write(XFormAttachmentSQL)):
form.get_attachment_meta('form.xml')
with self.assertNumQueries(2, using=db_for_read_write(XFormAttachmentSQL)):
form = FormAccessorSQL.get_with_attachments(form.form_id)
self._check_simple_form(form)
with self.assertNumQueries(0, using=db_for_read_write(XFormAttachmentSQL)):
attachment_meta = form.get_attachment_meta('form.xml')
self.assertEqual(form.form_id, attachment_meta.form_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
def test_get_forms_by_last_modified(self):
start = datetime(2016, 1, 1)
end = datetime(2018, 1, 1)
form1 = create_form_for_test(DOMAIN, received_on=datetime(2017, 1, 1))
create_form_for_test(DOMAIN, received_on=datetime(2015, 1, 1))
# Test that it gets all states
form2 = create_form_for_test(DOMAIN,
state=XFormInstanceSQL.ARCHIVED,
received_on=datetime(2017, 1, 1))
# Test that other date fields are properly fetched
form3 = create_form_for_test(
DOMAIN,
received_on=datetime(2015, 1, 1),
edited_on=datetime(2017, 1, 1),
)
forms = list(FormAccessorSQL.iter_forms_by_last_modified(start, end))
self.assertEqual(3, len(forms))
self.assertEqual(
{form1.form_id, form2.form_id, form3.form_id},
{form.form_id
for form in forms},
)
def test_update_form_problem_and_state(self):
form = create_form_for_test(DOMAIN)
self.assertEqual(XFormInstanceSQL.NORMAL, form.state)
original_domain = form.domain
problem = 'Houston, we have a problem'
form.state = XFormInstanceSQL.ERROR
form.problem = problem
form.domain = 'new domain' # shouldn't get saved
FormAccessorSQL.update_form_problem_and_state(form)
saved_form = FormAccessorSQL.get_form(form.form_id)
self.assertEqual(XFormInstanceSQL.ERROR, saved_form.state)
self.assertEqual(problem, saved_form.problem)
self.assertEqual(original_domain, saved_form.domain)
def test_get_forms_by_type(self):
form1 = create_form_for_test(DOMAIN)
form2 = create_form_for_test(DOMAIN)
# basic check
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormInstance', 5)
self.assertEqual(2, len(forms))
self.assertEqual({form1.form_id, form2.form_id},
{f.form_id
for f in forms})
# check reverse ordering
forms = FormAccessorSQL.get_forms_by_type(DOMAIN,
'XFormInstance',
5,
recent_first=True)
self.assertEqual(2, len(forms))
self.assertEqual([form2.form_id, form1.form_id],
[f.form_id for f in forms])
# check limit
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormInstance', 1)
self.assertEqual(1, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
# change state of form1
self.archive_form(form1, 'user1')
# check filtering by state
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormArchived', 2)
self.assertEqual(1, len(forms))
self.assertEqual(form1.form_id, forms[0].form_id)
forms = FormAccessorSQL.get_forms_by_type(DOMAIN, 'XFormInstance', 2)
self.assertEqual(1, len(forms))
self.assertEqual(form2.form_id, forms[0].form_id)
def test_update_form_problem_and_state(self):
form = create_form_for_test(DOMAIN)
self.assertEqual(XFormInstanceSQL.NORMAL, form.state)
original_domain = form.domain
problem = 'Houston, we have a problem'
form.state = XFormInstanceSQL.ERROR
form.problem = problem
form.domain = 'new domain' # shouldn't get saved
FormAccessorSQL.update_form_problem_and_state(form)
saved_form = FormAccessorSQL.get_form(form.form_id)
self.assertEqual(XFormInstanceSQL.ERROR, saved_form.state)
self.assertEqual(problem, saved_form.problem)
self.assertEqual(original_domain, saved_form.domain)
def test_get_attachment_by_name(self):
form = create_form_for_test(DOMAIN)
form_xml = get_simple_form_xml(form.form_id)
form_db = get_db_alias_for_partitioned_doc(form.form_id)
with self.assertRaises(AttachmentNotFound):
FormAccessorSQL.get_attachment_by_name(form.form_id, 'not_a_form.xml')
with self.assertNumQueries(1, using=form_db):
attachment_meta = FormAccessorSQL.get_attachment_by_name(form.form_id, 'form.xml')
self.assertEqual(form.form_id, attachment_meta.parent_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
with attachment_meta.open() as content:
self.assertEqual(form_xml, content.read().decode('utf-8'))
def test_objects_only_in_one_db(self):
case_id = uuid4().hex
form = create_form_for_test(DOMAIN, case_id=case_id)
dbs_with_form = []
dbs_with_case = []
for db in partition_config.get_form_processing_dbs():
form_in_db = XFormInstanceSQL.objects.using(db).filter(form_id=form.form_id).exists()
if form_in_db:
dbs_with_form.append(db)
case_in_db = CommCareCaseSQL.objects.using(db).filter(case_id=case_id).exists()
if case_in_db:
dbs_with_case.append(db)
self.assertEqual(1, len(dbs_with_form))
self.assertEqual(1, len(dbs_with_case))
def test_objects_only_in_one_db(self):
case_id = uuid4().hex
form = create_form_for_test(DOMAIN, case_id=case_id)
dbs_with_form = []
dbs_with_case = []
for db in partition_config.get_form_processing_dbs():
form_in_db = XFormInstanceSQL.objects.using(db).filter(form_id=form.form_id).exists()
if form_in_db:
dbs_with_form.append(db)
case_in_db = CommCareCaseSQL.objects.using(db).filter(case_id=case_id).exists()
if case_in_db:
dbs_with_case.append(db)
self.assertEqual(1, len(dbs_with_form))
self.assertEqual(1, len(dbs_with_case))
def test_get_attachment_by_name(self):
form = create_form_for_test(DOMAIN)
form_xml = get_simple_form_xml(form.form_id)
form_db = get_db_alias_for_partitioned_doc(form.form_id)
with self.assertRaises(AttachmentNotFound):
FormAccessorSQL.get_attachment_by_name(form.form_id,
'not_a_form.xml')
with self.assertNumQueries(1, using=form_db):
attachment_meta = FormAccessorSQL.get_attachment_by_name(
form.form_id, 'form.xml')
self.assertEqual(form.form_id, attachment_meta.parent_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
with attachment_meta.open() as content:
self.assertEqual(form_xml, content.read().decode('utf-8'))
def test_get_with_attachments(self):
form = create_form_for_test(DOMAIN)
form = FormAccessorSQL.get_form(form.form_id) # refetch to clear cached attachments
form_db = get_db_alias_for_partitioned_doc(form.form_id)
with self.assertNumQueries(1, using=form_db):
form.get_attachment_meta('form.xml')
with self.assertNumQueries(1, using=form_db):
form.get_attachment_meta('form.xml')
with self.assertNumQueries(2, using=form_db):
form = FormAccessorSQL.get_with_attachments(form.form_id)
self._check_simple_form(form)
with self.assertNumQueries(0, using=form_db):
attachment_meta = form.get_attachment_meta('form.xml')
self.assertEqual(form.form_id, attachment_meta.parent_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
def test_get_with_attachments(self):
form = create_form_for_test(DOMAIN)
form = FormAccessorSQL.get_form(
form.form_id) # refetch to clear cached attachments
form_db = get_db_alias_for_partitioned_doc(form.form_id)
with self.assertNumQueries(1, using=form_db):
form.get_attachment_meta('form.xml')
with self.assertNumQueries(1, using=form_db):
form.get_attachment_meta('form.xml')
with self.assertNumQueries(2, using=form_db):
form = FormAccessorSQL.get_with_attachments(form.form_id)
self._check_simple_form(form)
with self.assertNumQueries(0, using=form_db):
attachment_meta = form.get_attachment_meta('form.xml')
self.assertEqual(form.form_id, attachment_meta.parent_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
def test_get_with_attachments(self):
form = create_form_for_test(DOMAIN)
with self.assertNumQueries(1, using=db_for_read_write(XFormAttachmentSQL)):
form.get_attachment_meta('form.xml')
with ExitStack() as stack:
if settings.USE_PARTITIONED_DATABASE:
proxy_queries = 1
stack.enter_context(self.assertNumQueries(1, using=form.db))
else:
proxy_queries = 2
stack.enter_context(self.assertNumQueries(proxy_queries, using=db_for_read_write(XFormAttachmentSQL)))
form = FormAccessorSQL.get_with_attachments(form.form_id)
self._check_simple_form(form)
with self.assertNumQueries(0, using=db_for_read_write(XFormAttachmentSQL)):
attachment_meta = form.get_attachment_meta('form.xml')
self.assertEqual(form.form_id, attachment_meta.form_id)
self.assertEqual('form.xml', attachment_meta.name)
self.assertEqual('text/xml', attachment_meta.content_type)
def test_get_form_operations(self):
form = create_form_for_test(DOMAIN)
operations = FormAccessorSQL.get_form_operations('missing_form')
self.assertEqual([], operations)
operations = FormAccessorSQL.get_form_operations(form.form_id)
self.assertEqual([], operations)
# don't call form.archive to avoid sending the signals
FormAccessorSQL.archive_form(form, user_id='user1')
FormAccessorSQL.unarchive_form(form, user_id='user2')
operations = FormAccessorSQL.get_form_operations(form.form_id)
self.assertEqual(2, len(operations))
self.assertEqual('user1', operations[0].user_id)
self.assertEqual(XFormOperationSQL.ARCHIVE, operations[0].operation)
self.assertIsNotNone(operations[0].date)
self.assertEqual('user2', operations[1].user_id)
self.assertEqual(XFormOperationSQL.UNARCHIVE, operations[1].operation)
self.assertIsNotNone(operations[1].date)
self.assertGreater(operations[1].date, operations[0].date)
def test_get_form_operations(self):
form = create_form_for_test(DOMAIN)
operations = FormAccessorSQL.get_form_operations('missing_form')
self.assertEqual([], operations)
operations = FormAccessorSQL.get_form_operations(form.form_id)
self.assertEqual([], operations)
# don't call form.archive to avoid sending the signals
self.archive_form(form, user_id='user1')
self.unarchive_form(form, user_id='user2')
operations = FormAccessorSQL.get_form_operations(form.form_id)
self.assertEqual(2, len(operations))
self.assertEqual('user1', operations[0].user_id)
self.assertEqual(XFormOperationSQL.ARCHIVE, operations[0].operation)
self.assertIsNotNone(operations[0].date)
self.assertEqual('user2', operations[1].user_id)
self.assertEqual(XFormOperationSQL.UNARCHIVE, operations[1].operation)
self.assertIsNotNone(operations[1].date)
self.assertGreater(operations[1].date, operations[0].date)
def test_archive_unarchive_form(self):
case_id = uuid.uuid4().hex
form = create_form_for_test(DOMAIN, case_id=case_id)
self.assertEqual(XFormInstanceSQL.NORMAL, form.state)
self.assertEqual(0, len(form.history))
transactions = CaseAccessorSQL.get_transactions(case_id)
self.assertEqual(1, len(transactions))
self.assertFalse(transactions[0].revoked)
# archive twice to check that it's idempotent
for i in range(2):
self.archive_form(form, 'user1')
form = FormAccessorSQL.get_form(form.form_id)
self.assertEqual(XFormInstanceSQL.ARCHIVED, form.state)
operations = form.history
self.assertEqual(i + 1, len(operations))
self.assertEqual(form.form_id, operations[i].form_id)
self.assertEqual('user1', operations[i].user_id)
transactions = CaseAccessorSQL.get_transactions(case_id)
self.assertEqual(1, len(transactions), transactions)
self.assertTrue(transactions[0].revoked)
# unarchive twice to check that it's idempotent
for i in range(2, 4):
self.unarchive_form(form, 'user2')
form = FormAccessorSQL.get_form(form.form_id)
self.assertEqual(XFormInstanceSQL.NORMAL, form.state)
operations = form.history
self.assertEqual(i + 1, len(operations))
self.assertEqual(form.form_id, operations[i].form_id)
self.assertEqual('user2', operations[i].user_id)
transactions = CaseAccessorSQL.get_transactions(case_id)
self.assertEqual(1, len(transactions))
self.assertFalse(transactions[0].revoked)
def test_update_dependent_case(self):
sync_log = SimplifiedSyncLog(
case_ids_on_phone={'bran', 'hodor'},
dependent_case_ids_on_phone={'hodor'},
index_tree=IndexTree(indices={'bran': {
'legs': 'hodor'
}}),
user_id="someuser")
xform_id = uuid.uuid4().hex
xform = create_form_for_test("domain", form_id=xform_id, save=False)
form_actions = [
CaseAction(
action_type=CASE_ACTION_UPDATE,
updated_known_properties={},
indices=[],
)
]
with patch.object(CommCareCase,
'get_actions_for_form',
return_value=form_actions):
parent_case = CommCareCase(case_id='hodor')
# before this test was added, the following call raised a SyncLogAssertionError on legacy logs.
# this test just ensures it doesn't still do that.
sync_log.update_phone_lists(xform, [parent_case])
def _create_form(domain, doc_type):
form = create_form_for_test(domain, state=doc_type_to_state[doc_type])
return form.form_id
def test_get_form_by_id(self):
form = create_form_for_test(DOMAIN)
with self.assertNumQueries(1, using=db_for_read_write(XFormInstanceSQL)):
form = FormAccessorSQL.get_form(form.form_id)
self._check_simple_form(form)
def _create_docs(cls, domain, count):
case_ids = [uuid.uuid4().hex for i in range(count)]
[create_form_for_test(domain, case_id=case_id) for case_id in case_ids]
return CaseAccessorSQL.get_cases(case_ids, ordered=True)
def _create_docs(cls, domain, count):
return [create_form_for_test(domain) for i in range(count)]
def test_get_form_by_id(self):
form = create_form_for_test(DOMAIN)
with self.assertNumQueries(1, using=form.db):
form = FormAccessorSQL.get_form(form.form_id)
self._check_simple_form(form)
def _create_docs(cls, domain, count):
case_ids = [uuid.uuid4().hex for i in range(count)]
[create_form_for_test(domain, case_id=case_id) for case_id in case_ids]
return CommCareCase.objects.get_cases(case_ids, ordered=True)
def setUpClass(cls):
super().setUpClass()
cls.es = get_es_new()
initialize_index_and_mapping(cls.es, USER_INDEX_INFO)
initialize_index_and_mapping(cls.es, XFORM_INDEX_INFO)
cls.date_start, cls.date_end = get_start_and_end_dates_of_month(
2021, 11)
cls.domain = Domain.get_or_create_with_name('test-partner-analytics',
is_active=True)
# Data for Mobile Workers Tests
DomainUserHistory.objects.create(domain=cls.domain.name,
record_date=cls.date_end,
num_users=3)
DomainUserHistory.objects.create(domain=cls.domain.name,
record_date=cls.date_start -
datetime.timedelta(days=1),
num_users=6)
DomainUserHistory.objects.create(domain=cls.domain.name,
record_date=cls.date_end +
datetime.timedelta(days=1),
num_users=9)
# Data for Web Users tests
cls.users = [
WebUser.create(cls.domain.name,
'*****@*****.**',
'testpwd',
None,
None,
date=cls.date_start + datetime.timedelta(days=5)),
WebUser.create(cls.domain.name,
'*****@*****.**',
'testpwd',
None,
None,
date=cls.date_start - datetime.timedelta(days=1)),
WebUser.create(cls.domain.name,
'*****@*****.**',
'testpwd',
None,
None,
date=cls.date_end + datetime.timedelta(days=1)),
WebUser.create(cls.domain.name,
'*****@*****.**',
'testpwd',
None,
None,
date=cls.date_start + datetime.timedelta(days=6)),
]
for user in cls.users:
elastic_user = transform_user_for_elasticsearch(user.to_json())
send_to_elasticsearch('users', elastic_user)
invitations = [
Invitation.objects.create(
domain=cls.domain.name,
email='*****@*****.**',
invited_by='*****@*****.**',
invited_on=cls.date_start + datetime.timedelta(days=5),
),
Invitation.objects.create(
domain=cls.domain.name,
email='*****@*****.**',
invited_by='*****@*****.**',
invited_on=cls.date_start - datetime.timedelta(days=1),
),
Invitation.objects.create(
domain=cls.domain.name,
email='*****@*****.**',
invited_by='*****@*****.**',
invited_on=cls.date_end + datetime.timedelta(days=1),
),
]
for invitation in invitations:
invitation.is_accepted = True
invitation.save()
# Data for forms tests
forms = [
create_form_for_test(cls.domain.name,
received_on=cls.date_start -
datetime.timedelta(days=1)),
create_form_for_test(cls.domain.name,
received_on=cls.date_end +
datetime.timedelta(days=1)),
create_form_for_test(cls.domain.name,
received_on=cls.date_start +
datetime.timedelta(days=3)),
create_form_for_test(cls.domain.name,
received_on=cls.date_start +
datetime.timedelta(days=12)),
]
for form in forms:
elastic_form = transform_xform_for_elasticsearch(form.to_json())
send_to_elasticsearch('forms', elastic_form)
cls.es.indices.refresh(USER_INDEX_INFO.alias)
cls.es.indices.refresh(XFORM_INDEX_INFO.alias)
def test_get_form_by_id(self):
form = create_form_for_test(DOMAIN)
with self.assertNumQueries(1, using=db_for_read_write(XFormInstanceSQL)):
form = FormAccessorSQL.get_form(form.form_id)
self._check_simple_form(form)
def _create_form(domain, doc_type):
form = create_form_for_test(domain, state=doc_type_to_state[doc_type])
return form.form_id
