def test_get_forms_received_since(self):
# since this test depends on the global form list just wipe everything
FormProcessorTestUtils.delete_all_sql_forms()
form1 = create_form_for_test(DOMAIN)
form2 = create_form_for_test(DOMAIN)
middle = datetime.utcnow()
time.sleep(.01)
form3 = create_form_for_test(DOMAIN)
form4 = create_form_for_test(DOMAIN)
time.sleep(.01)
end = datetime.utcnow()
forms_back = list(FormAccessorSQL.get_all_forms_received_since())
self.assertEqual(4, len(forms_back))
self.assertEqual(set(form.form_id for form in forms_back),
set([form1.form_id, form2.form_id, form3.form_id, form4.form_id]))
forms_back = list(FormAccessorSQL.get_all_forms_received_since(middle))
self.assertEqual(2, len(forms_back))
self.assertEqual(set(form.form_id for form in forms_back),
set([form3.form_id, form4.form_id]))
self.assertEqual(0, len(list(FormAccessorSQL.get_all_forms_received_since(end))))
self.assertEqual(1, len(list(FormAccessorSQL.get_forms_received_since(limit=1))))
def test_unknown_user_pillow(self):
FormProcessorTestUtils.delete_all_xforms()
user_id = 'test-unknown-user'
metadata = TestFormMetadata(domain=TEST_DOMAIN, user_id='test-unknown-user')
form = get_form_ready_to_save(metadata)
FormProcessorInterface(domain=TEST_DOMAIN).save_processed_models([form])
# send to kafka
topic = topics.FORM_SQL if settings.TESTS_SHOULD_USE_SQL_BACKEND else topics.FORM
since = self._get_kafka_seq()
producer.send_change(topic, _form_to_change_meta(form))
# send to elasticsearch
pillow = get_xform_pillow()
pillow.process_changes(since=since, forever=False)
self.elasticsearch.indices.refresh(self.index_info.index)
# the default query doesn't include unknown users so should have no results
self.assertEqual(0, UserES().run().total)
# clear the default filters which hide unknown users
user_es = UserES().remove_default_filters()
results = user_es.run()
self.assertEqual(1, results.total)
user_doc = results.hits[0]
self.assertEqual(TEST_DOMAIN, user_doc['domain'])
self.assertEqual(user_id, user_doc['_id'])
self.assertEqual('UnknownUser', user_doc['doc_type'])
def setUp(self):
super(XFormPillowTest, self).setUp()
FormProcessorTestUtils.delete_all_xforms()
with trap_extra_setup(ConnectionError):
self.elasticsearch = get_es_new()
initialize_index_and_mapping(self.elasticsearch, XFORM_INDEX_INFO)
delete_es_index(XFORM_INDEX_INFO.index)
def setUp(self):
FormProcessorTestUtils.delete_all_cases()
FormProcessorTestUtils.delete_all_xforms()
self.domain = "domain"
self.project = Domain(name=self.domain)
self.user = User(user_id='user', username='******', password="******",
date_joined=datetime(2011, 6, 9))
self.factory = CaseFactory()
self.extension_ids = ['1', '2', '3']
self.host = CaseStructure(case_id='host')
self.extension = CaseStructure(
case_id=self.extension_ids[0],
indices=[CaseIndex(
related_structure=self.host,
relationship="extension",
)],
)
self.extension_2 = CaseStructure(
case_id=self.extension_ids[1],
indices=[CaseIndex(
related_structure=self.extension,
relationship="extension",
)],
)
self.extension_3 = CaseStructure(
case_id=self.extension_ids[2],
indices=[CaseIndex(
related_structure=self.extension_2,
relationship="extension",
)],
)
def setUp(self):
super(ReportCaseReindexerTest, self).setUp()
FormProcessorTestUtils.delete_all_xforms()
FormProcessorTestUtils.delete_all_cases()
with trap_extra_setup(ConnectionError):
self.elasticsearch = get_es_new()
ensure_index_deleted(REPORT_CASE_INDEX_INFO.index)
def setUp(self):
super(AppPillowTest, self).setUp()
FormProcessorTestUtils.delete_all_cases()
with trap_extra_setup(ConnectionError):
self.es = get_es_new()
ensure_index_deleted(APP_INDEX_INFO.index)
initialize_index_and_mapping(self.es, APP_INDEX_INFO)
def tearDownClass(cls):
cls.backend_mapping.delete()
cls.backend.delete()
FormProcessorTestUtils.delete_all_cases(cls.domain)
cls.teardown_subscription()
cls.domain_obj.delete()
clear_plan_version_cache()
super(OptTestCase, cls).tearDownClass()
def tearDown(self):
FormProcessorTestUtils.delete_all_cases(self.domain)
PhoneBlacklist.objects.all().delete()
delete_domain_phone_numbers(self.domain)
self.backend_mapping.delete()
self.backend.delete()
self.domain_obj.delete()
self.teardown_subscription()
def setUp(self):
FormProcessorTestUtils.delete_all_cases(self.domain)
FormProcessorTestUtils.delete_all_ledgers(self.domain)
with trap_extra_setup(ConnectionError):
self.pillow = get_ledger_to_elasticsearch_pillow()
self.elasticsearch = get_es_new()
ensure_index_deleted(LEDGER_INDEX_INFO.index)
initialize_index_and_mapping(get_es_new(), LEDGER_INDEX_INFO)
def setUp(self):
super(FormPillowTest, self).setUp()
FormProcessorTestUtils.delete_all_xforms()
self.pillow = get_app_form_submission_tracker_pillow()
factory = AppFactory(domain=self.domain)
self.app = factory.app
self.app.save()
def test_xform_reindexer(self):
FormProcessorTestUtils.delete_all_xforms()
form = _create_and_save_a_form()
call_command('ptop_fast_reindex_xforms', noinput=True, bulk=True)
self._assert_form_is_in_es(form)
form.delete()
def setUp(self):
FormProcessorTestUtils.delete_all_cases()
self.elasticsearch = get_es_new()
self.pillow = get_case_search_to_elasticsearch_pillow()
ensure_index_deleted(CASE_SEARCH_INDEX)
# Bootstrap ES
initialize_index_and_mapping(get_es_new(), CASE_SEARCH_INDEX_INFO)
def setUpClass(cls):
super(LedgerDBAccessorTest, cls).setUpClass()
cls.domain = uuid.uuid4().hex
with override_settings(TESTS_SHOULD_USE_SQL_BACKEND=True):
FormProcessorTestUtils.delete_all_cases_forms_ledgers(cls.domain)
cls.product_a = make_product(cls.domain, 'A Product', 'prodcode_a')
cls.product_b = make_product(cls.domain, 'B Product', 'prodcode_b')
cls.product_c = make_product(cls.domain, 'C Product', 'prodcode_c')
def test_case_reindexer_v2(self):
FormProcessorTestUtils.delete_all_cases()
case = _create_and_save_a_case()
index_id = 'sql-case' if settings.TESTS_SHOULD_USE_SQL_BACKEND else 'case'
call_command('ptop_reindexer_v2', index_id, cleanup=True, noinput=True, reset=True)
self._assert_case_is_in_es(case)
def test_xform_reindexer_v2(self):
FormProcessorTestUtils.delete_all_xforms()
form = create_and_save_a_form(DOMAIN)
index_id = 'sql-form' if settings.TESTS_SHOULD_USE_SQL_BACKEND else 'form'
call_command('ptop_reindexer_v2', index_id, cleanup=True, noinput=True, reset=True)
self._assert_form_is_in_es(form)
def setUp(self):
super(FormPillowTest, self).setUp()
FormProcessorTestUtils.delete_all_xforms()
self.pillow = get_xform_pillow(skip_ucr=True)
factory = AppFactory(domain=self.domain)
self.app = factory.app
self.app.save()
def tearDownClass(cls):
cls.product_a.delete()
cls.product_b.delete()
cls.product_c.delete()
FormProcessorTestUtils.delete_all_cases_forms_ledgers(DOMAIN)
with override_settings(TESTS_SHOULD_USE_SQL_BACKEND=True):
FormProcessorTestUtils.delete_all_cases_forms_ledgers(DOMAIN)
super(LedgerTests, cls).tearDownClass()
def setUp(self):
self.domain = create_domain("submit-errors")
self.couch_user = WebUser.create(None, "test", "foobar")
self.couch_user.add_domain_membership(self.domain.name, is_admin=True)
self.couch_user.save()
self.client = Client()
self.client.login(**{'username': '******', 'password': '******'})
self.url = reverse("receiver_post", args=[self.domain])
FormProcessorTestUtils.delete_all_xforms(self.domain.name)
def setUp(self):
super(BaseMigrationTestCase, self).setUp()
FormProcessorTestUtils.delete_all_cases_forms_ledgers()
self.domain_name = uuid.uuid4().hex
self.domain = create_domain(self.domain_name)
# all new domains are set complete when they are created
DomainMigrationProgress.objects.filter(domain=self.domain_name).delete()
self.assertFalse(should_use_sql_backend(self.domain_name))
def setUp(self):
super(CaseSearchPillowTest, self).setUp()
FormProcessorTestUtils.delete_all_cases()
self.elasticsearch = get_es_new()
self.pillow = get_case_pillow(skip_ucr=True)
ensure_index_deleted(CASE_SEARCH_INDEX)
# Bootstrap ES
initialize_index_and_mapping(get_es_new(), CASE_SEARCH_INDEX_INFO)
def setUp(self):
FormProcessorTestUtils.delete_all_cases()
FormProcessorTestUtils.delete_all_xforms()
delete_all_users()
self.domain = "domain"
self.project = Domain(name=self.domain)
self.user = create_restore_user(self.domain, username='******', password="******")
self.factory = CaseFactory(domain=self.domain)
self.extension_ids = ['1', '2', '3']
self.host_id = 'host'
def test_new_case_reindexer(self):
FormProcessorTestUtils.delete_all_cases()
case = _create_and_save_a_case()
ensure_index_deleted(CASE_INDEX) # new reindexer doesn't force delete the index so do it in the test
index_id = 'sql-case' if settings.TESTS_SHOULD_USE_SQL_BACKEND else 'case'
call_command('ptop_reindexer_v2', index_id)
CasePillow().get_es_new().indices.refresh(CASE_INDEX) # as well as refresh the index
self._assert_case_is_in_es(case)
def test_new_xform_reindexer(self):
FormProcessorTestUtils.delete_all_xforms()
form = _create_and_save_a_form()
ensure_index_deleted(XFORM_INDEX)
index_id = 'sql-form' if settings.TESTS_SHOULD_USE_SQL_BACKEND else 'form'
call_command('ptop_reindexer_v2', index_id)
XFormPillow().get_es_new().indices.refresh(XFORM_INDEX)
self._assert_form_is_in_es(form)
def test_sync_log_invalidation_bug(self):
sync_log = FormProcessorInterface().sync_log_model(
user_id='6dac4940-913e-11e0-9d4b-005056aa7fb5'
)
sync_log.save()
_, _, [case] = self._doCreateCaseWithMultimedia()
# this used to fail before we fixed http://manage.dimagi.com/default.asp?158373
self._doSubmitUpdateWithMultimedia(new_attachments=['commcare_logo_file'], removes=[],
sync_token=sync_log._id)
FormProcessorTestUtils.delete_all_sync_logs()
def test_case_reindexer(self):
FormProcessorTestUtils.delete_all_cases()
case_name = 'reindexer-test-case-{}'.format(uuid.uuid4().hex)
with drop_connected_signals(case_post_save):
CommCareCase(domain=self.domain, name=case_name).save()
call_command('ptop_fast_reindex_cases', noinput=True, bulk=True)
results = CaseES().run()
self.assertEqual(1, results.total)
case_doc = results.hits[0]
self.assertEqual(self.domain, case_doc['domain'])
self.assertEqual(case_name, case_doc['name'])
self.assertEqual('CommCareCase', case_doc['doc_type'])
def test_xform_reindexers(self):
FormProcessorTestUtils.delete_all_xforms()
metadata = TestFormMetadata(domain=self.domain)
form = get_form_ready_to_save(metadata)
FormProcessorInterface(domain=self.domain).save_processed_models([form])
call_command('ptop_fast_reindex_xforms', noinput=True, bulk=True)
results = FormES().run()
self.assertEqual(1, results.total)
form_doc = results.hits[0]
self.assertEqual(self.domain, form_doc['domain'])
self.assertEqual(metadata.xmlns, form_doc['xmlns'])
self.assertEqual('XFormInstance', form_doc['doc_type'])
form.delete()
def setUp(self):
super(BaseMigrationTestCase, self).setUp()
with trap_extra_setup(AttributeError, msg="S3_BLOB_DB_SETTINGS not configured"):
config = settings.S3_BLOB_DB_SETTINGS
self.s3db = TemporaryS3BlobDB(config)
assert get_blob_db() is self.s3db, (get_blob_db(), self.s3db)
FormProcessorTestUtils.delete_all_cases_forms_ledgers()
self.domain_name = uuid.uuid4().hex
self.domain = create_domain(self.domain_name)
# all new domains are set complete when they are created
DomainMigrationProgress.objects.filter(domain=self.domain_name).delete()
self.assertFalse(should_use_sql_backend(self.domain_name))
def tearDownClass(cls):
for user in cls.user_records:
user.delete()
for domain in cls.domain_records:
domain.delete()
FormProcessorTestUtils.delete_all_sql_forms(cls.domain)
FormStagingTable.clear_records()
FormFact.clear_records()
DomainStagingTable.clear_records()
DomainDim.clear_records()
UserStagingTable.clear_records()
UserDim.clear_records()
super(FormFactIntegrationTest, cls).tearDownClass()
def test_case_search_reindexer(self):
es = get_es_new()
FormProcessorTestUtils.delete_all_cases()
case = _create_and_save_a_case()
ensure_index_deleted(CASE_SEARCH_INDEX)
# With case search not enabled, case should not make it to ES
CaseSearchConfig.objects.all().delete()
call_command('ptop_reindexer_v2', 'case-search')
es.indices.refresh(CASE_SEARCH_INDEX) # as well as refresh the index
self._assert_es_empty(esquery=CaseSearchES())
# With case search enabled, it should get indexed
CaseSearchConfig.objects.create(domain=self.domain, enabled=True)
self.addCleanup(CaseSearchConfig.objects.all().delete)
call_command('ptop_reindexer_v2', 'case-search')
es.indices.refresh(CASE_SEARCH_INDEX) # as well as refresh the index
self._assert_case_is_in_es(case, esquery=CaseSearchES())
def test_unknown_user_reindexer(self):
FormProcessorTestUtils.delete_all_xforms()
user_id = 'test-unknown-user'
metadata = TestFormMetadata(domain=self.domain, user_id='test-unknown-user')
form = get_form_ready_to_save(metadata)
FormProcessorInterface(domain=self.domain).save_processed_models([form])
ensure_index_deleted(USER_INDEX)
call_command('ptop_fast_reindex_unknownusers', noinput=True, bulk=True)
# the default query doesn't include unknown users so should have no results
self.assertEqual(0, UserES().run().total)
user_es = UserES()
# hack: clear the default filters which hide unknown users
# todo: find a better way to do this.
user_es._default_filters = ESQuery.default_filters
results = user_es.run()
self.assertEqual(1, results.total)
user_doc = results.hits[0]
self.assertEqual(self.domain, user_doc['domain'])
self.assertEqual(user_id, user_doc['_id'])
self.assertEqual('UnknownUser', user_doc['doc_type'])
form.delete()
delete_es_index(USER_INDEX)
def setUpClass(cls):
super(SqlUpdateStrategyTest, cls).setUpClass()
FormProcessorTestUtils.delete_all_sql_forms()
FormProcessorTestUtils.delete_all_sql_cases()
def tearDown(self):
FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain)
super(ReprocessSubmissionStubTests, self).tearDown()
def tearDownClass(cls):
FormProcessorTestUtils.delete_all_sql_forms()
FormProcessorTestUtils.delete_all_sql_cases()
super(SqlUpdateStrategyTest, cls).tearDownClass()
def delete_all_ledgers():
FormProcessorTestUtils.delete_all_ledgers()
def tearDown(self):
FormProcessorTestUtils.delete_all_xforms(DOMAIN)
super(FormAccessorsTests, self).tearDown()
def tearDownClass(cls):
FormProcessorTestUtils.delete_all_cases_forms_ledgers(cls.domain_name)
super(TestSQLDumpLoadShardedModels, cls).tearDownClass()
def delete_all_cases():
FormProcessorTestUtils.delete_all_cases()
def tearDown(self):
FormProcessorTestUtils.delete_all_xforms(self.domain.name)
FormProcessorTestUtils.delete_all_cases(self.domain.name)
self.couch_user.delete()
self.domain.delete()
super(SubmissionTest, self).tearDown()
def delete_all_xforms():
FormProcessorTestUtils.delete_all_xforms()
def tearDown(self):
FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain)
super().tearDown()
def delete_all_sync_logs():
FormProcessorTestUtils.delete_all_sync_logs()
def tearDown(self):
FormProcessorTestUtils.delete_all_xforms(DOMAIN)
FormProcessorTestUtils.delete_all_cases(DOMAIN)
def tearDownClass(cls):
cls.product_a.delete()
FormProcessorTestUtils.delete_all_cases_forms_ledgers(DOMAIN)
with override_settings(TESTS_SHOULD_USE_SQL_BACKEND=True):
FormProcessorTestUtils.delete_all_cases_forms_ledgers(DOMAIN)
super(TestLedgerDocumentStore, cls).tearDownClass()
def tearDown(self):
FormProcessorTestUtils.delete_all_sql_forms(DOMAIN)
FormProcessorTestUtils.delete_all_sql_cases(DOMAIN)
super(ShardingTests, self).tearDown()
def tearDownClass(cls):
FormProcessorTestUtils.delete_all_cases_forms_ledgers(cls.domain)
ensure_index_deleted(cls.index)
super().tearDownClass()
def tearDownClass(cls):
FormProcessorTestUtils.delete_all_cases_forms_ledgers(cls.domain)
super(ReprocessXFormErrorsTest, cls).tearDownClass()
def setUp(self):
super(BaseCaseMultimediaTest, self).setUp()
self.formdb = FormAccessors()
FormProcessorTestUtils.delete_all_cases()
FormProcessorTestUtils.delete_all_xforms()
def tearDownClass(cls):
FormProcessorTestUtils.delete_all_xforms(cls.domain)
super(TestDBAccessors, cls).tearDownClass()
def tearDownClass(cls):
FormProcessorTestUtils.delete_all_cases_forms_ledgers()
super(CaseBugTest, cls).tearDownClass()
def tearDown(self):
FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain)
super(TestReprocessDuringSubmission, self).tearDown()
def tearDown(self):
FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain_name)
delete_all_repeat_records()
super(FormPayloadGeneratorTest, self).tearDown()
def tearDown(self):
CommCareUser.get_db().delete_doc(self.user._id)
FormProcessorTestUtils.delete_all_xforms(self.domain)
self.domain_obj.delete()
super(UserModelTest, self).tearDown()
def tearDown(self):
FormProcessorTestUtils.delete_all_cases(self.domain_name)
delete_all_repeat_records()
super(CaseRepeaterTest, self).tearDown()
def tearDown(self):
self.repeater.delete()
FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain)
delete_all_repeat_records()
super(TestRepeaterFormat, self).tearDown()
def tearDown(self):
ensure_index_deleted(CASE_INDEX_INFO.alias)
ensure_index_deleted(DOMAIN_INDEX_INFO.alias)
FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain)
super(ESAccessorsTest, self).tearDown()
def tearDownClass(cls):
cls.user.delete()
cls.domain_object.delete()
FormProcessorTestUtils.delete_all_cases()
super(TestZapierCreateCaseAction, cls).tearDownClass()
def setUp(self):
FormProcessorTestUtils.delete_all_xforms(self.domain.name)
def tearDown(self):
FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain.name)
UnfinishedSubmissionStub.objects.all().delete()
def tearDown(self):
FormProcessorTestUtils.delete_all_sql_forms(DOMAIN)
FormProcessorTestUtils.delete_all_sql_cases(DOMAIN)
super(CaseAccessorTestsSQL, self).tearDown()
def tearDown(self):
FormProcessorTestUtils.delete_all_xforms(self.domain)
FormProcessorTestUtils.delete_all_ledgers(self.domain)
FormProcessorTestUtils.delete_all_cases(self.domain)
super(SubmissionSQLTransactionsTest, self).tearDown()
