def test_local_domain_sql_backend_override(self): domain_name = uuid.uuid4().hex set_local_domain_sql_backend_override(domain_name) self.assertTrue(get_local_domain_sql_backend_override(domain_name)) clear_local_domain_sql_backend_override(domain_name) self.assertFalse(get_local_domain_sql_backend_override(domain_name))
def test_couch_with_missing_forms(self): form1 = make_test_form("form-1", age="33", date="2016-08-04T18:25:56.656Z") form2 = make_test_form("form-2", age="32", date="2015-08-04T18:25:56.656Z") self.submit_form(THING_FORM) self.submit_form(form1) self.submit_form(form2) case = self._get_case("test-case") self.assertEqual(case.age, "33") self.assertEqual(case.thing, "1") del case.thing case.actions = [a for a in case.actions if a.form_id != "thing-form"] case.save() with self.assertRaises(AttributeError): self._get_case("test-case").thing with self.diff_without_rebuild(): self._do_migration() self._compare_diffs([ ('CommCareCase', Diff('diff', ['age'], old='33', new='32')), ('CommCareCase', Diff('missing', ['thing'], old=MISSING, new='1')), ]) clear_local_domain_sql_backend_override(self.domain_name) self.do_case_diffs(cases="with-diffs") sql_case = self._get_case("test-case") self.assertEqual(sql_case.dynamic_case_properties()["age"], "33") self._compare_diffs([])
def test_reconcile_transaction_order(self): from ..rebuildcase import SortTransactionsRebuild form1 = make_test_form("form-1", age="33", date="2016-08-04T18:25:56.656Z") form2 = make_test_form("form-2", age="32", date="2015-08-04T18:25:56.656Z") self.submit_form(form1) self.submit_form(form2) self.assertEqual(self._get_case("test-case").age, "33") with self.diff_without_rebuild(): self.do_migration() self.compare_diffs([ Diff('test-case', 'diff', ['age'], old='33', new='32'), ]) clear_local_domain_sql_backend_override(self.domain_name) self.do_case_diffs(cases="with-diffs") sql_case = self._get_case("test-case") self.assertEqual(sql_case.dynamic_case_properties()["age"], "33") self.compare_diffs() details = sql_case.transactions[-1].details self.assertEqual(details["reason"], SortTransactionsRebuild._REASON) server_dates = details["original_server_dates"] self.assertEqual(len(server_dates), 1, server_dates)
def test_dry_run(self): self.assertFalse(should_use_sql_backend(self.domain_name)) call_command( 'migrate_domain_from_couch_to_sql', self.domain_name, MIGRATE=True, no_input=True, dry_run=True ) clear_local_domain_sql_backend_override(self.domain_name) with self.assertRaises(CommandError): call_command('migrate_domain_from_couch_to_sql', self.domain_name, COMMIT=True, no_input=True) self.assertFalse(Domain.get_by_name(self.domain_name).use_sql_backend) xml = """<?xml version="1.0" ?> <n0:registration xmlns:n0="http://openrosa.org/user/registration"> <username>W4</username> <password>2</password> <uuid>P8DU7OLHVLZXU21JR10H3W8J2</uuid> <date>2013-11-19</date> <registering_phone_id>8H1N48EFPF6PA4UOO8YGZ2KFZ</registering_phone_id> <user_data> <data key="user_type">standard</data> </user_data> </n0:registration> """ submit_form_locally(xml, self.domain_name) couch_form_ids = self._get_form_ids() self.assertEqual(1, len(couch_form_ids)) call_command('migrate_domain_from_couch_to_sql', self.domain_name, blow_away=True, no_input=True) self.assertFalse(Domain.get_by_name(self.domain_name).use_sql_backend)
def test_patch_case_with_non_ascii_name(self): self.submit_form(make_test_form("form1", case_name="ऊषा")) self.do_migration(case_diff="none") clear_local_domain_sql_backend_override(self.domain_name) with self.augmented_couch_case("test-case") as case: case.type = "old-type" case.save() self.do_case_diffs()
def test_diff_specific_case(self): self.submit_form(make_test_form("form-1", case_id="case-1")) self.do_migration(case_diff="none") clear_local_domain_sql_backend_override(self.domain_name) with self.augmented_couch_case("case-1") as case: case.age = '35' case.save() self.do_case_diffs(cases="case-1")
def commit_migration(domain_name): domain = Domain.get_by_name(domain_name, strict=True) domain.use_sql_backend = True domain.save() clear_local_domain_sql_backend_override(domain_name) if not should_use_sql_backend(domain_name): Domain.get_by_name.clear(Domain, domain_name) assert should_use_sql_backend(domain_name)
def test_commit(self): self._do_migration_and_assert_flags(self.domain_name) clear_local_domain_sql_backend_override(self.domain_name) call_command('migrate_domain_from_couch_to_sql', self.domain_name, COMMIT=True, no_input=True) self.assertTrue(Domain.get_by_name(self.domain_name).use_sql_backend)
def commit_migration(domain_name): domain = Domain.get_by_name(domain_name, strict=True) domain.use_sql_backend = True domain.save() clear_local_domain_sql_backend_override(domain_name) if not should_use_sql_backend(domain_name): Domain.get_by_name.clear(Domain, domain_name) assert should_use_sql_backend(domain_name) datadog_counter("commcare.couch_sql_migration.total_committed") _logger.info("committed migration for {}".format(domain))
def test_test_local_domain_sql_backend_override_overrides(self): domain_name = uuid.uuid4().hex create_domain(domain_name) self.assertFalse(should_use_sql_backend(domain_name)) set_local_domain_sql_backend_override(domain_name) self.assertTrue(should_use_sql_backend(domain_name)) clear_local_domain_sql_backend_override(domain_name) self.assertFalse(should_use_sql_backend(domain_name))
def test_patch_closed_case(self): from casexml.apps.case.cleanup import close_case self.submit_form(make_test_form("form-1", case_id="case-1")) close_case("case-1", self.domain_name, "system", "test") self.do_migration(case_diff="none") clear_local_domain_sql_backend_override(self.domain_name) with self.augmented_couch_case("case-1") as case: case.name = "Zena" case.save() self.do_case_diffs()
def commit_migration(domain_name): domain_obj = Domain.get_by_name(domain_name, strict=True) domain_obj.use_sql_backend = True domain_obj.save() clear_local_domain_sql_backend_override(domain_name) if not should_use_sql_backend(domain_name): Domain.get_by_name.clear(Domain, domain_name) assert should_use_sql_backend(domain_name) datadog_counter("commcare.couch_sql_migration.total_committed") _logger.info("committed migration for {}".format(domain_name))
def commit_migration(domain_name): domain_obj = Domain.get_by_name(domain_name, strict=True) domain_obj.use_sql_backend = True domain_obj.save() clear_local_domain_sql_backend_override(domain_name) if not should_use_sql_backend(domain_name): Domain.get_by_name.clear(Domain, domain_name) assert should_use_sql_backend(domain_name), \ "could not set use_sql_backend for domain %s (try again)" % domain_name datadog_counter("commcare.couch_sql_migration.total_committed") log.info("committed migration for {}".format(domain_name))
def test_pending_diff(self): def diff_none(case_ids, log_cases=None): return casediff.DiffData([]) self.submit_form(make_test_form("form-1", case_id="case-1")) self._do_migration(case_diff='none') clear_local_domain_sql_backend_override(self.domain_name) with self.augmented_couch_case("case-1") as case: case.age = '35' case.save() with patch("corehq.apps.couch_sql_migration.casedifftool.diff_cases", diff_none): result = self.do_case_diffs() self.assertEqual(result, mod.PENDING_WARNING) self.do_case_diffs(cases="pending")
def test_patch_known_properties(self): self.submit_form(make_test_form("form-1", case_id="case-1")) self.do_migration(case_diff="none") clear_local_domain_sql_backend_override(self.domain_name) open_date = datetime(2010, 9, 8) with self.augmented_couch_case("case-1") as case: case.name = "Zena" case.type = "old-type" case.user_id = "old-user" case.owner_id = "old-owner" case.opened_on = open_date case.save() self.do_case_diffs()
def test_diff_case_with_wrong_domain(self): wrong_domain = create_domain("wrong") self.addCleanup(wrong_domain.delete) self.submit_form(make_test_form("form-1"), domain="wrong") self.do_migration(case_diff="none", domain="wrong") self.do_migration(case_diff="none") clear_local_domain_sql_backend_override(self.domain_name) with capture_log_output("corehq.apps.couch_sql_migration") as log, \ self.augmented_couch_case("test-case") as case: # modify case so it would have a diff (if it were diffed) case.age = '35' case.save() # try to diff case in wrong domain self.do_case_diffs(cases="test-case")
def test_ledger_dup_transaction_diff(self): product_id = self.create_form_with_duplicate_stock_transaction() self._do_migration(case_diff='none') self._compare_diffs([], ignore_fail=True) clear_local_domain_sql_backend_override(self.domain_name) self.do_case_diffs() self._compare_diffs(changes=[Diff( f"test-case/things/{product_id}", reason="duplicate stock transaction", type="diff", path=["balance"], old=2, new=1, kind="stock state", )])
def test_convert_error_form_for_case_missing_in_couch(self): def find_forms(case_id): return ["form-1"] self.submit_form(make_test_form("form-1", case_id="case-1")) self.do_migration(case_diff="none") CommCareCase.get_db().delete_doc("case-1") clear_local_domain_sql_backend_override(self.domain_name) form = self._get_form("form-1") form.problem = "something went wrong" form.save() self.do_case_diffs("pending") self.compare_diffs([ Diff('case-1', 'missing', ['*'], old=MISSING, new='present'), ]) with patch.object(casediff, "find_form_ids_updating_case", find_forms): self.do_migration(forms="missing", diffs=[])
def test_ledger_dup_transaction_diff(self): product_id = self.create_form_with_duplicate_stock_transaction() self._do_migration(case_diff='none') self._compare_diffs([], ignore_fail=True) clear_local_domain_sql_backend_override(self.domain_name) self.do_case_diffs() self._compare_diffs([]) db = open_state_db(self.domain_name, self.state_dir) self.assertEqual( list(db.iter_changes()), [Change( kind="stock state", doc_id=f"test-case/things/{product_id}", reason="duplicate stock transaction", diff_type="diff", path=["balance"], old_value=2, new_value=1, )], )
def test_patch_skips_deleted_case(self): self.submit_form(make_test_form("form-1", case_id="del")) self.submit_form(make_test_form("form-2", case_id="mar")) case = self._get_case("del") case.name = "Del" case.opened_by = "someone" case.save() case = self._get_case("mar") case.name = "Mar" case.opened_by = "someone" case.save() CaseAccessors(self.domain.name).soft_delete_cases(["del"], datetime.utcnow()) self.do_migration(diffs=[ Diff('del', 'diff', ['name'], old='Del', kind="CommCareCase-Deleted"), Diff('del', 'diff', ['opened_by'], old='someone', kind="CommCareCase-Deleted"), Diff('mar', 'diff', ['name'], old='Mar'), Diff('mar', 'diff', ['opened_by'], old='someone'), ]) clear_local_domain_sql_backend_override(self.domain_name) self.do_case_patch(cases="with-diffs") self.compare_diffs(diffs=[ Diff('del', 'diff', ['name'], old='Del', kind="CommCareCase-Deleted"), Diff('del', 'diff', ['opened_by'], old='someone', kind="CommCareCase-Deleted"), ])
def abort(self, domain, state_dir): set_couch_sql_migration_not_started(domain) clear_local_domain_sql_backend_override(domain) blow_away_migration(domain, state_dir)
def commit_migration(domain_name): domain = Domain.get_by_name(domain_name) domain.use_sql_backend = True domain.save() clear_local_domain_sql_backend_override(domain_name) assert should_use_sql_backend(domain_name)
def abort(self, domain): set_couch_sql_migration_not_started(domain) clear_local_domain_sql_backend_override(domain) blow_away_migration(domain)
def _sigterm_handler(signal, frame): _logger.error("{} signal received".format(signal)) set_couch_sql_migration_not_started(domain) clear_local_domain_sql_backend_override(domain) _blow_away_migration(domain) sys.exit(1)