def test_case_indices_order(self):
couch_case = {
'doc_type': 'CommCareCase',
'indices': [
{
"case_id": "fb698d47-4832-42b2-b28c-86d13adb45a2",
"identifier": "parent",
"referenced_id": "7ab03ccc-e5b7-4c8f-b88f-43ee3b0543a5",
"referenced_type": "Patient",
"relationship": "child"
},
{
"case_id": "fb698d47-4832-42b2-b28c-86d13adb45a2",
"identifier": "goal",
"referenced_id": "c2e938d9-7406-4fdf-87ab-67d92296705e",
"referenced_type": "careplan_goal",
"relationship": "child"
}
]
}
sql_case = {
'doc_type': 'CommCareCase',
'indices': list(reversed(couch_case['indices']))
}
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
self.assertEqual(6, len(diffs))
filtered_diffs = filter_case_diffs(couch_case, sql_case, diffs)
self.assertEqual([], filtered_diffs)
def test_single_case_indices_real_diff(self):
couch_case = {
'doc_type': 'CommCareCase',
'indices': [
{
"doc_type": "CommCareCaseIndex",
"case_id": "fb698d47-4832-42b2-b28c-86d13adb45a2",
"identifier": "parent",
"referenced_id": "7ab03ccc-e5b7-4c8f-b88f-43ee3b0543a5",
"referenced_type": "Patient",
"relationship": "child"
}
]
}
sql_case = {
'doc_type': 'CommCareCase',
'indices': deepcopy(couch_case['indices'])
}
del sql_case['indices'][0]['doc_type']
sql_case['indices'][0]['relationship'] = 'extension'
expected_diffs = [
FormJsonDiff(diff_type='diff', path=('indices', '[*]', 'relationship'), old_value=u'child', new_value=u'extension')
]
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
filtered_diffs = filter_case_diffs(couch_case, sql_case, diffs)
self.assertEqual(expected_diffs, filtered_diffs)
def test_multiple_case_indices_real_diff(self):
couch_case = {
'doc_type': 'CommCareCase',
'indices': [
{
"case_id": "fb698d47-4832-42b2-b28c-86d13adb45a2",
"identifier": "parent",
"referenced_id": "7ab03ccc-e5b7-4c8f-b88f-43ee3b0543a5",
"referenced_type": "Patient",
"relationship": "child"
},
{
"case_id": "fb698d47-4832-42b2-b28c-86d13adb45a2",
"identifier": "goal",
"referenced_id": "c2e938d9-7406-4fdf-87ab-67d92296705e",
"referenced_type": "careplan_goal",
"relationship": "child"
}
]
}
sql_case = {
'doc_type': 'CommCareCase',
'indices': deepcopy(couch_case['indices'])
}
sql_case['indices'][0]['identifier'] = 'mother'
expected_diffs = [
FormJsonDiff(
diff_type='diff', path=('indices', '[*]', 'identifier'),
old_value=u'parent', new_value=u'mother')
]
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
filtered_diffs = filter_case_diffs(couch_case, sql_case, diffs)
self.assertEqual(expected_diffs, filtered_diffs)
def _diff_cases(self, couch_cases):
from corehq.apps.tzmigration.timezonemigration import json_diff
self.log_debug('Calculating case diffs for {} cases'.format(len(couch_cases)))
case_ids = list(couch_cases)
sql_cases = CaseAccessorSQL.get_cases(case_ids)
for sql_case in sql_cases:
couch_case = couch_cases[sql_case.case_id]
sql_case_json = sql_case.to_json()
diffs = json_diff(couch_case, sql_case_json, track_list_indices=False)
diffs = filter_case_diffs(
couch_case, sql_case_json, diffs, self.forms_that_touch_cases_without_actions
)
if diffs and not sql_case.is_deleted:
couch_case, diffs = self._rebuild_couch_case_and_re_diff(couch_case, sql_case_json)
if diffs:
self.diff_db.add_diffs(
couch_case['doc_type'], sql_case.case_id,
diffs
)
self._diff_ledgers(case_ids)
self.processed_docs += len(case_ids)
self._log_case_diff_count(throttled=True)
def test_filter_case_deletion_fields(self):
couch_case = {
'doc_type': 'CommCareCase-Deleted',
'-deletion_id': 'abc',
'-deletion_date': '123',
}
sql_case = {
'doc_type': 'CommCareCase-Deleted',
'deletion_id': 'abc',
'deleted_on': '123',
}
filtered = filter_case_diffs(couch_case, sql_case, DELETION_DIFFS + REAL_DIFFS)
self.assertEqual(filtered, REAL_DIFFS)
def test_filter_modified_on(self):
couch_case = {
'doc_type': 'CommCareCase',
'modified_on': '2015-03-23T14:36:53Z'
}
sql_case = {
'doc_type': 'CommCareCase',
'modified_on': '2015-03-23T14:36:53.073000Z'
}
date_diffs = json_diff(couch_case, sql_case)
self.assertEqual(1, len(date_diffs))
filtered = filter_case_diffs(couch_case, sql_case, date_diffs)
self.assertEqual(filtered, [])
def test_filter_modified_on(self):
couch_case = {
'doc_type': 'CommCareCase',
'modified_on': '2015-03-23T14:36:53Z'
}
sql_case = {
'doc_type': 'CommCareCase',
'modified_on': '2015-03-23T14:36:53.073000Z'
}
date_diffs = json_diff(couch_case, sql_case)
self.assertEqual(1, len(date_diffs))
filtered = filter_case_diffs(couch_case, sql_case, date_diffs)
self.assertEqual(filtered, [])
def test_filter_combo_fields(self):
couch_case = {
'doc_type': 'CommCareCase',
'@date_modified': '2015-03-23T14:36:53Z'
}
sql_case = {
'doc_type': 'CommCareCase',
'modified_on': '2015-03-23T14:36:53.073000Z'
}
rename_date_diffs = json_diff(couch_case, sql_case)
self.assertEqual(2, len(rename_date_diffs))
filtered = filter_case_diffs(couch_case, sql_case, rename_date_diffs)
self.assertEqual(filtered, [])
def _rebuild_couch_case_and_re_diff(self, couch_case, sql_case_json):
from corehq.form_processor.backends.couch.processor import FormProcessorCouch
from corehq.apps.tzmigration.timezonemigration import json_diff
rebuilt_case = FormProcessorCouch.hard_rebuild_case(
self.domain, couch_case['_id'], None, save=False, lock=False
)
rebuilt_case_json = rebuilt_case.to_json()
diffs = json_diff(rebuilt_case_json, sql_case_json, track_list_indices=False)
diffs = filter_case_diffs(
rebuilt_case_json, sql_case_json, diffs, self.forms_that_touch_cases_without_actions
)
return rebuilt_case_json, diffs
def test_filter_combo_fields(self):
couch_case = {
'doc_type': 'CommCareCase',
'@date_modified': '2015-03-23T14:36:53Z'
}
sql_case = {
'doc_type': 'CommCareCase',
'modified_on': '2015-03-23T14:36:53.073000Z'
}
rename_date_diffs = json_diff(couch_case, sql_case)
self.assertEqual(2, len(rename_date_diffs))
filtered = filter_case_diffs(couch_case, sql_case, rename_date_diffs)
self.assertEqual(filtered, [])
def test_filter_twice_deleted_case(self):
couch_case = {
'doc_type': 'CommCareCase-Deleted-Deleted',
'-deletion_id': 'abc',
'-deletion_date': '123',
}
sql_case = {
'doc_type': 'CommCareCase-Deleted',
'deletion_id': 'abc',
'deleted_on': '123',
}
filtered = filter_case_diffs(couch_case, sql_case, DELETION_DIFFS + REAL_DIFFS)
self.assertEqual(filtered, REAL_DIFFS)
def _rebuild_couch_case_and_re_diff(self, couch_case, sql_case_json):
from corehq.form_processor.backends.couch.processor import FormProcessorCouch
from corehq.apps.tzmigration.timezonemigration import json_diff
rebuilt_case = FormProcessorCouch.hard_rebuild_case(
self.domain, couch_case['_id'], None, save=False, lock=False
)
rebuilt_case_json = rebuilt_case.to_json()
diffs = json_diff(rebuilt_case_json, sql_case_json, track_list_indices=False)
diffs = filter_case_diffs(
rebuilt_case_json, sql_case_json, diffs, self.forms_that_touch_cases_without_actions
)
return rebuilt_case_json, diffs
def test_filter_case_xform_id_diffs_good(self):
couch_case = {
'doc_type': 'CommCareCase',
'xform_ids': ['123', '456']
}
sql_case = {
'doc_type': 'CommCareCase',
'xform_ids': ['456', '123']
}
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
self.assertEqual(2, len(diffs))
filtered = filter_case_diffs(couch_case, sql_case, diffs + REAL_DIFFS)
self.assertEqual(filtered, REAL_DIFFS)
def test_case_with_location_and_referrals(self):
couch_case = {
"doc_type": "CommCareCase",
"location_": [],
"referrals": [],
}
sql_case = {
"doc_type": "CommCareCase",
"location_": '[]',
"referrals": '[]',
}
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
filtered = filter_case_diffs(couch_case, sql_case, diffs)
self.assertEqual(filtered, [])
def _diff_cases(self, couch_cases):
from corehq.apps.tzmigration.timezonemigration import json_diff
self.log_debug('Calculating case diffs for {} cases'.format(len(couch_cases)))
case_ids = list(couch_cases)
sql_cases = CaseAccessorSQL.get_cases(case_ids)
for sql_case in sql_cases:
couch_case = couch_cases[sql_case.case_id]
sql_case_json = sql_case.to_json()
diffs = json_diff(couch_case, sql_case_json, track_list_indices=False)
self.diff_db.add_diffs(
couch_case['doc_type'], sql_case.case_id,
filter_case_diffs(couch_case, sql_case_json, diffs, self.forms_that_touch_cases_without_actions)
)
self._diff_ledgers(case_ids)
def _diff_cases(self, couch_cases):
from corehq.apps.tzmigration.timezonemigration import json_diff
self.log_debug('Calculating case diffs for {} cases'.format(len(couch_cases)))
case_ids = list(couch_cases)
sql_cases = CaseAccessorSQL.get_cases(case_ids)
for sql_case in sql_cases:
couch_case = couch_cases[sql_case.case_id]
sql_case_json = sql_case.to_json()
diffs = json_diff(couch_case, sql_case_json, track_list_indices=False)
self.diff_db.add_diffs(
couch_case['doc_type'], sql_case.case_id,
filter_case_diffs(couch_case, sql_case_json, diffs, self.forms_that_touch_cases_without_actions)
)
self._diff_ledgers(case_ids)
def test_filter_combo_fields(self):
couch_case = {'doc_type': 'CommCareCase'}
rename_date_diffs = [
FormJsonDiff(diff_type='missing',
path=('@date_modified', ),
old_value='2015-03-23T14:36:53Z',
new_value=Ellipsis),
FormJsonDiff(diff_type='missing',
path=('modified_on', ),
old_value=Ellipsis,
new_value='2015-03-23T14:36:53.073000Z'),
]
diffs = rename_date_diffs + REAL_DIFFS
filtered = filter_case_diffs(couch_case, {}, diffs)
self.assertEqual(filtered, REAL_DIFFS)
def test_filter_case_xform_id_diffs_good(self):
couch_case = {'doc_type': 'CommCareCase', 'xform_ids': ['123', '456']}
sql_case = {'doc_type': 'CommCareCase', 'xform_ids': ['456', '123']}
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
self.assertEqual(2, len(diffs))
filtered = filter_case_diffs(couch_case, sql_case, diffs + REAL_DIFFS)
self.assertEqual(
set(filtered),
set([
FormJsonDiff(diff_type='list_order',
path=('xform_ids', '[*]'),
old_value=None,
new_value=None)
] + REAL_DIFFS))
def test_filter_case_user_id(self):
couch_case = {
'doc_type': 'CommCareCase',
'user_id': u'e7ad965c70802884a7a67add763939e8',
'@user_id': u'e7ad965c70802884a7a67add763939e8',
'@case_id': u'5ac45838-da5b-49f5-b236-0675ff924e9f'
}
sql_case = {
'doc_type': 'CommCareCase',
'user_id': u'e7ad965c70802884a7a67add763939e8',
'case_id': u'5ac45838-da5b-49f5-b236-0675ff924e9f'
}
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
filtered = filter_case_diffs(couch_case, sql_case, diffs)
self.assertEqual(filtered, [])
def test_form_with_opened_by_diff(self):
couch_case = {
"doc_type": "XFormInstance",
"opened_by": "somebody",
"actions": [
{"action_type": "close"},
{"action_type": "rebuild"},
],
}
sql_case = {
"doc_type": "XFormInstance",
"opened_by": "somebody else",
}
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
filtered = filter_case_diffs(couch_case, sql_case, diffs)
self.assertEqual(filtered, [])
def test_filter_case_xform_id_diffs_good(self):
couch_case = {
'doc_type': 'CommCareCase',
'xform_ids': ['123', '456']
}
sql_case = {
'doc_type': 'CommCareCase',
'xform_ids': ['456', '123']
}
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
self.assertEqual(2, len(diffs))
filtered = filter_case_diffs(couch_case, sql_case, diffs + REAL_DIFFS)
self.assertEqual(set(filtered), set([
FormJsonDiff(diff_type='list_order', path=('xform_ids', '[*]'), old_value=None, new_value=None)
] + REAL_DIFFS))
def test_filter_case_user_id(self):
couch_case = {
'doc_type': 'CommCareCase',
'user_id': 'e7ad965c70802884a7a67add763939e8',
'@user_id': 'e7ad965c70802884a7a67add763939e8',
'@case_id': '5ac45838-da5b-49f5-b236-0675ff924e9f'
}
sql_case = {
'doc_type': 'CommCareCase',
'user_id': 'e7ad965c70802884a7a67add763939e8',
'case_id': '5ac45838-da5b-49f5-b236-0675ff924e9f'
}
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
filtered = filter_case_diffs(couch_case, sql_case, diffs)
self.assertEqual(filtered, [])
def test_filter_usercase_diff(self):
couch_case = {
'doc_type': 'CommCareCase',
'hq_user_id': '123',
'external_id': '',
'type': 'commcare-user'
}
user_case_diffs = [
FormJsonDiff(diff_type=u'diff',
path=(u'external_id', ),
old_value=u'',
new_value=u'123')
]
filtered = filter_case_diffs(couch_case, {},
user_case_diffs + REAL_DIFFS)
self.assertEqual(filtered, REAL_DIFFS)
def _diff_cases(self, couch_cases):
from corehq.apps.tzmigration.timezonemigration import json_diff
log.debug('Calculating case diffs for {} cases'.format(
len(couch_cases)))
statedb = self.statedb
counts = defaultdict(int)
case_ids = list(couch_cases)
sql_cases = CaseAccessorSQL.get_cases(case_ids)
sql_case_ids = set()
for sql_case in sql_cases:
sql_case_ids.add(sql_case.case_id)
couch_case = couch_cases[sql_case.case_id]
sql_case_json = sql_case.to_json()
diffs = json_diff(couch_case,
sql_case_json,
track_list_indices=False)
diffs = filter_case_diffs(couch_case, sql_case_json, diffs,
statedb)
if diffs and not sql_case.is_deleted:
try:
couch_case, diffs = self._rebuild_couch_case_and_re_diff(
couch_case, sql_case_json)
except Exception as err:
log.warning('Case {} rebuild -> {}: {}'.format(
sql_case.case_id,
type(err).__name__, err))
if diffs:
statedb.add_diffs(couch_case['doc_type'], sql_case.case_id,
diffs)
counts[couch_case['doc_type']] += 1
self._diff_ledgers(case_ids)
if len(case_ids) != len(sql_case_ids):
couch_ids = set(case_ids)
assert not (sql_case_ids - couch_ids), sql_case_ids - couch_ids
missing_cases = [couch_cases[x] for x in couch_ids - sql_case_ids]
log.debug("Found %s missing SQL cases", len(missing_cases))
for doc_type, doc_ids in self._filter_missing_cases(missing_cases):
statedb.add_missing_docs(doc_type, doc_ids)
counts[doc_type] += len(doc_ids)
for doc_type, count in six.iteritems(counts):
statedb.increment_counter(doc_type, count)
self.processed_docs += len(case_ids)
self._log_case_diff_count(throttled=True)
def test_filter_case_xform_id_diffs_good(self):
couch_case = {'doc_type': 'CommCareCase', 'xform_ids': ['123', '456']}
sql_case = {'doc_type': 'CommCareCase', 'xform_ids': ['456', '123']}
diffs = [
FormJsonDiff(diff_type=u'diff',
path=('xform_ids', '[*]'),
old_value=u'123',
new_value=u'456'),
FormJsonDiff(diff_type=u'diff',
path=('xform_ids', '[*]'),
old_value=u'455',
new_value=u'123')
]
filtered = filter_case_diffs(couch_case, sql_case, diffs + REAL_DIFFS)
self.assertEqual(filtered, REAL_DIFFS)
def test_filter_usercase_diff(self):
couch_case = {
'doc_type': 'CommCareCase',
'hq_user_id': '123',
'external_id': '',
'type': 'commcare-user'
}
sql_case = {
'doc_type': 'CommCareCase',
'external_id': '123',
'type': 'commcare-user'
}
user_case_diffs = json_diff(couch_case, sql_case)
self.assertEqual(2, len(user_case_diffs))
filtered = filter_case_diffs(couch_case, sql_case, user_case_diffs + REAL_DIFFS)
self.assertEqual(filtered, REAL_DIFFS)
def test_non_user_owner_mapping_case_with_opened_and_user_diffs(self):
couch_case = {
"case_id": "eca7a8",
"actions": [{"action_type": "create", "user_id": "somebody"}],
"doc_type": "CommCareCase",
"opened_by": None,
"user_id": "",
}
sql_case = {
"case_id": "eca7a8",
"actions": [{"action_type": "create", "user_id": "somebody"}],
"doc_type": "CommCareCase",
"opened_by": "somebody",
"user_id": "somebody",
}
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
filtered = filter_case_diffs(couch_case, sql_case, diffs)
self.assertEqual(filtered, [])
def test_filter_usercase_diff_bad(self):
couch_case = {
'doc_type': 'CommCareCase',
'hq_user_id': '123',
'type': 'commcare-user'
}
sql_case = {'doc_type': 'CommCareCase', 'type': 'commcare-user'}
user_case_diffs = json_diff(couch_case, sql_case)
self.assertEqual(1, len(user_case_diffs))
filtered = filter_case_diffs(couch_case, sql_case, user_case_diffs)
self.assertEqual(filtered, [
FormJsonDiff(diff_type='missing',
path=('hq_user_id', ),
old_value='123',
new_value=MISSING)
])
def test_filter_usercase_diff(self):
couch_case = {
'doc_type': 'CommCareCase',
'hq_user_id': '123',
'external_id': '',
'type': 'commcare-user'
}
sql_case = {
'doc_type': 'CommCareCase',
'external_id': '123',
'type': 'commcare-user'
}
user_case_diffs = json_diff(couch_case, sql_case)
self.assertEqual(2, len(user_case_diffs))
filtered = filter_case_diffs(couch_case, sql_case, user_case_diffs + REAL_DIFFS)
self.assertEqual(filtered, REAL_DIFFS)
def test_case_with_opened_by_diff2(self):
couch_case = {
"doc_type": "CommCareCase",
"actions": [
{
"action_type": "close",
"user_id": "somebody",
},
],
"opened_by": None,
}
sql_case = {
"doc_type": "CommCareCase",
"opened_by": "somebody",
}
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
filtered = filter_case_diffs(couch_case, sql_case, diffs)
self.assertEqual(filtered, [])
def test_filter_case_xform_id_diffs_bad(self):
couch_case = {'doc_type': 'CommCareCase', 'xform_ids': ['123', '456']}
sql_case = {'doc_type': 'CommCareCase', 'xform_ids': ['123', 'abc']}
diffs = [
FormJsonDiff(diff_type=u'diff',
path=('xform_ids', '[*]'),
old_value=u'456',
new_value=u'abc')
]
expected_diffs = REAL_DIFFS + [
FormJsonDiff(diff_type='set_mismatch',
path=('xform_ids', '[*]'),
old_value='456',
new_value='abc')
]
filtered = filter_case_diffs(couch_case, sql_case, diffs + REAL_DIFFS)
self.assertEqual(filtered, expected_diffs)
def test_weird_case_attributes(self):
couch_case = {
"case_id": "eca7a8",
"actions": [{"action_type": "create", "user_id": "person-2"}],
"doc_type": "CommCareCase",
"@user_id": "person-1",
"user_id": "person-2",
"@date_modified": "2016-02-12",
"modified_on": "2016-02-12T00:00:00.000000Z",
}
sql_case = {
"case_id": "eca7a8",
"actions": [{"action_type": "create", "user_id": "person-2"}],
"doc_type": "CommCareCase",
"user_id": "person-2",
"modified_on": "2016-02-12T10:00:00.000000Z",
}
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
filtered = filter_case_diffs(couch_case, sql_case, diffs)
self.assertEqual(filtered, [])
def test_filter_case_xform_id_diffs_bad(self):
couch_case = {
'doc_type': 'CommCareCase',
'xform_ids': ['123', '456']
}
sql_case = {
'doc_type': 'CommCareCase',
'xform_ids': ['123', 'abc']
}
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
self.assertEqual(1, len(diffs))
diffs = [
FormJsonDiff(diff_type=u'diff', path=('xform_ids', '[*]'), old_value=u'456', new_value=u'abc')
]
expected_diffs = REAL_DIFFS + [
FormJsonDiff(diff_type='set_mismatch', path=('xform_ids', '[*]'), old_value='456', new_value='abc')
]
filtered = filter_case_diffs(couch_case, sql_case, diffs + REAL_DIFFS)
self.assertEqual(filtered, expected_diffs)
def test_filter_usercase_diff_bad(self):
couch_case = {
'doc_type': 'CommCareCase',
'hq_user_id': '123',
'type': 'commcare-user'
}
sql_case = {
'doc_type': 'CommCareCase',
'type': 'commcare-user'
}
user_case_diffs = json_diff(couch_case, sql_case)
self.assertEqual(1, len(user_case_diffs))
filtered = filter_case_diffs(couch_case, sql_case, user_case_diffs)
self.assertEqual(filtered, [
FormJsonDiff(
diff_type='complex', path=('hq_user_id', 'external_id'),
old_value='123', new_value=Ellipsis
)
])
def test_case_attachments(self):
couch_case = {
'doc_type': 'CommCareCase',
'case_attachments': {
'xyz': {
'doc_type': 'ignored',
'attachment_properties': 'ignored',
'attachment_from': 'ignored',
'attachment_src': 'ignored',
'server_mime': 'ignored',
'attachment_name': 'ignored',
'server_md5': 'ignored',
'identifier': 'xyz',
'attachment_size': 123,
'unexpected': 'value',
'properties': 'value',
},
},
}
sql_case = {
'doc_type': 'CommCareCase',
'case_attachments': {
'xyz': {
'name': 'xyz',
'content_length': 123,
'content_type': 'ignored-sql',
# for testing only, not an expected transformation
'properties': 'eulav',
},
},
}
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
filtered = filter_case_diffs(couch_case, sql_case, diffs)
self.assertEqual(filtered, [
FormJsonDiff('missing', ('case_attachments', 'xyz', 'unexpected'),
'value', MISSING),
FormJsonDiff('diff', ('case_attachments', 'xyz', 'properties'),
'value', 'eulav'),
])
def test_case_attachments(self):
couch_case = {
'doc_type': 'CommCareCase',
'case_attachments': {
'xyz': {
'doc_type': 'ignored',
'attachment_properties': 'ignored',
'attachment_from': 'ignored',
'attachment_src': 'ignored',
'server_mime': 'ignored',
'attachment_name': 'ignored',
'server_md5': 'ignored',
'identifier': 'xyz',
'attachment_size': 123,
'unexpected': 'value',
'properties': 'value',
},
},
}
sql_case = {
'doc_type': 'CommCareCase',
'case_attachments': {
'xyz': {
'name': 'xyz',
'content_length': 123,
'content_type': 'ignored-sql',
# for testing only, not an expected transformation
'properties': 'eulav',
},
},
}
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
filtered = filter_case_diffs(couch_case, sql_case, diffs)
self.assertEqual(filtered, [
FormJsonDiff('missing', ('case_attachments', 'xyz', 'unexpected'), 'value', MISSING),
FormJsonDiff('diff', ('case_attachments', 'xyz', 'properties'), 'value', 'eulav'),
])
def test_filter_case_deletion_fields(self):
couch_case = {'doc_type': 'CommCareCase-Deleted'}
filtered = filter_case_diffs(couch_case, {}, DELETION_DIFFS + REAL_DIFFS)
self.assertEqual(filtered, REAL_DIFFS)
def test_filter_case_diffs(self):
couch_case = {'doc_type': 'CommCareCase'}
diffs = _make_ignored_diffs('CommCareCase') + DATE_DIFFS + REAL_DIFFS
filtered = filter_case_diffs(couch_case, {}, diffs)
self.assertEqual(filtered, REAL_DIFFS)
def test_filter_case_diffs(self):
couch_case = {'doc_type': 'CommCareCase'}
diffs = _make_ignored_diffs('CommCareCase') + DATE_DIFFS + REAL_DIFFS
filtered = filter_case_diffs(couch_case, {}, diffs)
self.assertEqual(filtered, REAL_DIFFS)
def test_case_name_diff(self):
couch_case = {'doc_type': 'CommCareCase', 'name': 'shorter'}
sql_case = {'doc_type': 'CommCareCase', 'name': 'short'}
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
filtered = filter_case_diffs(couch_case, sql_case, diffs)
self.assertEqual(filtered, [FormJsonDiff('diff', ('name',), 'shorter', 'short')])
def test_filter_case_deletion_fields(self):
couch_case = {'doc_type': 'CommCareCase-Deleted'}
filtered = filter_case_diffs(couch_case, {},
DELETION_DIFFS + REAL_DIFFS)
self.assertEqual(filtered, REAL_DIFFS)
def test_case_name_missing(self):
couch_case = {'doc_type': 'CommCareCase'}
sql_case = {'doc_type': 'CommCareCase', 'name': 'thing'}
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
filtered = filter_case_diffs(couch_case, sql_case, diffs)
self.assertEqual(filtered, [FormJsonDiff('missing', ('name',), MISSING, 'thing')])
if case_id:
print("%s: diffs" % case_id)
for diff in diffs:
print(diff)
def is_problem_case(case, form_db):
orig_case_json = case.to_json()
sorted_actions = sorted(case.actions, key=lambda x: x.server_date)
sorted_form_ids = [action.xform_id for action in sorted_actions]
if sorted_form_ids != case.xform_ids:
rebuild_case_from_sorted_actions(case, sorted_actions)
rebuilt_case_json = case.to_json()
diffs = json_diff(orig_case_json, rebuilt_case_json)
diffs = filter_case_diffs(
orig_case_json, rebuilt_case_json, diffs
)
diffs = [diff for diff in diffs if diff.path[0] not in ['actions', 'xform_ids']]
diffs = [d for d in diffs if d.old_value or d.new_value]
if diffs:
return (case.case_id, diffs)
return False, None
def rebuild_case_from_sorted_actions(case, sorted_actions):
strategy = CouchCaseUpdateStrategy(case)
strategy.reset_case_state()
case.xform_ids = []
case.actions = [a for a in sorted_actions if not a.deprecated]
for a in case.actions:
strategy._apply_action(a, None)
def test_case_name_truncated(self):
couch_case = {'doc_type': 'CommCareCase', 'name': 'ha' * 128}
sql_case = {'doc_type': 'CommCareCase', 'name': 'ha' * 126 + 'hah'}
diffs = json_diff(couch_case, sql_case, track_list_indices=False)
filtered = filter_case_diffs(couch_case, sql_case, diffs)
self.assertEqual(filtered, [])
def test_filter_case_diffs(self):
couch_case = {'doc_type': 'CommCareCase'}
partial_diffs = _get_partial_diffs('CommCareCase')
diffs = list(CASE_IGNORED_DIFFS) + partial_diffs + DATE_DIFFS + REAL_DIFFS
filtered = filter_case_diffs(couch_case, {}, diffs)
self.assertEqual(filtered, REAL_DIFFS)
