def handle(self, *args, **options): from casexml.apps.phone.models import properly_wrap_sync_log, SyncLog, SimplifiedSyncLog if len(args) < 1: print "Usage: ./manage.py sync_log_debugger <filename1> [<filename2>] [<filename3>]..." sys.exit(0) logs = [] log_names = [] for filename in args: if os.path.isdir(filename): filenames = [os.path.join(filename, item) for item in sorted(os.listdir(filename))] else: filenames = [filename] for filename in filenames: log_name = os.path.basename(filename) log_names.append(log_name) with open(filename) as f: wrapped_log = properly_wrap_sync_log(json.loads(f.read())) logs.append(wrapped_log) if isinstance(wrapped_log, SyncLog): log_names.append("migrated-{}".format(log_name)) logs.append(SimplifiedSyncLog.from_other_format(wrapped_log)) elif getattr(wrapped_log, "migrated_from", None): log_names.append("migrated_from-{}".format(log_name)) logs.append(properly_wrap_sync_log(wrapped_log.to_json()["migrated_from"])) print "state hashes" for i in range(len(log_names)): print "{} ({}): {}".format(log_names[i], logs[i]._id, logs[i].get_state_hash()) print "\ncase diffs" for i in range(len(log_names)): for j in range(len(log_names)): if i != j: case_diff = set(logs[i].get_footprint_of_cases_on_phone()) - set( logs[j].get_footprint_of_cases_on_phone() ) if case_diff: print "cases on {} and not {}: {}".format( log_names[i], log_names[j], ", ".join(sorted(case_diff)) ) if options["debugger"]: union_of_ids = set().union(*[set(log.get_footprint_of_cases_on_phone()) for log in logs]) intersection_of_ids = set().intersection(*[set(log.get_footprint_of_cases_on_phone()) for log in logs]) import pdb pdb.set_trace() if options["check_hash"]: log_to_check = logs[int(options["index"])] result = _brute_force_search( log_to_check.case_ids_on_phone, options["check_hash"], depth=int(options["depth"]) ) if result: print "check successful - missing ids {}".format(result) else: print "no match found"
def test_update_dependent_case(self): sync_log = SyncLog( cases_on_phone=[ CaseState( case_id='bran', indices=[ CommCareCaseIndex(identifier='legs', referenced_id='hodor') ], ), ], dependent_cases_on_phone=[CaseState(case_id='hodor')], user_id="someuser") xform_id = uuid.uuid4().hex xform = XFormInstance(_id=xform_id) form_actions = [CommCareCaseAction(action_type=CASE_ACTION_UPDATE, )] with patch.object(CommCareCase, 'get_actions_for_form', return_value=form_actions): parent_case = CommCareCase(_id='hodor') # before this test was added, the following call raised a SyncLogAssertionError on legacy logs. # this test just ensures it doesn't still do that. for log in [ sync_log, SimplifiedSyncLog.from_other_format(sync_log) ]: log.update_phone_lists(xform, [parent_case])
def test_indices(self): parents = ["catelyn", "ned", "cersei", "jaimie"] index_structure = { "bran": [{"identifier": "mom", "referenced_id": "catelyn"}, {"identifier": "dad", "referenced_id": "ned"}], "myrcella": [ {"identifier": "mom", "referenced_id": "cersei"}, {"identifier": "dad", "referenced_id": "jaimie"}, ], } sync_log = SyncLog( cases_on_phone=[ CaseState(case_id="bran", indices=[CommCareCaseIndex(**args) for args in index_structure["bran"]]), CaseState( case_id="myrcella", indices=[CommCareCaseIndex(**args) for args in index_structure["myrcella"]] ), ], dependent_cases_on_phone=[CaseState(case_id=parent) for parent in parents], ) migrated = SimplifiedSyncLog.from_other_format(sync_log) for case_id, indices in index_structure.items(): self.assertTrue(case_id in migrated.index_tree.indices) for index in indices: self.assertEqual(index["referenced_id"], migrated.index_tree.indices[case_id][index["identifier"]]) for parent in parents: self.assertTrue(parent in migrated.case_ids_on_phone) self.assertTrue(parent in migrated.dependent_case_ids_on_phone)
def test_update_dependent_case_owner_still_present(self): dependent_case_state = CaseState(case_id="d1", indices=[]) sync_log = SyncLog(domain="domain", user_id="user", cases_on_phone=[ CaseState(case_id="c1", indices=[ CommCareCaseIndex( identifier="d1-id", referenced_id="d1") ]) ], dependent_cases_on_phone=[dependent_case_state], owner_ids_on_phone=['user1']) xform_id = uuid.uuid4().hex xform = XFormInstance(_id=xform_id) form_actions = [ CommCareCaseAction(action_type=CASE_ACTION_UPDATE, updated_known_properties={'owner_id': 'user2'}) ] with patch.object(CommCareCase, 'get_actions_for_form', return_value=form_actions): parent_case = CommCareCase(_id='d1') # before this test was added, the following call raised a ValueError on legacy logs. for log in [ sync_log, SimplifiedSyncLog.from_other_format(sync_log) ]: log.update_phone_lists(xform, [parent_case]) self.assertIn(dependent_case_state, log.test_only_get_dependent_cases_on_phone())
def test_cases_on_phone(self): case_ids = ["nymeria", "lady"] sync_log = SyncLog(cases_on_phone=[CaseState(case_id=case_id) for case_id in case_ids]) migrated = SimplifiedSyncLog.from_other_format(sync_log) for case_id in case_ids: self.assertTrue(case_id in migrated.case_ids_on_phone) self.assertFalse(case_id in migrated.dependent_case_ids_on_phone)
def test_indices(self): parents = ['catelyn', 'ned', 'cersei', 'jaimie'] index_structure = { 'bran': [ {'identifier': 'mom', 'referenced_id': 'catelyn'}, {'identifier': 'dad', 'referenced_id': 'ned'}, ], 'myrcella': [ {'identifier': 'mom', 'referenced_id': 'cersei'}, {'identifier': 'dad', 'referenced_id': 'jaimie'}, ] } sync_log = SyncLog( cases_on_phone=[ CaseState(case_id='bran', indices=[ CommCareCaseIndex(**args) for args in index_structure['bran'] ]), CaseState(case_id='myrcella', indices=[ CommCareCaseIndex(**args) for args in index_structure['myrcella'] ]) ], dependent_cases_on_phone=[ CaseState(case_id=parent) for parent in parents ] ) migrated = SimplifiedSyncLog.from_other_format(sync_log) for case_id, indices in index_structure.items(): self.assertTrue(case_id in migrated.index_tree.indices) for index in indices: self.assertEqual(index['referenced_id'], migrated.index_tree.indices[case_id][index['identifier']]) for parent in parents: self.assertTrue(parent in migrated.case_ids_on_phone) self.assertTrue(parent in migrated.dependent_case_ids_on_phone)
def last_sync_log(self): if self._last_sync_log is Ellipsis: if self.params.sync_log_id: # if we are in loose mode, return an HTTP 412 so that the phone will # just force a fresh sync # This raises MissingSyncLog exception if synclog not found sync_log = get_properly_wrapped_sync_log( self.params.sync_log_id) if sync_log.doc_type not in ('SyncLog', 'SimplifiedSyncLog'): raise InvalidSyncLogException( 'Bad sync log doc type for {}'.format( self.params.sync_log_id)) elif sync_log.user_id != self.restore_user.user_id: raise SyncLogUserMismatch( 'Sync log {} does not match user id {} (was {})'. format(self.params.sync_log_id, self.restore_user.user_id, sync_log.user_id)) # convert to the right type if necessary if not isinstance(sync_log, SimplifiedSyncLog): # this call can fail with an IncompatibleSyncLogType error sync_log = SimplifiedSyncLog.from_other_format(sync_log) self._last_sync_log = sync_log else: self._last_sync_log = None return self._last_sync_log
def test_prune_on_migrate(self): sync_log = SyncLog( cases_on_phone=[CaseState(case_id="robert"), CaseState(case_id="cersei")], dependent_cases_on_phone=[CaseState(case_id="gendry")], ) migrated = SimplifiedSyncLog.from_other_format(sync_log) self.assertTrue("gendry" not in migrated.case_ids_on_phone) self.assertEqual(sync_log.get_state_hash(), migrated.get_state_hash())
def test_cases_on_phone(self): case_ids = ['nymeria', 'lady'] sync_log = SyncLog(cases_on_phone=[ CaseState(case_id=case_id) for case_id in case_ids ], ) migrated = SimplifiedSyncLog.from_other_format(sync_log) for case_id in case_ids: self.assertTrue(case_id in migrated.case_ids_on_phone) self.assertFalse(case_id in migrated.dependent_case_ids_on_phone)
def test_properties_deleted(self): sync_log = SyncLog( cases_on_phone=[CaseState(case_id="nymeria")], dependent_cases_on_phone=[CaseState(case_id="lady")] ) self.assertTrue(hasattr(sync_log, "cases_on_phone")) self.assertTrue(hasattr(sync_log, "dependent_cases_on_phone")) migrated = SimplifiedSyncLog.from_other_format(sync_log) self.assertFalse(hasattr(migrated, "cases_on_phone")) self.assertFalse(hasattr(migrated, "dependent_cases_on_phone"))
def handle(self, sync_logs, **options): from casexml.apps.phone.models import properly_wrap_sync_log, SyncLog, SimplifiedSyncLog logs = [] log_names = [] for filename in sync_logs: if os.path.isdir(filename): filenames = [os.path.join(filename, item) for item in sorted(os.listdir(filename))] else: filenames = [filename] for filename in filenames: log_name = os.path.basename(filename) log_names.append(log_name) with open(filename, encoding='utf-8') as f: wrapped_log = properly_wrap_sync_log(json.loads(f.read())) logs.append(wrapped_log) if isinstance(wrapped_log, SyncLog): log_names.append('migrated-{}'.format(log_name)) logs.append(SimplifiedSyncLog.from_other_format(wrapped_log)) elif getattr(wrapped_log, 'migrated_from', None): log_names.append('migrated_from-{}'.format(log_name)) logs.append(properly_wrap_sync_log(wrapped_log.to_json()['migrated_from'])) print('state hashes') for i in range(len(log_names)): print('{} ({}): {}'.format(log_names[i], logs[i]._id, logs[i].get_state_hash())) print('\ncase diffs') for i in range(len(log_names)): for j in range(len(log_names)): if i != j: case_diff = set(logs[i].get_footprint_of_cases_on_phone()) - \ set(logs[j].get_footprint_of_cases_on_phone()) if case_diff: print('cases on {} and not {}: {}'.format( log_names[i], log_names[j], ', '.join(sorted(case_diff)) )) if options['debugger']: union_of_ids = set().union(*[set(log.get_footprint_of_cases_on_phone()) for log in logs]) intersection_of_ids = set().intersection(*[set(log.get_footprint_of_cases_on_phone()) for log in logs]) import pdb pdb.set_trace() if options['check_hash']: log_to_check = logs[int(options['index'])] result = _brute_force_search( log_to_check.case_ids_on_phone, options['check_hash'], depth=int(options['depth']) ) if result: print('check successful - missing ids {}'.format(result)) else: print('no match found')
def test_purge_on_migrate(self): sync_log = SyncLog( cases_on_phone=[ CaseState(case_id='robert'), CaseState(case_id='cersei'), ], dependent_cases_on_phone=[CaseState(case_id='gendry')]) migrated = SimplifiedSyncLog.from_other_format(sync_log) self.assertTrue('gendry' not in migrated.case_ids_on_phone) self.assertEqual(sync_log.get_state_hash(), migrated.get_state_hash())
def test_properties_deleted(self): sync_log = SyncLog( cases_on_phone=[CaseState(case_id='nymeria')], dependent_cases_on_phone=[CaseState(case_id='lady')], ) self.assertTrue(hasattr(sync_log, 'cases_on_phone')) self.assertTrue(hasattr(sync_log, 'dependent_cases_on_phone')) migrated = SimplifiedSyncLog.from_other_format(sync_log) self.assertFalse(hasattr(migrated, 'cases_on_phone')) self.assertFalse(hasattr(migrated, 'dependent_cases_on_phone'))
def test_dependent_cases_on_phone(self): sync_log = SyncLog( cases_on_phone=[ CaseState(case_id="bran", indices=[CommCareCaseIndex(identifier="legs", referenced_id="hodor")]) ], dependent_cases_on_phone=[CaseState(case_id="hodor")], ) migrated = SimplifiedSyncLog.from_other_format(sync_log) self.assertTrue("bran" in migrated.case_ids_on_phone) self.assertTrue("hodor" in migrated.case_ids_on_phone) self.assertTrue("hodor" in migrated.dependent_case_ids_on_phone)
def test_shared_properties_migrate(self): attrs = { 'date': datetime.utcnow(), 'user_id': 'ned', 'previous_log_id': 'previous-log', 'duration': 10, 'owner_ids': ['arya', 'sansa'], } sync_log = SyncLog(**attrs) migrated = SimplifiedSyncLog.from_other_format(sync_log) for k, v in attrs.items(): self.assertEqual(v, getattr(migrated, k))
def test_shared_properties_migrate(self): attrs = { "date": datetime.utcnow(), "user_id": "ned", "previous_log_id": "previous-log", "duration": 10, "owner_ids": ["arya", "sansa"], } sync_log = SyncLog(**attrs) migrated = SimplifiedSyncLog.from_other_format(sync_log) for k, v in attrs.items(): self.assertEqual(v, getattr(migrated, k))
def test_purge_on_migrate(self): sync_log = SyncLog( cases_on_phone=[ CaseState(case_id='robert'), CaseState(case_id='cersei'), ], dependent_cases_on_phone=[ CaseState(case_id='gendry') ] ) migrated = SimplifiedSyncLog.from_other_format(sync_log) self.assertTrue('gendry' not in migrated.case_ids_on_phone) self.assertEqual(sync_log.get_state_hash(), migrated.get_state_hash())
def handle(self, *args, **options): from casexml.apps.phone.models import properly_wrap_sync_log, SyncLog, SimplifiedSyncLog if len(args) < 1: print 'Usage: ./manage.py sync_log_debugger <filename1> [<filename2>] [<filename3>]...' sys.exit(0) logs = [] log_names = [] for filename in args: if os.path.isdir(filename): filenames = [ os.path.join(filename, item) for item in sorted(os.listdir(filename)) ] else: filenames = [filename] for filename in filenames: log_name = os.path.basename(filename) log_names.append(log_name) with open(filename) as f: wrapped_log = properly_wrap_sync_log(json.loads(f.read())) logs.append(wrapped_log) if isinstance(wrapped_log, SyncLog): log_names.append('migrated-{}'.format(log_name)) logs.append( SimplifiedSyncLog.from_other_format(wrapped_log)) print 'state hashes' for i in range(len(log_names)): print '{} ({}): {}'.format(log_names[i], logs[i]._id, logs[i].get_state_hash()) print '\ncase diffs' for i in range(len(log_names)): for j in range(len(log_names)): if i != j: case_diff = set(logs[i].get_footprint_of_cases_on_phone()) - \ set(logs[j].get_footprint_of_cases_on_phone()) if case_diff: print 'cases on {} and not {}: {}'.format( log_names[i], log_names[j], ', '.join(case_diff)) if options['debugger']: union_of_ids = set().union( *[set(log.get_footprint_of_cases_on_phone()) for log in logs]) intersection_of_ids = set().intersection( *[set(log.get_footprint_of_cases_on_phone()) for log in logs]) import pdb pdb.set_trace()
def test_dependent_cases_on_phone(self): sync_log = SyncLog( cases_on_phone=[ CaseState( case_id='bran', indices=[CommCareCaseIndex(identifier='legs', referenced_id='hodor')], ), ], dependent_cases_on_phone=[CaseState(case_id='hodor')] ) migrated = SimplifiedSyncLog.from_other_format(sync_log) self.assertTrue('bran' in migrated.case_ids_on_phone) self.assertTrue('hodor' in migrated.case_ids_on_phone) self.assertTrue('hodor' in migrated.dependent_case_ids_on_phone)
def test_update_dependent_case(self): sync_log = SyncLog( cases_on_phone=[ CaseState(case_id="bran", indices=[CommCareCaseIndex(identifier="legs", referenced_id="hodor")]) ], dependent_cases_on_phone=[CaseState(case_id="hodor")], ) xform_id = uuid.uuid4().hex xform = XFormInstance(_id=xform_id) form_actions = [CommCareCaseAction(action_type=CASE_ACTION_UPDATE)] with patch.object(CommCareCase, "get_actions_for_form", return_value=form_actions): parent_case = CommCareCase(_id="hodor") # before this test was added, the following call raised a SyncLogAssertionError on legacy logs. # this test just ensures it doesn't still do that. for log in [sync_log, SimplifiedSyncLog.from_other_format(sync_log)]: log.update_phone_lists(xform, [parent_case])
def test_update_dependent_case_owner_still_present(self): dependent_case_state = CaseState(case_id="d1", indices=[]) sync_log = SyncLog( cases_on_phone=[ CaseState(case_id="c1", indices=[ CommCareCaseIndex(identifier="d1-id", referenced_id="d1") ]) ], dependent_cases_on_phone=[dependent_case_state], owner_ids_on_phone=['user1'] ) xform_id = uuid.uuid4().hex xform = XFormInstance(_id=xform_id) form_actions = [CommCareCaseAction(action_type=CASE_ACTION_UPDATE, updated_known_properties={'owner_id': 'user2'})] with patch.object(CommCareCase, 'get_actions_for_form', return_value=form_actions): parent_case = CommCareCase(_id='d1') # before this test was added, the following call raised a ValueError on legacy logs. for log in [sync_log, SimplifiedSyncLog.from_other_format(sync_log)]: log.update_phone_lists(xform, [parent_case]) self.assertIn(dependent_case_state, log.test_only_get_dependent_cases_on_phone())
def last_sync_log(self): if self._last_sync_log is Ellipsis: if self.params.sync_log_id: # if we are in loose mode, return an HTTP 412 so that the phone will # just force a fresh sync # This raises MissingSyncLog exception if synclog not found sync_log = get_properly_wrapped_sync_log(self.params.sync_log_id) if sync_log.doc_type not in ('SyncLog', 'SimplifiedSyncLog'): raise InvalidSyncLogException('Bad sync log doc type for {}'.format(self.params.sync_log_id)) elif sync_log.user_id != self.restore_user.user_id: raise SyncLogUserMismatch('Sync log {} does not match user id {} (was {})'.format( self.params.sync_log_id, self.restore_user.user_id, sync_log.user_id )) # convert to the right type if necessary if not isinstance(sync_log, SimplifiedSyncLog): # this call can fail with an IncompatibleSyncLogType error sync_log = SimplifiedSyncLog.from_other_format(sync_log) self._last_sync_log = sync_log else: self._last_sync_log = None return self._last_sync_log
def handle(self, *args, **options): from casexml.apps.phone.models import properly_wrap_sync_log, SyncLog, SimplifiedSyncLog if len(args) < 1: print( "Usage:\n" "./manage.py sync_log_debugger <synclog1> [synclog2 synclog3]...\n" " <synclog> is a json file of the synclog you are trying to compare. Passing\n" " in a folder will compare all of the files in that folder.\n" ) sys.exit(0) logs = [] log_names = [] for filename in args: if os.path.isdir(filename): filenames = [os.path.join(filename, item) for item in sorted(os.listdir(filename))] else: filenames = [filename] for filename in filenames: log_name = os.path.basename(filename) log_names.append(log_name) with open(filename) as f: wrapped_log = properly_wrap_sync_log(json.loads(f.read())) logs.append(wrapped_log) if isinstance(wrapped_log, SyncLog): log_names.append('migrated-{}'.format(log_name)) logs.append(SimplifiedSyncLog.from_other_format(wrapped_log)) elif getattr(wrapped_log, 'migrated_from', None): log_names.append('migrated_from-{}'.format(log_name)) logs.append(properly_wrap_sync_log(wrapped_log.to_json()['migrated_from'])) print 'state hashes' for i in range(len(log_names)): print '{} ({}): {}'.format(log_names[i], logs[i]._id, logs[i].get_state_hash()) print '\ncase diffs' for i in range(len(log_names)): for j in range(len(log_names)): if i != j: case_diff = set(logs[i].get_footprint_of_cases_on_phone()) - \ set(logs[j].get_footprint_of_cases_on_phone()) if case_diff: print 'cases on {} and not {}: {}'.format( log_names[i], log_names[j], ', '.join(sorted(case_diff)) ) if options['debugger']: union_of_ids = set().union(*[set(log.get_footprint_of_cases_on_phone()) for log in logs]) intersection_of_ids = set().intersection(*[set(log.get_footprint_of_cases_on_phone()) for log in logs]) import pdb pdb.set_trace() if options['check_hash']: log_to_check = logs[int(options['index'])] result = _brute_force_search( log_to_check.case_ids_on_phone, options['check_hash'], depth=int(options['depth']) ) if result: print 'check successful - missing ids {}'.format(result) else: print 'no match found'
def test_log_format_chages(self): sync_log = SyncLog() self.assertEqual(LOG_FORMAT_LEGACY, sync_log.log_format) migrated = SimplifiedSyncLog.from_other_format(sync_log) self.assertEqual(LOG_FORMAT_SIMPLIFIED, migrated.log_format)
def handle(self, *args, **options): from casexml.apps.phone.models import properly_wrap_sync_log, SyncLog, SimplifiedSyncLog if len(args) < 1: print( "Usage:\n" "./manage.py sync_log_debugger <synclog1> [synclog2 synclog3]...\n" " <synclog> is a json file of the synclog you are trying to compare. Passing\n" " in a folder will compare all of the files in that folder.\n" ) sys.exit(0) logs = [] log_names = [] for filename in args: if os.path.isdir(filename): filenames = [ os.path.join(filename, item) for item in sorted(os.listdir(filename)) ] else: filenames = [filename] for filename in filenames: log_name = os.path.basename(filename) log_names.append(log_name) with open(filename) as f: wrapped_log = properly_wrap_sync_log(json.loads(f.read())) logs.append(wrapped_log) if isinstance(wrapped_log, SyncLog): log_names.append('migrated-{}'.format(log_name)) logs.append( SimplifiedSyncLog.from_other_format(wrapped_log)) elif getattr(wrapped_log, 'migrated_from', None): log_names.append('migrated_from-{}'.format(log_name)) logs.append( properly_wrap_sync_log( wrapped_log.to_json()['migrated_from'])) print 'state hashes' for i in range(len(log_names)): print '{} ({}): {}'.format(log_names[i], logs[i]._id, logs[i].get_state_hash()) print '\ncase diffs' for i in range(len(log_names)): for j in range(len(log_names)): if i != j: case_diff = set(logs[i].get_footprint_of_cases_on_phone()) - \ set(logs[j].get_footprint_of_cases_on_phone()) if case_diff: print 'cases on {} and not {}: {}'.format( log_names[i], log_names[j], ', '.join(sorted(case_diff))) if options['debugger']: union_of_ids = set().union( *[set(log.get_footprint_of_cases_on_phone()) for log in logs]) intersection_of_ids = set().intersection( *[set(log.get_footprint_of_cases_on_phone()) for log in logs]) import pdb pdb.set_trace() if options['check_hash']: log_to_check = logs[int(options['index'])] result = _brute_force_search(log_to_check.case_ids_on_phone, options['check_hash'], depth=int(options['depth'])) if result: print 'check successful - missing ids {}'.format(result) else: print 'no match found'
def test_livequery_to_simplified(self): sync_log = SimplifiedSyncLog(log_format=LOG_FORMAT_LIVEQUERY) with self.assertRaises(IncompatibleSyncLogType): SimplifiedSyncLog.from_other_format(sync_log)