def test_should_sync_when_changed(self, *args): self.user._couch_user.add_to_assigned_locations( self.locations['Boston']) last_sync_time = datetime.utcnow() sync_log = SimplifiedSyncLog(date=last_sync_time) locations_queryset = SQLLocation.objects.filter( pk=self.locations['Boston'].pk) restore_state = MockRestoreState(self.user, RestoreParams()) self.assertFalse( should_sync_locations(sync_log, locations_queryset, restore_state)) self.assertEqual( len( call_fixture_generator(related_locations_fixture_generator, self.user, last_sync=sync_log)), 0) LocationRelation.objects.create(location_a=self.locations["Revere"], location_b=self.locations["Boston"]) self.assertTrue( should_sync_locations(SimplifiedSyncLog(date=last_sync_time), locations_queryset, restore_state)) # length 2 for index definition + data self.assertEqual( len( call_fixture_generator(related_locations_fixture_generator, self.user, last_sync=sync_log)), 2)
def setUpClass(cls): db = SimplifiedSyncLog.get_db() # datetime.min is not compatible for `json_format_datetime` for synclog_id in get_synclog_ids_by_date(datetime(1970, 1, 1), datetime.max): db.delete_doc(synclog_id) # Needed because other tests do not always clean up their users. delete_all_users() hard_delete_deleted_users() cls.g1 = Group(domain=cls.domain, name='group') cls.g1.save() cls.g2 = Group(domain=cls.domain, name='group') cls.g2.soft_delete() cls.domain_obj = Domain( name=cls.domain, is_active=True, ) cls.domain_obj.save() cls.web_user = WebUser.create(cls.domain, 'web-user', '***') cls.commcare_user = CommCareUser.create(cls.domain, 'cc-user', '***') cls.commcare_user.retire() cls.synclog = SimplifiedSyncLog( domain=cls.domain, build_id='1234', user_id='5678', date=datetime.utcnow(), ) cls.synclog.save()
def test_update_dependent_case_owner_still_present(self): sync_log = SimplifiedSyncLog( domain="domain", case_ids_on_phone={'c1', 'd1'}, dependent_case_ids_on_phone={'d1'}, index_tree=IndexTree(indices={'c1': { 'd1-id': 'd1' }}), user_id="user", owner_ids_on_phone={'user1'}) xform_id = uuid.uuid4().hex xform = create_form_for_test("domain", form_id=xform_id, save=False) form_actions = [ CaseAction( action_type=CASE_ACTION_UPDATE, updated_known_properties={'owner_id': 'user2'}, indices=[], ) ] with patch.object(CommCareCase, 'get_actions_for_form', return_value=form_actions): parent_case = CommCareCase(case_id='d1') # before this test was added, the following call raised a ValueError on legacy logs. sync_log.update_phone_lists(xform, [parent_case]) self.assertIn("d1", sync_log.dependent_case_ids_on_phone)
def test_archiving_location_should_resync(self): """ When locations are archived, we should resync them """ location = make_location( domain=self.domain, name='winterfell', location_type=self.location_type.name, ) location.save() after_save = datetime.utcnow() self.assertEqual('winterfell', location.name) locations_queryset = SQLLocation.objects.filter(pk=location.pk) restore_state = MockRestoreState(self.user.to_ota_restore_user(), RestoreParams()) # Should not resync if last sync was after location save self.assertFalse( should_sync_locations(SimplifiedSyncLog(date=after_save), locations_queryset, restore_state)) # archive the location location.archive() after_archive = datetime.utcnow() location = SQLLocation.objects.last() locations_queryset = SQLLocation.objects.filter(pk=location.pk) # Should resync if last sync was after location was saved but before location was archived self.assertTrue( should_sync_locations(SimplifiedSyncLog(date=after_save), locations_queryset, restore_state)) # Should not resync if last sync was after location was deleted self.assertFalse( should_sync_locations(SimplifiedSyncLog(date=after_archive), locations_queryset, restore_state))
def test_sync_log(self): from casexml.apps.phone.models import SyncLog, SimplifiedSyncLog from corehq.apps.users.models import WebUser, CommCareUser from casexml.apps.phone.models import get_sync_log_class_by_format web_user = WebUser.create( domain=self.domain_name, username='******', password='******', email='*****@*****.**', ) mobile_user = CommCareUser.create( self.domain_name, 'mobile_user1', 'secret' ) other_user = CommCareUser.create( 'other_domain', 'mobile_user2', 'secret' ) self.addCleanup(other_user.delete) l1 = SyncLog(user_id=web_user._id) l1.save() l2 = SimplifiedSyncLog(user_id=mobile_user._id) l2.save() other_log = SyncLog(user_id=other_user._id) other_log.save() def _synclog_to_class(doc): if doc['doc_type'] == 'SyncLog': return get_sync_log_class_by_format(doc.get('log_format')) expected_docs = [web_user, mobile_user, l1, l2] not_expected_docs = [other_user, other_log] self._dump_and_load(expected_docs, not_expected_docs, doc_to_doc_class=_synclog_to_class)
def test_purge_multiple_children(self): [grandparent_id, parent_id, child_id_1, child_id_2] = all_ids = ['rickard', 'ned', 'bran', 'arya'] tree = IndexTree( indices={ child_id_1: convert_list_to_dict([parent_id]), child_id_2: convert_list_to_dict([parent_id]), parent_id: convert_list_to_dict([grandparent_id]), }) sync_log = SimplifiedSyncLog(index_tree=tree, case_ids_on_phone=set(all_ids)) # first purge the parent and grandparent sync_log.purge(grandparent_id) sync_log.purge(parent_id) self.assertTrue(grandparent_id in sync_log.case_ids_on_phone) self.assertTrue(grandparent_id in sync_log.dependent_case_ids_on_phone) self.assertTrue(parent_id in sync_log.case_ids_on_phone) self.assertTrue(parent_id in sync_log.dependent_case_ids_on_phone) # just purging one child should preserve the parent index sync_log.purge(child_id_1) self.assertTrue(grandparent_id in sync_log.case_ids_on_phone) self.assertTrue(grandparent_id in sync_log.dependent_case_ids_on_phone) self.assertTrue(parent_id in sync_log.case_ids_on_phone) self.assertTrue(parent_id in sync_log.dependent_case_ids_on_phone) self.assertFalse(child_id_1 in sync_log.case_ids_on_phone) # purging the other one should wipe it sync_log.purge(child_id_2) for id in all_ids: self.assertFalse(id in sync_log.case_ids_on_phone) self.assertFalse(id in sync_log.dependent_case_ids_on_phone)
def test_pillow(self): from corehq.apps.change_feed.topics import get_topic_offset from corehq.pillows.synclog import get_user_sync_history_pillow consumer = get_test_kafka_consumer(topics.SYNCLOG_SQL) # get the seq id before the change is published kafka_seq = get_topic_offset(topics.SYNCLOG_SQL) # make sure user has empty reporting-metadata before a sync ccuser = CommCareUser.get(self.ccuser._id) self.assertEqual(ccuser.reporting_metadata.last_syncs, []) # do a sync synclog = SimplifiedSyncLog(domain=self.domain.name, user_id=self.ccuser._id, date=datetime.datetime(2015, 7, 1, 0, 0), app_id='123') synclog.save() # make sure kafka change updates the user with latest sync info message = next(consumer) change_meta = change_meta_from_kafka_message(message.value) synclog = self._get_latest_synclog() self.assertEqual(change_meta.document_id, synclog._id) self.assertEqual(change_meta.domain, self.domain.name) # make sure processor updates the user correctly pillow = get_user_sync_history_pillow() pillow.process_changes(since=kafka_seq) process_reporting_metadata_staging() ccuser = CommCareUser.get(self.ccuser._id) self.assertEqual(len(ccuser.reporting_metadata.last_syncs), 1) self.assertEqual(ccuser.reporting_metadata.last_syncs[0].sync_date, synclog.date) self.assertEqual(ccuser.reporting_metadata.last_sync_for_user.sync_date, synclog.date)
def test_purge_self_indexing(self): [id] = ['recursive'] tree = IndexTree(indices={ id: convert_list_to_dict([id]), }) sync_log = SimplifiedSyncLog(index_tree=tree, case_ids_on_phone=set([id])) sync_log.purge(id) self.assertFalse(id in sync_log.case_ids_on_phone) self.assertFalse(id in sync_log.dependent_case_ids_on_phone)
def test_prune_self_indexing(self): [id] = ['recursive'] tree = IndexTree(indices={ id: convert_list_to_dict([id]), }) sync_log = SimplifiedSyncLog(index_tree=tree, case_ids_on_phone=set([id])) sync_log.prune_case(id) self.assertFalse(id in sync_log.case_ids_on_phone) self.assertFalse(id in sync_log.dependent_case_ids_on_phone)
def test_should_sync_timezone(self): domain = Domain(name='test', default_timezone='Africa/Johannesburg') # yesterday at 21:59:59 = yesterday at 23:59:59 locally last_sync = datetime.combine(date.today() - timedelta(days=1), time(21, 59, 59)) # yesterday at 21:59:59 = today at 00:00:00 locally utcnow = datetime.combine(date.today() - timedelta(days=1), time(22, 00, 00)) self.assertTrue(should_sync(domain, SimplifiedSyncLog(date=last_sync), utcnow=utcnow)) domain = Domain(name='test', default_timezone='UTC') self.assertFalse(should_sync(domain, SimplifiedSyncLog(date=last_sync), utcnow=utcnow))
def test_prune_multiple_children(self): [grandparent_id, parent_id, child_id_1, child_id_2] = all_ids = ['rickard', 'ned', 'bran', 'arya'] tree = IndexTree(indices={ child_id_1: convert_list_to_dict([parent_id]), child_id_2: convert_list_to_dict([parent_id]), parent_id: convert_list_to_dict([grandparent_id]), }) sync_log = SimplifiedSyncLog(index_tree=tree, case_ids_on_phone=set(all_ids)) # first prune the parent and grandparent sync_log.prune_case(grandparent_id) sync_log.prune_case(parent_id) self.assertTrue(grandparent_id in sync_log.case_ids_on_phone) self.assertTrue(grandparent_id in sync_log.dependent_case_ids_on_phone) self.assertTrue(parent_id in sync_log.case_ids_on_phone) self.assertTrue(parent_id in sync_log.dependent_case_ids_on_phone) # just pruning one child should preserve the parent index sync_log.prune_case(child_id_1) self.assertTrue(grandparent_id in sync_log.case_ids_on_phone) self.assertTrue(grandparent_id in sync_log.dependent_case_ids_on_phone) self.assertTrue(parent_id in sync_log.case_ids_on_phone) self.assertTrue(parent_id in sync_log.dependent_case_ids_on_phone) self.assertFalse(child_id_1 in sync_log.case_ids_on_phone) # pruning the other one should wipe it sync_log.prune_case(child_id_2) for id in all_ids: self.assertFalse(id in sync_log.case_ids_on_phone) self.assertFalse(id in sync_log.dependent_case_ids_on_phone)
def test_purge_extension_non_dependent_host(self): """Purging an extension should not remove the host or itself if the host is directly owned """ [host_id, extension_id] = all_ids = ['host', 'extension'] extension_tree = IndexTree(indices={ extension_id: convert_list_to_dict([host_id]), }) sync_log = SimplifiedSyncLog(extension_index_tree=extension_tree, case_ids_on_phone=set(all_ids)) sync_log.purge(extension_id) self.assertTrue(extension_id in sync_log.case_ids_on_phone) self.assertTrue(host_id in sync_log.case_ids_on_phone)
def test_sync_log_invalidation_bug(self): sync_log = SimplifiedSyncLog( user_id='6dac4940-913e-11e0-9d4b-005056aa7fb5') sync_log.save() self.addCleanup(FormProcessorTestUtils.delete_all_sync_logs) _, case = self._doCreateCaseWithMultimedia() # this used to fail before we fixed http://manage.dimagi.com/default.asp?158373 self._doSubmitUpdateWithMultimedia( new_attachments=['commcare_logo_file'], removes=[], sync_token=sync_log._id)
def test_purge_partial_children(self): [parent_id, child_id_1, child_id_2] = all_ids = ['parent', 'child1', 'child2'] tree = IndexTree(indices={ child_id_1: convert_list_to_dict([parent_id]), child_id_2: convert_list_to_dict([parent_id]), }) sync_log = SimplifiedSyncLog( index_tree=tree, case_ids_on_phone=set(all_ids), dependent_case_ids_on_phone=set([parent_id, child_id_2]) ) # this used to fail with an AssertionError sync_log.purge(parent_id)
def test_return_412_between_bug_dates(self): log = SimplifiedSyncLog(user_id=self.restore_user.user_id, date=datetime(2016, 7, 19, 19, 20)) log.save() restore_config = RestoreConfig(project=self.project, restore_user=self.restore_user, params=RestoreParams( sync_log_id=log._id, version="2.0", ), cache_settings=RestoreCacheSettings()) response = restore_config.get_response() self.assertEqual(response.status_code, 412)
def test_purge_multiple_parents(self): [grandparent_id, mother_id, father_id, child_id] = all_ids = ['heart-tree', 'catelyn', 'ned', 'arya'] tree = IndexTree( indices={ child_id: convert_list_to_dict([mother_id, father_id]), mother_id: convert_list_to_dict([grandparent_id]), father_id: convert_list_to_dict([grandparent_id]), }) sync_log = SimplifiedSyncLog(index_tree=tree, case_ids_on_phone=set(all_ids)) # first purge everything but the child sync_log.purge(grandparent_id) sync_log.purge(mother_id) sync_log.purge(father_id) # everything should still be relevant because of the child for id in all_ids: self.assertTrue(id in sync_log.case_ids_on_phone) # purging the child should wipe everything else sync_log.purge(child_id) for id in all_ids: self.assertFalse(id in sync_log.case_ids_on_phone) self.assertFalse(id in sync_log.dependent_case_ids_on_phone)
def test_purge_extension(self, ): """Purging extension removes host """ [host_id, extension_id] = all_ids = ['host', 'extension'] extension_tree = IndexTree(indices={ extension_id: convert_list_to_dict([host_id]), }) sync_log = SimplifiedSyncLog(extension_index_tree=extension_tree, dependent_case_ids_on_phone=set([host_id]), case_ids_on_phone=set(all_ids)) sync_log.purge(extension_id) self.assertFalse(extension_id in sync_log.case_ids_on_phone) self.assertFalse(host_id in sync_log.case_ids_on_phone)
def test_purge_tiered_bottom_up(self): [grandparent_id, parent_id, child_id] = all_ids = ['grandparent', 'parent', 'child'] tree = IndexTree( indices={ child_id: convert_list_to_dict([parent_id]), parent_id: convert_list_to_dict([grandparent_id]), }) sync_log = SimplifiedSyncLog(index_tree=tree, case_ids_on_phone=set(all_ids)) # just purging the child should purge just the child sync_log.purge(child_id) self.assertTrue(grandparent_id in sync_log.case_ids_on_phone) self.assertTrue(parent_id in sync_log.case_ids_on_phone) self.assertFalse(child_id in sync_log.case_ids_on_phone) # same for the parent sync_log.purge(parent_id) self.assertTrue(grandparent_id in sync_log.case_ids_on_phone) self.assertFalse(parent_id in sync_log.case_ids_on_phone) # same for the grandparentparent sync_log.purge(grandparent_id) self.assertFalse(grandparent_id in sync_log.case_ids_on_phone)
def test_purge_tiered_top_down(self): [grandparent_id, parent_id, child_id] = all_ids = ['grandparent', 'parent', 'child'] tree = IndexTree( indices={ child_id: convert_list_to_dict([parent_id]), parent_id: convert_list_to_dict([grandparent_id]), }) sync_log = SimplifiedSyncLog(index_tree=tree, case_ids_on_phone=set(all_ids)) # this has no effect other than to move the grandparent to dependent sync_log.purge(grandparent_id) for id in all_ids: self.assertTrue(id in sync_log.case_ids_on_phone) self.assertTrue(grandparent_id in sync_log.dependent_case_ids_on_phone) self.assertFalse(parent_id in sync_log.dependent_case_ids_on_phone) self.assertFalse(child_id in sync_log.dependent_case_ids_on_phone) # likewise, this should have no effect other than to move the parent to dependent sync_log.purge(parent_id) for id in all_ids: self.assertTrue(id in sync_log.case_ids_on_phone) self.assertTrue(grandparent_id in sync_log.dependent_case_ids_on_phone) self.assertTrue(parent_id in sync_log.dependent_case_ids_on_phone) self.assertFalse(child_id in sync_log.dependent_case_ids_on_phone) # this should now purge everything sync_log.purge(child_id) for id in all_ids: self.assertFalse(id in sync_log.case_ids_on_phone) self.assertFalse(id in sync_log.dependent_case_ids_on_phone)
def test_synced_during_and_after_bug_resolution_returns_200(self): during = SimplifiedSyncLog( user_id=self.restore_user.user_id, date=datetime(2016, 7, 19, 20, 0) # during bug ) during.save() after = SimplifiedSyncLog( user_id=self.restore_user.user_id, previous_log_id=during._id, date=datetime(2016, 7, 21, 19, 0) # after resolution ) after.save() restore_config = RestoreConfig( project=self.project, restore_user=self.restore_user, params=RestoreParams( sync_log_id=after._id, version="2.0", ), cache_settings=RestoreCacheSettings() ) response = restore_config.get_response() self.assertEqual(response.status_code, 200)
def test_synced_before_and_after_bug_resolution_200(self): before = SimplifiedSyncLog( user_id=self.restore_user.user_id, date=datetime(2016, 7, 19, 18, 0) # synced before bug was introduced ) before.save() restore_config = RestoreConfig(project=self.project, restore_user=self.restore_user, params=RestoreParams( sync_log_id=before._id, version="2.0", ), cache_settings=RestoreCacheSettings()) response = restore_config.get_response() self.assertEqual(response.status_code, 200) after = SimplifiedSyncLog( user_id=self.restore_user.user_id, previous_log_id=before._id, date=datetime(2016, 7, 21, 19, 0) # after resolution ) after.save() restore_config = RestoreConfig(project=self.project, restore_user=self.restore_user, params=RestoreParams( sync_log_id=after._id, version="2.0", ), cache_settings=RestoreCacheSettings()) response = restore_config.get_response() self.assertEqual(response.status_code, 200)
def test_purge_extension_host_has_multiple_extensions(self): """Purging an extension should remove host and its other extensions """ [host_id, extension_id, extension_id_2] = all_ids = ['host', 'extension', 'extension_2'] extension_tree = IndexTree(indices={ extension_id: convert_list_to_dict([host_id]), extension_id_2: convert_list_to_dict([host_id]), }) sync_log = SimplifiedSyncLog(extension_index_tree=extension_tree, dependent_case_ids_on_phone=set([host_id, extension_id_2]), case_ids_on_phone=set(all_ids)) sync_log.purge(extension_id) self.assertFalse(extension_id in sync_log.case_ids_on_phone) self.assertFalse(extension_id_2 in sync_log.case_ids_on_phone) self.assertFalse(host_id in sync_log.case_ids_on_phone)
def handle(self, user_id, date, **options): results = SimplifiedSyncLog.view( "phone/sync_logs_by_user", startkey=[user_id, {}], endkey=[user_id, date], descending=True, reduce=False, include_docs=True, ) logs = [] for log in results: log.case_ids_on_phone = {'broken to force 412'} logs.append(log) SimplifiedSyncLog.bulk_save(logs)
def test_open_extension_of_extension(self): all_ids = ['host', 'extension', 'extension_of_extension'] host_id, extension_id, extension_of_extension_id = all_ids extension_tree = IndexTree(indices={ extension_id: convert_list_to_dict([host_id]), extension_of_extension_id: convert_list_to_dict([extension_id]), }) sync_log = SimplifiedSyncLog(extension_index_tree=extension_tree, dependent_case_ids_on_phone=set([host_id, extension_id]), closed_cases=set([host_id, extension_id]), case_ids_on_phone=set(all_ids)) sync_log.purge(host_id) self.assertFalse(host_id in sync_log.case_ids_on_phone) self.assertFalse(extension_id in sync_log.case_ids_on_phone) self.assertFalse(extension_of_extension_id in sync_log.case_ids_on_phone)
def test_cases_on_phone(self): case_ids = ["nymeria", "lady"] sync_log = SyncLog(cases_on_phone=[CaseState(case_id=case_id) for case_id in case_ids]) migrated = SimplifiedSyncLog.from_other_format(sync_log) for case_id in case_ids: self.assertTrue(case_id in migrated.case_ids_on_phone) self.assertFalse(case_id in migrated.dependent_case_ids_on_phone)
def test_update_dependent_case_owner_still_present(self): dependent_case_state = CaseState(case_id="d1", indices=[]) sync_log = SyncLog(domain="domain", user_id="user", cases_on_phone=[ CaseState(case_id="c1", indices=[ CommCareCaseIndex( identifier="d1-id", referenced_id="d1") ]) ], dependent_cases_on_phone=[dependent_case_state], owner_ids_on_phone=['user1']) xform_id = uuid.uuid4().hex xform = XFormInstance(_id=xform_id) form_actions = [ CommCareCaseAction(action_type=CASE_ACTION_UPDATE, updated_known_properties={'owner_id': 'user2'}) ] with patch.object(CommCareCase, 'get_actions_for_form', return_value=form_actions): parent_case = CommCareCase(_id='d1') # before this test was added, the following call raised a ValueError on legacy logs. for log in [ sync_log, SimplifiedSyncLog.from_other_format(sync_log) ]: log.update_phone_lists(xform, [parent_case]) self.assertIn(dependent_case_state, log.test_only_get_dependent_cases_on_phone())
def test_indices(self): parents = ["catelyn", "ned", "cersei", "jaimie"] index_structure = { "bran": [{"identifier": "mom", "referenced_id": "catelyn"}, {"identifier": "dad", "referenced_id": "ned"}], "myrcella": [ {"identifier": "mom", "referenced_id": "cersei"}, {"identifier": "dad", "referenced_id": "jaimie"}, ], } sync_log = SyncLog( cases_on_phone=[ CaseState(case_id="bran", indices=[CommCareCaseIndex(**args) for args in index_structure["bran"]]), CaseState( case_id="myrcella", indices=[CommCareCaseIndex(**args) for args in index_structure["myrcella"]] ), ], dependent_cases_on_phone=[CaseState(case_id=parent) for parent in parents], ) migrated = SimplifiedSyncLog.from_other_format(sync_log) for case_id, indices in index_structure.items(): self.assertTrue(case_id in migrated.index_tree.indices) for index in indices: self.assertEqual(index["referenced_id"], migrated.index_tree.indices[case_id][index["identifier"]]) for parent in parents: self.assertTrue(parent in migrated.case_ids_on_phone) self.assertTrue(parent in migrated.dependent_case_ids_on_phone)
def last_sync_log(self): if self._last_sync_log is Ellipsis: if self.params.sync_log_id: # if we are in loose mode, return an HTTP 412 so that the phone will # just force a fresh sync # This raises MissingSyncLog exception if synclog not found sync_log = get_properly_wrapped_sync_log( self.params.sync_log_id) if sync_log.doc_type not in ('SyncLog', 'SimplifiedSyncLog'): raise InvalidSyncLogException( 'Bad sync log doc type for {}'.format( self.params.sync_log_id)) elif sync_log.user_id != self.restore_user.user_id: raise SyncLogUserMismatch( 'Sync log {} does not match user id {} (was {})'. format(self.params.sync_log_id, self.restore_user.user_id, sync_log.user_id)) # convert to the right type if necessary if not isinstance(sync_log, SimplifiedSyncLog): # this call can fail with an IncompatibleSyncLogType error sync_log = SimplifiedSyncLog.from_other_format(sync_log) self._last_sync_log = sync_log else: self._last_sync_log = None return self._last_sync_log
def test_update_dependent_case(self): sync_log = SyncLog( cases_on_phone=[ CaseState( case_id='bran', indices=[ CommCareCaseIndex(identifier='legs', referenced_id='hodor') ], ), ], dependent_cases_on_phone=[CaseState(case_id='hodor')], user_id="someuser") xform_id = uuid.uuid4().hex xform = XFormInstance(_id=xform_id) form_actions = [CommCareCaseAction(action_type=CASE_ACTION_UPDATE, )] with patch.object(CommCareCase, 'get_actions_for_form', return_value=form_actions): parent_case = CommCareCase(_id='hodor') # before this test was added, the following call raised a SyncLogAssertionError on legacy logs. # this test just ensures it doesn't still do that. for log in [ sync_log, SimplifiedSyncLog.from_other_format(sync_log) ]: log.update_phone_lists(xform, [parent_case])
def handle(self, *args, **options): from casexml.apps.phone.models import properly_wrap_sync_log, SyncLog, SimplifiedSyncLog if len(args) < 1: print "Usage: ./manage.py sync_log_debugger <filename1> [<filename2>] [<filename3>]..." sys.exit(0) logs = [] log_names = [] for filename in args: if os.path.isdir(filename): filenames = [os.path.join(filename, item) for item in sorted(os.listdir(filename))] else: filenames = [filename] for filename in filenames: log_name = os.path.basename(filename) log_names.append(log_name) with open(filename) as f: wrapped_log = properly_wrap_sync_log(json.loads(f.read())) logs.append(wrapped_log) if isinstance(wrapped_log, SyncLog): log_names.append("migrated-{}".format(log_name)) logs.append(SimplifiedSyncLog.from_other_format(wrapped_log)) elif getattr(wrapped_log, "migrated_from", None): log_names.append("migrated_from-{}".format(log_name)) logs.append(properly_wrap_sync_log(wrapped_log.to_json()["migrated_from"])) print "state hashes" for i in range(len(log_names)): print "{} ({}): {}".format(log_names[i], logs[i]._id, logs[i].get_state_hash()) print "\ncase diffs" for i in range(len(log_names)): for j in range(len(log_names)): if i != j: case_diff = set(logs[i].get_footprint_of_cases_on_phone()) - set( logs[j].get_footprint_of_cases_on_phone() ) if case_diff: print "cases on {} and not {}: {}".format( log_names[i], log_names[j], ", ".join(sorted(case_diff)) ) if options["debugger"]: union_of_ids = set().union(*[set(log.get_footprint_of_cases_on_phone()) for log in logs]) intersection_of_ids = set().intersection(*[set(log.get_footprint_of_cases_on_phone()) for log in logs]) import pdb pdb.set_trace() if options["check_hash"]: log_to_check = logs[int(options["index"])] result = _brute_force_search( log_to_check.case_ids_on_phone, options["check_hash"], depth=int(options["depth"]) ) if result: print "check successful - missing ids {}".format(result) else: print "no match found"
def test_indices(self): parents = ['catelyn', 'ned', 'cersei', 'jaimie'] index_structure = { 'bran': [ {'identifier': 'mom', 'referenced_id': 'catelyn'}, {'identifier': 'dad', 'referenced_id': 'ned'}, ], 'myrcella': [ {'identifier': 'mom', 'referenced_id': 'cersei'}, {'identifier': 'dad', 'referenced_id': 'jaimie'}, ] } sync_log = SyncLog( cases_on_phone=[ CaseState(case_id='bran', indices=[ CommCareCaseIndex(**args) for args in index_structure['bran'] ]), CaseState(case_id='myrcella', indices=[ CommCareCaseIndex(**args) for args in index_structure['myrcella'] ]) ], dependent_cases_on_phone=[ CaseState(case_id=parent) for parent in parents ] ) migrated = SimplifiedSyncLog.from_other_format(sync_log) for case_id, indices in index_structure.items(): self.assertTrue(case_id in migrated.index_tree.indices) for index in indices: self.assertEqual(index['referenced_id'], migrated.index_tree.indices[case_id][index['identifier']]) for parent in parents: self.assertTrue(parent in migrated.case_ids_on_phone) self.assertTrue(parent in migrated.dependent_case_ids_on_phone)
def _new_sync_log(self): previous_log_id = None if self.is_initial else self.last_sync_log._id new_synclog = SimplifiedSyncLog( _id=uuid.uuid1().hex.lower(), domain=self.restore_user.domain, build_id=self.params.app_id, user_id=self.restore_user.user_id, owner_ids_on_phone=set(self.owner_ids), date=datetime.utcnow(), previous_log_id=previous_log_id, extensions_checked=True, device_id=self.params.device_id, ) if self.is_livequery: new_synclog.log_format = LOG_FORMAT_LIVEQUERY return new_synclog
def test_prune_circular_loops(self): [peer_id_1, peer_id_2] = all_ids = ['jaime', 'cersei'] tree = IndexTree(indices={ peer_id_1: convert_list_to_dict([peer_id_2]), peer_id_2: convert_list_to_dict([peer_id_1]), }) sync_log = SimplifiedSyncLog(index_tree=tree, case_ids_on_phone=set(all_ids)) # pruning one peer should keep everything around sync_log.prune_case(peer_id_1) for id in all_ids: self.assertTrue(id in sync_log.case_ids_on_phone) # pruning the second peer should remove everything sync_log.prune_case(peer_id_2) for id in all_ids: self.assertFalse(id in sync_log.case_ids_on_phone)
def test_prune_on_migrate(self): sync_log = SyncLog( cases_on_phone=[CaseState(case_id="robert"), CaseState(case_id="cersei")], dependent_cases_on_phone=[CaseState(case_id="gendry")], ) migrated = SimplifiedSyncLog.from_other_format(sync_log) self.assertTrue("gendry" not in migrated.case_ids_on_phone) self.assertEqual(sync_log.get_state_hash(), migrated.get_state_hash())
def test_prune_parent_then_child(self): [parent_id, child_id] = all_ids = ['parent', 'child'] tree = IndexTree(indices={ child_id: convert_list_to_dict([parent_id]), }) sync_log = SimplifiedSyncLog(index_tree=tree, case_ids_on_phone=set(all_ids)) # this has no effect sync_log.prune_case(parent_id) self.assertTrue(child_id in sync_log.case_ids_on_phone) self.assertTrue(parent_id in sync_log.case_ids_on_phone) self.assertFalse(child_id in sync_log.dependent_case_ids_on_phone) self.assertTrue(parent_id in sync_log.dependent_case_ids_on_phone) # this should prune it entirely sync_log.prune_case(child_id) self.assertFalse(child_id in sync_log.case_ids_on_phone) self.assertFalse(parent_id in sync_log.case_ids_on_phone)
def test_force_empty_when_user_has_no_locations(self, *args): sync_log = SimplifiedSyncLog(date=datetime.utcnow()) # no relations have been touched since this SimplifiedSyncLog, but it still pushes down the empty list self.assertEqual( len( call_fixture_generator(related_locations_fixture_generator, self.user, last_sync=sync_log)), 2)
def test_prune_tiered_top_down(self): [grandparent_id, parent_id, child_id] = all_ids = ['grandparent', 'parent', 'child'] tree = IndexTree(indices={ child_id: convert_list_to_dict([parent_id]), parent_id: convert_list_to_dict([grandparent_id]), }) sync_log = SimplifiedSyncLog(index_tree=tree, case_ids_on_phone=set(all_ids)) # this has no effect other than to move the grandparent to dependent sync_log.prune_case(grandparent_id) for id in all_ids: self.assertTrue(id in sync_log.case_ids_on_phone) self.assertTrue(grandparent_id in sync_log.dependent_case_ids_on_phone) self.assertFalse(parent_id in sync_log.dependent_case_ids_on_phone) self.assertFalse(child_id in sync_log.dependent_case_ids_on_phone) # likewise, this should have no effect other than to move the parent to dependent sync_log.prune_case(parent_id) for id in all_ids: self.assertTrue(id in sync_log.case_ids_on_phone) self.assertTrue(grandparent_id in sync_log.dependent_case_ids_on_phone) self.assertTrue(parent_id in sync_log.dependent_case_ids_on_phone) self.assertFalse(child_id in sync_log.dependent_case_ids_on_phone) # this should now prune everything sync_log.prune_case(child_id) for id in all_ids: self.assertFalse(id in sync_log.case_ids_on_phone) self.assertFalse(id in sync_log.dependent_case_ids_on_phone)
def test_prune_very_circular_loops(self): [peer_id_1, peer_id_2, peer_id_3] = all_ids = ['drogon', 'rhaegal', 'viserion'] tree = IndexTree(indices={ peer_id_1: convert_list_to_dict([peer_id_2]), peer_id_2: convert_list_to_dict([peer_id_3]), peer_id_3: convert_list_to_dict([peer_id_1]), }) sync_log = SimplifiedSyncLog(index_tree=tree, case_ids_on_phone=set(all_ids)) # pruning the first two, should still keep everything around sync_log.prune_case(peer_id_1) sync_log.prune_case(peer_id_2) for id in all_ids: self.assertTrue(id in sync_log.case_ids_on_phone) sync_log.prune_case(peer_id_3) for id in all_ids: self.assertFalse(id in sync_log.case_ids_on_phone)
def test_properties_deleted(self): sync_log = SyncLog( cases_on_phone=[CaseState(case_id="nymeria")], dependent_cases_on_phone=[CaseState(case_id="lady")] ) self.assertTrue(hasattr(sync_log, "cases_on_phone")) self.assertTrue(hasattr(sync_log, "dependent_cases_on_phone")) migrated = SimplifiedSyncLog.from_other_format(sync_log) self.assertFalse(hasattr(migrated, "cases_on_phone")) self.assertFalse(hasattr(migrated, "dependent_cases_on_phone"))
def test_copy_payload(self): sync_log = SimplifiedSyncLog(case_ids_on_phone=set(['case-1', 'case-2'])) sync_log.save() payload = dummy_restore_xml(sync_log._id).strip() fd, path = tempfile.mkstemp() with os.fdopen(fd, 'wb') as f: f.write(payload) with open(path, 'r') as f: updated_fileref = copy_payload_and_synclog_and_get_new_file(f) updated_payload = updated_fileref.file.read() updated_id = synclog_id_from_restore_payload(updated_payload) self.assertNotEqual(sync_log._id, updated_id) self.assertTrue(_restore_id_block(updated_id) in updated_payload) self.assertFalse(sync_log._id in updated_payload) updated_log = get_properly_wrapped_sync_log(updated_id) self.assertEqual(updated_log.case_ids_on_phone, sync_log.case_ids_on_phone)
def test_cases_on_phone(self): case_ids = ['nymeria', 'lady'] sync_log = SyncLog(cases_on_phone=[ CaseState(case_id=case_id) for case_id in case_ids ], ) migrated = SimplifiedSyncLog.from_other_format(sync_log) for case_id in case_ids: self.assertTrue(case_id in migrated.case_ids_on_phone) self.assertFalse(case_id in migrated.dependent_case_ids_on_phone)
def test_copy_payload(self): sync_log = SimplifiedSyncLog(case_ids_on_phone=set(["case-1", "case-2"])) sync_log.save() payload = dummy_restore_xml(sync_log._id).strip() fd, path = tempfile.mkstemp() with os.fdopen(fd, "wb") as f: f.write(payload) with open(path, "r") as f: updated_fileref = copy_payload_and_synclog_and_get_new_file(f) updated_payload = updated_fileref.file.read() updated_id = synclog_id_from_restore_payload(updated_payload) self.assertNotEqual(sync_log._id, updated_id) self.assertTrue(_restore_id_block(updated_id) in updated_payload) self.assertFalse(sync_log._id in updated_payload) updated_log = get_properly_wrapped_sync_log(updated_id) self.assertEqual(updated_log.case_ids_on_phone, sync_log.case_ids_on_phone)
def test_purge_child_then_parent(self): [parent_id, child_id] = all_ids = ['parent', 'child'] tree = IndexTree(indices={ child_id: convert_list_to_dict([parent_id]), }) sync_log = SimplifiedSyncLog(index_tree=tree, case_ids_on_phone=set(all_ids)) # this should purge the child but not the parent sync_log.purge(child_id) self.assertFalse(child_id in sync_log.case_ids_on_phone) self.assertTrue(parent_id in sync_log.case_ids_on_phone) self.assertFalse(child_id in sync_log.dependent_case_ids_on_phone) self.assertFalse(parent_id in sync_log.dependent_case_ids_on_phone) # then purging the parent should purge it sync_log.purge(parent_id) self.assertFalse(parent_id in sync_log.case_ids_on_phone) self.assertFalse(parent_id in sync_log.dependent_case_ids_on_phone)
def handle(self, sync_logs, **options): from casexml.apps.phone.models import properly_wrap_sync_log, SyncLog, SimplifiedSyncLog logs = [] log_names = [] for filename in sync_logs: if os.path.isdir(filename): filenames = [os.path.join(filename, item) for item in sorted(os.listdir(filename))] else: filenames = [filename] for filename in filenames: log_name = os.path.basename(filename) log_names.append(log_name) with open(filename, encoding='utf-8') as f: wrapped_log = properly_wrap_sync_log(json.loads(f.read())) logs.append(wrapped_log) if isinstance(wrapped_log, SyncLog): log_names.append('migrated-{}'.format(log_name)) logs.append(SimplifiedSyncLog.from_other_format(wrapped_log)) elif getattr(wrapped_log, 'migrated_from', None): log_names.append('migrated_from-{}'.format(log_name)) logs.append(properly_wrap_sync_log(wrapped_log.to_json()['migrated_from'])) print('state hashes') for i in range(len(log_names)): print('{} ({}): {}'.format(log_names[i], logs[i]._id, logs[i].get_state_hash())) print('\ncase diffs') for i in range(len(log_names)): for j in range(len(log_names)): if i != j: case_diff = set(logs[i].get_footprint_of_cases_on_phone()) - \ set(logs[j].get_footprint_of_cases_on_phone()) if case_diff: print('cases on {} and not {}: {}'.format( log_names[i], log_names[j], ', '.join(sorted(case_diff)) )) if options['debugger']: union_of_ids = set().union(*[set(log.get_footprint_of_cases_on_phone()) for log in logs]) intersection_of_ids = set().intersection(*[set(log.get_footprint_of_cases_on_phone()) for log in logs]) import pdb pdb.set_trace() if options['check_hash']: log_to_check = logs[int(options['index'])] result = _brute_force_search( log_to_check.case_ids_on_phone, options['check_hash'], depth=int(options['depth']) ) if result: print('check successful - missing ids {}'.format(result)) else: print('no match found')
def test_purge_extension_host_is_parent(self): """Purging an extension should not purge the host or the extension if the host is a depenency for a child """ [host_id, extension_id, child_id] = all_ids = ['host', 'extension', 'child'] child_tree = IndexTree(indices={ child_id: convert_list_to_dict([host_id]), }) extension_tree = IndexTree(indices={ extension_id: convert_list_to_dict([host_id]), }) sync_log = SimplifiedSyncLog(extension_index_tree=extension_tree, index_tree=child_tree, dependent_case_ids_on_phone=set([host_id]), case_ids_on_phone=set(all_ids)) sync_log.purge(extension_id) self.assertTrue(extension_id in sync_log.case_ids_on_phone) self.assertTrue(child_id in sync_log.case_ids_on_phone) self.assertTrue(host_id in sync_log.case_ids_on_phone)
def test_purge_child_of_extension(self): """Purging child of extension should remove extension and host """ [host_id, extension_id, child_id] = all_ids = ['host', 'extension', 'child'] child_tree = IndexTree(indices={ child_id: convert_list_to_dict([extension_id]), }) extension_tree = IndexTree(indices={ extension_id: convert_list_to_dict([host_id]), }) sync_log = SimplifiedSyncLog(extension_index_tree=extension_tree, index_tree=child_tree, dependent_case_ids_on_phone=set([host_id, extension_id]), case_ids_on_phone=set(all_ids)) sync_log.purge(child_id) self.assertFalse(extension_id in sync_log.case_ids_on_phone) self.assertFalse(child_id in sync_log.case_ids_on_phone) self.assertFalse(host_id in sync_log.case_ids_on_phone)
def test_dependent_cases_on_phone(self): sync_log = SyncLog( cases_on_phone=[ CaseState(case_id="bran", indices=[CommCareCaseIndex(identifier="legs", referenced_id="hodor")]) ], dependent_cases_on_phone=[CaseState(case_id="hodor")], ) migrated = SimplifiedSyncLog.from_other_format(sync_log) self.assertTrue("bran" in migrated.case_ids_on_phone) self.assertTrue("hodor" in migrated.case_ids_on_phone) self.assertTrue("hodor" in migrated.dependent_case_ids_on_phone)
def handle(self, *args, **options): if len(args) != 2: raise CommandError("Usage is ./manage.py invalidate_sync_heads %s" % self.args) user_id = args[0] date = args[1] results = synclog_view( "phone/sync_logs_by_user", startkey=[user_id, {}], endkey=[user_id, date], descending=True, reduce=False, include_docs=True, ) logs = [] for res in results: log = SimplifiedSyncLog.wrap(res['doc']) log.case_ids_on_phone = {'broken to force 412'} logs.append(log) SimplifiedSyncLog.bulk_save(logs)
def test_open_child_of_extension(self): [host_id, extension_id, child_of_extension_id] = all_ids = ['host', 'extension', 'child_of_extension'] extension_tree = IndexTree(indices={ extension_id: convert_list_to_dict([host_id]), }) child_tree = IndexTree(indices={ child_of_extension_id: convert_list_to_dict([extension_id]), }) sync_log = SimplifiedSyncLog(extension_index_tree=extension_tree, index_tree=child_tree, dependent_case_ids_on_phone=set([host_id, extension_id]), closed_cases=set([host_id, extension_id]), case_ids_on_phone=set(all_ids)) for case_id in [host_id, extension_id]: sync_log.purge(case_id) self.assertTrue(host_id in sync_log.case_ids_on_phone) self.assertTrue(extension_id in sync_log.case_ids_on_phone) self.assertTrue(child_of_extension_id in sync_log.case_ids_on_phone)
def test_shared_properties_migrate(self): attrs = { "date": datetime.utcnow(), "user_id": "ned", "previous_log_id": "previous-log", "duration": 10, "owner_ids": ["arya", "sansa"], } sync_log = SyncLog(**attrs) migrated = SimplifiedSyncLog.from_other_format(sync_log) for k, v in attrs.items(): self.assertEqual(v, getattr(migrated, k))
def test_shared_properties_migrate(self): attrs = { 'date': datetime.utcnow(), 'user_id': 'ned', 'previous_log_id': 'previous-log', 'duration': 10, 'owner_ids': ['arya', 'sansa'], } sync_log = SyncLog(**attrs) migrated = SimplifiedSyncLog.from_other_format(sync_log) for k, v in attrs.items(): self.assertEqual(v, getattr(migrated, k))
def test_purge_on_migrate(self): sync_log = SyncLog( cases_on_phone=[ CaseState(case_id='robert'), CaseState(case_id='cersei'), ], dependent_cases_on_phone=[ CaseState(case_id='gendry') ] ) migrated = SimplifiedSyncLog.from_other_format(sync_log) self.assertTrue('gendry' not in migrated.case_ids_on_phone) self.assertEqual(sync_log.get_state_hash(), migrated.get_state_hash())