def get_restore_response(domain, couch_user, app_id=None, since=None, version='1.0', state=None, items=False, force_cache=False, cache_timeout=None, overwrite_cache=False, force_restore_mode=None): # not a view just a view util if not couch_user.is_commcare_user(): return HttpResponse("No linked chw found for %s" % couch_user.username, status=401) # Authentication Failure elif domain != couch_user.domain: return HttpResponse("%s was not in the domain %s" % (couch_user.username, domain), status=401) project = Domain.get_by_name(domain) app = get_app(domain, app_id) if app_id else None restore_config = RestoreConfig( project=project, user=couch_user.to_casexml_user(), params=RestoreParams( sync_log_id=since, version=version, state_hash=state, include_item_count=items, force_restore_mode=force_restore_mode, app=app, ), cache_settings=RestoreCacheSettings( force_cache=force_cache, cache_timeout=cache_timeout, overwrite_cache=overwrite_cache ), ) return restore_config.get_response()
def handle(self, *args, **options): username = options['username'] if not username: print "You need a username!" print "Usage: ./manage.py mem_profile_restore --username [email protected]" return couch_user = CommCareUser.get_by_username(username) project = couch_user.project restore_config = RestoreConfig( project=project, restore_user=couch_user.to_ota_restore_user(), params=RestoreParams( version=V2, include_item_count=True, ), cache_settings=RestoreCacheSettings( force_cache=True, cache_timeout=1, overwrite_cache=False, )) with resident_set_size(): restore_config.get_payload()
def testCacheNonInvalidation(self): original_payload = RestoreConfig( self.user, version=V2, caching_enabled=True, restore_id=self.sync_log._id, ).get_payload() self.sync_log = SyncLog.get(self.sync_log._id) self.assertTrue(self.sync_log.has_cached_payload(V2)) # posting a case associated with this sync token should invalidate the cache # submitting a case not with the token will not touch the cache for that token case_id = "cache_noninvalidation" post_case_blocks([ CaseBlock( create=True, case_id=case_id, user_id=self.user.user_id, owner_id=self.user.user_id, case_type=PARENT_TYPE, version=V2, ).as_xml() ]) next_payload = RestoreConfig( self.user, version=V2, caching_enabled=True, restore_id=self.sync_log._id, ).get_payload() self.assertEqual(original_payload, next_payload) self.assertFalse(case_id in next_payload)
def setUp(self): super(CommTrackSyncTest, self).setUp() # reused stuff self.restore_user = self.user.to_ota_restore_user() self.sp_block = CaseBlock(case_id=self.sp.case_id, ).as_xml() # bootstrap ota stuff self.ct_settings.consumption_config = ConsumptionConfig( min_transactions=0, min_window=0, optimal_window=60, ) self.ct_settings.ota_restore_config = StockRestoreConfig( section_to_consumption_types={'stock': 'consumption'}) set_default_monthly_consumption_for_domain(self.domain.name, 5) self.ota_settings = self.ct_settings.get_ota_restore_settings() # get initial restore token restore_config = RestoreConfig( project=self.domain, restore_user=self.restore_user, params=RestoreParams(version=V2), ) self.sync_log_id = synclog_id_from_restore_payload( restore_config.get_payload().as_string())
def testCacheInvalidation(self): original_payload = RestoreConfig( self.user, version=V2, caching_enabled=True, restore_id=self.sync_log._id, ).get_payload() self.sync_log = SyncLog.get(self.sync_log._id) self.assertTrue(self.sync_log.has_cached_payload(V2)) # posting a case associated with this sync token should invalidate the cache case_id = "cache_invalidation" self._createCaseStubs([case_id]) self.sync_log = SyncLog.get(self.sync_log._id) self.assertFalse(self.sync_log.has_cached_payload(V2)) # resyncing should recreate the cache next_payload = RestoreConfig( self.user, version=V2, caching_enabled=True, restore_id=self.sync_log._id, ).get_payload() self.sync_log = SyncLog.get(self.sync_log._id) self.assertTrue(self.sync_log.has_cached_payload(V2)) self.assertNotEqual(original_payload, next_payload) self.assertFalse(case_id in original_payload) self.assertTrue(case_id in next_payload)
def _get_ota_balance_blocks(project, user): restore_config = RestoreConfig( project=project, restore_user=user.to_ota_restore_user(), params=RestoreParams(version=V2), ) return extract_balance_xml(restore_config.get_payload().as_string())
def setUp(self): super(CommTrackSyncTest, self).setUp() # reused stuff self.casexml_user = self.user.to_casexml_user() self.sp_block = CaseBlock( case_id=self.sp._id, version=V2, ).as_xml() # bootstrap ota stuff self.ct_settings.consumption_config = ConsumptionConfig( min_transactions=0, min_window=0, optimal_window=60, ) self.ct_settings.ota_restore_config = StockRestoreConfig( section_to_consumption_types={'stock': 'consumption'} ) set_default_monthly_consumption_for_domain(self.domain.name, 5) self.ota_settings = self.ct_settings.get_ota_restore_settings() # get initial restore token restore_config = RestoreConfig( self.casexml_user, version=V2, stock_settings=self.ota_settings, ) self.sync_log_id = synclog_id_from_restore_payload(restore_config.get_payload())
def get_restore_response(domain, couch_user, since=None, version='1.0', state=None, items=False, force_cache=False, cache_timeout=None, overwrite_cache=False, force_restore_mode=None): # not a view just a view util if not couch_user.is_commcare_user(): return HttpResponse("No linked chw found for %s" % couch_user.username, status=401) # Authentication Failure elif domain != couch_user.domain: return HttpResponse("%s was not in the domain %s" % (couch_user.username, domain), status=401) project = Domain.get_by_name(domain) restore_config = RestoreConfig( project=project, user=couch_user.to_casexml_user(), params=RestoreParams( sync_log_id=since, version=version, state_hash=state, include_item_count=items, force_restore_mode=force_restore_mode, ), cache_settings=RestoreCacheSettings( force_cache=force_cache, cache_timeout=cache_timeout, overwrite_cache=overwrite_cache ), ) return restore_config.get_response()
def handle(self, *args, **options): username = options['username'] if not username: print "You need a username!" print "Usage: ./manage.py mem_profile_restore --username [email protected]" return couch_user = CommCareUser.get_by_username(username) project = couch_user.project restore_config = RestoreConfig( project=project, restore_user=couch_user.to_ota_restore_user(), params=RestoreParams( version=V2, include_item_count=True, ), cache_settings=RestoreCacheSettings( force_cache=True, cache_timeout=1, overwrite_cache=False, ) ) with resident_set_size(): restore_config.get_payload()
def generate_restore_payload(project, user, restore_id="", version=V1, state_hash="", items=False, overwrite_cache=False, force_cache=False): """ Gets an XML payload suitable for OTA restore. user: who the payload is for restore_id: last sync token for this user version: the restore API version returns: the xml payload of the sync operation """ config = RestoreConfig( project=project, user=user, params=RestoreParams( sync_log_id=restore_id, version=version, state_hash=state_hash, include_item_count=items ), cache_settings=RestoreCacheSettings( overwrite_cache=overwrite_cache, force_cache=force_cache, ) ) return config.get_payload().as_string()
def test_simple_factor(self): self.user.loadtest_factor = 3 self.user.save() case1 = self.factory.create_case(case_name='case1') case2 = self.factory.create_case(case_name='case2') restore_config = RestoreConfig( project=self.domain, restore_user=self.user.to_ota_restore_user(), params=RestoreParams(version=V2), ) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(6, len(caseblocks)) self.assertEqual( 1, len( filter(lambda cb: cb.get_case_id() == case1.case_id, caseblocks))) self.assertEqual( 1, len( filter(lambda cb: cb.get_case_id() == case2.case_id, caseblocks))) self.assertEqual( 3, len(filter(lambda cb: case1.name in cb.get_case_name(), caseblocks))) self.assertEqual( 3, len(filter(lambda cb: case2.name in cb.get_case_name(), caseblocks)))
def get_restore_response(domain, couch_user, since=None, version='1.0', state=None, items=False): # not a view just a view util if not couch_user.is_commcare_user(): return HttpResponse("No linked chw found for %s" % couch_user.username, status=401) # Authentication Failure elif domain != couch_user.domain: return HttpResponse("%s was not in the domain %s" % (couch_user.username, domain), status=401) project = Domain.get_by_name(domain) commtrack_settings = project.commtrack_settings stock_settings = commtrack_settings.get_ota_restore_settings( ) if commtrack_settings else None restore_config = RestoreConfig( couch_user.to_casexml_user(), since, version, state, caching_enabled=project.ota_restore_caching, items=items, stock_settings=stock_settings, ) return restore_config.get_response()
def test_synced_during_and_after_bug_resolution_returns_200(self): during = SimplifiedSyncLog( user_id=self.restore_user.user_id, date=datetime(2016, 7, 19, 20, 0) # during bug ) during.save() after = SimplifiedSyncLog( user_id=self.restore_user.user_id, previous_log_id=during._id, date=datetime(2016, 7, 21, 19, 0) # after resolution ) after.save() restore_config = RestoreConfig( project=self.project, restore_user=self.restore_user, params=RestoreParams( sync_log_id=after._id, version="2.0", ), cache_settings=RestoreCacheSettings() ) response = restore_config.get_response() self.assertEqual(response.status_code, 200)
def check_user_has_case(testcase, user, case_blocks, should_have=True, line_by_line=True, restore_id="", version=V2, purge_restore_cache=False, return_single=False): if restore_id and purge_restore_cache: SyncLog.get(restore_id).invalidate_cached_payloads() restore_config = RestoreConfig(project=user.project, restore_user=user, params=RestoreParams(restore_id, version=version)) payload_string = restore_config.get_payload().as_string() return check_payload_has_cases( testcase=testcase, payload_string=payload_string, username=user.username, case_blocks=case_blocks, should_have=should_have, line_by_line=line_by_line, version=version, return_single=return_single, restore_config=restore_config, )
def test_parent_child(self): self.user.loadtest_factor = 3 self.user.save() child, parent = self.factory.create_or_update_case( CaseStructure(attrs={'case_name': 'parent'}, relationships=[ CaseRelationship( CaseStructure(attrs={'case_name': 'child'})), ])) restore_config = RestoreConfig(project=self.domain, user=self.user, params=RestoreParams(version=V2)) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(6, len(caseblocks)) self.assertEqual( 1, len(filter(lambda cb: cb.get_case_id() == child._id, caseblocks))) self.assertEqual( 1, len(filter(lambda cb: cb.get_case_id() == parent._id, caseblocks))) self.assertEqual( 3, len(filter(lambda cb: child.name in cb.get_case_name(), caseblocks))) self.assertEqual( 3, len( filter(lambda cb: parent.name in cb.get_case_name(), caseblocks)))
def test_synced_after_bug_date_but_not_fixed(self): before = SimplifiedSyncLog( user_id=self.restore_user.user_id, date=datetime(2016, 7, 19, 18, 0) # synced before bug was introduced ) before.save() during = SimplifiedSyncLog( user_id=self.restore_user.user_id, previous_log_id=before._id, date=datetime(2016, 7, 19, 20, 0) # during bug ) during.save() after = SimplifiedSyncLog( user_id=self.restore_user.user_id, previous_log_id=during._id, date=datetime(2016, 7, 20, 19, 0) # after bug, before resolution ) after.save() restore_config = RestoreConfig(project=self.project, restore_user=self.restore_user, params=RestoreParams( sync_log_id=after._id, version="2.0", ), cache_settings=RestoreCacheSettings()) response = restore_config.get_response() self.assertEqual(response.status_code, 412)
def check_user_has_case(testcase, user, case_blocks, should_have=True, line_by_line=True, restore_id="", version=V2, purge_restore_cache=False, return_single=False): if not isinstance(case_blocks, list): case_blocks = [case_blocks] return_single = True XMLNS = NS_VERSION_MAP.get(version, 'http://openrosa.org/http/response') if restore_id and purge_restore_cache: SyncLog.get(restore_id).invalidate_cached_payloads() restore_config = RestoreConfig( project=user.project, restore_user=user, params=RestoreParams(restore_id, version=version) ) payload_string = restore_config.get_payload().as_string() blocks_from_restore = extract_caseblocks_from_xml(payload_string, version) def check_block(case_block): case_block.set('xmlns', XMLNS) case_block = RestoreCaseBlock(ElementTree.fromstring(ElementTree.tostring(case_block)), version=version) case_id = case_block.get_case_id() n = 0 def extra_info(): return "\n%s\n%s" % (case_block.to_string(), map(lambda b: b.to_string(), blocks_from_restore)) match = None for block in blocks_from_restore: if block.get_case_id() == case_id: if should_have: if line_by_line: check_xml_line_by_line( testcase, case_block.to_string(), block.to_string(), ) match = block n += 1 if n == 2: testcase.fail( "Block for case_id '%s' appears twice" " in ota restore for user '%s':%s" % (case_id, user.username, extra_info()) ) else: testcase.fail( "User '%s' gets case '%s' " "but shouldn't:%s" % (user.username, case_id, extra_info()) ) if not n and should_have: testcase.fail("Block for case_id '%s' doesn't appear in ota restore for user '%s':%s" % (case_id, user.username, extra_info())) return match matches = [check_block(case_block) for case_block in case_blocks] return restore_config, matches[0] if return_single else matches
def _get_ota_balance_blocks(ct_settings, user): ota_settings = ct_settings.get_ota_restore_settings() restore_config = RestoreConfig( user.to_casexml_user(), version=V2, stock_settings=ota_settings, ) return extract_balance_xml(restore_config.get_payload())
def test_new_then_old_sync(self): OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=True, namespace="domain") restore_config = RestoreConfig(self.project, user=self.user) sync_log = synclog_from_restore_payload(restore_config.get_payload().as_string()) self.assertEqual(SimplifiedSyncLog, type(sync_log)) OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=False, namespace="domain") restore_config = RestoreConfig(self.project, user=self.user, params=RestoreParams(sync_log_id=sync_log._id)) with self.assertRaises(IncompatibleSyncLogType): restore_config.get_payload()
def test_no_factor_set(self): self.user.loadtest_factor = None self.user.save() case = self.factory.create_case() restore_config = RestoreConfig(self.user, version=V2) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(1, len(caseblocks)) self.assertEqual(caseblocks[0].get_case_id(), case._id)
def test_legacy_support_toggle(self): restore_config = RestoreConfig(self.project, restore_user=self.user) factory = CaseFactory(domain=self.project.name, case_defaults={'owner_id': self.user_id}) # create a parent and child case (with index) from one user parent_id, child_id = [uuid.uuid4().hex for i in range(2)] factory.create_or_update_cases([ CaseStructure( case_id=child_id, attrs={'create': True}, indices=[ CaseIndex( CaseStructure(case_id=parent_id, attrs={'create': True}), relationship='child', related_type='parent', ) ], ) ]) restore_payload = restore_config.get_payload().as_string().decode( 'utf-8') self.assertTrue(child_id in restore_payload) self.assertTrue(parent_id in restore_payload) sync_log = deprecated_synclog_from_restore_payload(restore_payload) self.assertEqual(SimplifiedSyncLog, type(sync_log)) # make both cases irrelevant by changing the owner ids factory.create_or_update_cases([ CaseStructure(case_id=parent_id, attrs={'owner_id': 'different'}), CaseStructure(case_id=child_id, attrs={'owner_id': 'different'}), ], form_extras={ 'last_sync_token': sync_log._id }) # doing it again should fail since they are no longer relevant # todo: add this back in when we add the assertion back. see SimplifiedSyncLog.prune_case # with self.assertRaises(SimplifiedSyncAssertionError): # factory.create_or_update_cases([ # CaseStructure(case_id=child_id, attrs={'owner_id': 'different'}), # CaseStructure(case_id=parent_id, attrs={'owner_id': 'different'}), # ], form_extras={'last_sync_token': sync_log._id}) # enabling the toggle should prevent the failure the second time # though we also need to hackily set the request object in the threadlocals LEGACY_SYNC_SUPPORT.set(self.domain, True, namespace=NAMESPACE_DOMAIN) request = JsonObject(domain=self.domain, path='testsubmit') set_request(request) factory.create_or_update_cases([ CaseStructure(case_id=child_id, attrs={'owner_id': 'different'}), CaseStructure(case_id=parent_id, attrs={'owner_id': 'different'}), ], form_extras={ 'last_sync_token': sync_log._id })
def setUp(self): delete_all_cases() delete_all_xforms() delete_all_sync_logs() self.user = User(user_id=USER_ID, username="******", password="******", date_joined=datetime(2011, 6, 9)) # this creates the initial blank sync token in the database restore_config = RestoreConfig(self.user) self.sync_log = synclog_from_restore_payload(restore_config.get_payload())
def test_initial_cache(self): restore_config = RestoreConfig(self.user, force_cache=True) original_payload = restore_config.get_payload() self.assertNotIsInstance(original_payload, CachedResponse) restore_config = RestoreConfig(self.user) cached_payload = restore_config.get_payload() self.assertIsInstance(cached_payload, CachedResponse) self.assertEqual(original_payload.as_string(), cached_payload.as_string())
def test_old_then_new_sync(self): restore_config = RestoreConfig(self.project, user=self.user) case = CaseFactory(domain=self.project.name, case_defaults={ 'owner_id': self.user_id }).create_case() restore_payload = restore_config.get_payload().as_string() self.assertTrue(case._id in restore_payload) sync_log = synclog_from_restore_payload(restore_payload) self.assertEqual(SyncLog, type(sync_log)) restore_config = RestoreConfig( self.project, user=self.user, params=RestoreParams(sync_log_id=sync_log._id)) original_payload_back = restore_config.get_payload().as_string() self.assertFalse(case._id in original_payload_back) self.assertEqual( SyncLog, type(synclog_from_restore_payload(original_payload_back))) OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=True, namespace='domain') restore_config = RestoreConfig( self.project, user=self.user, params=RestoreParams(sync_log_id=sync_log._id), cache_settings=RestoreCacheSettings(overwrite_cache=True)) migrated_payload_back = restore_config.get_payload().as_string() self.assertFalse(case._id in migrated_payload_back) self.assertEqual( SimplifiedSyncLog, type(synclog_from_restore_payload(migrated_payload_back))) OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=False, namespace='domain')
def test_legacy_support_toggle(self): restore_config = RestoreConfig(self.project, user=self.user) factory = CaseFactory(domain=self.project.name, case_defaults={"owner_id": self.user_id}) # create a parent and child case (with index) from one user parent_id, child_id = [uuid.uuid4().hex for i in range(2)] factory.create_or_update_cases( [ CaseStructure( case_id=child_id, attrs={"create": True}, indices=[ CaseIndex( CaseStructure(case_id=parent_id, attrs={"create": True}), relationship="child", related_type="parent", ) ], ) ] ) restore_payload = restore_config.get_payload().as_string() self.assertTrue(child_id in restore_payload) self.assertTrue(parent_id in restore_payload) sync_log = synclog_from_restore_payload(restore_payload) self.assertEqual(SimplifiedSyncLog, type(sync_log)) # make both cases irrelevant by changing the owner ids factory.create_or_update_cases( [ CaseStructure(case_id=parent_id, attrs={"owner_id": "different"}), CaseStructure(case_id=child_id, attrs={"owner_id": "different"}), ], form_extras={"last_sync_token": sync_log._id}, ) # doing it again should fail since they are no longer relevant # todo: add this back in when we add the assertion back. see SimplifiedSyncLog.prune_case # with self.assertRaises(SimplifiedSyncAssertionError): # factory.create_or_update_cases([ # CaseStructure(case_id=child_id, attrs={'owner_id': 'different'}), # CaseStructure(case_id=parent_id, attrs={'owner_id': 'different'}), # ], form_extras={'last_sync_token': sync_log._id}) # enabling the toggle should prevent the failure the second time # though we also need to hackily set the request object in the threadlocals LEGACY_SYNC_SUPPORT.set(self.domain, True, namespace="domain") request = JsonObject(domain=self.domain) set_request(request) factory.create_or_update_cases( [ CaseStructure(case_id=child_id, attrs={"owner_id": "different"}), CaseStructure(case_id=parent_id, attrs={"owner_id": "different"}), ], form_extras={"last_sync_token": sync_log._id}, )
def get_restore_response(domain, couch_user, app_id=None, since=None, version='1.0', state=None, items=False, force_cache=False, cache_timeout=None, overwrite_cache=False, force_restore_mode=None, as_user=None, device_id=None, has_data_cleanup_privelege=False, openrosa_version=OPENROSA_DEFAULT_VERSION): # not a view just a view util is_permitted, message = is_permitted_to_restore( domain, couch_user, as_user, has_data_cleanup_privelege, ) if not is_permitted: return HttpResponse(message, status=401), None if couch_user.is_commcare_user() and couch_user.is_demo_user: # if user is in demo-mode, return demo restore return demo_user_restore_response(couch_user), None restore_user = get_restore_user(domain, couch_user, as_user) if not restore_user: return HttpResponse('Could not find user', status=404), None project = Domain.get_by_name(domain) app = get_app(domain, app_id) if app_id else None async_restore_enabled = (toggles.ASYNC_RESTORE.enabled(domain) and LooseVersion(openrosa_version) >= LooseVersion( OPENROSA_VERSION_MAP['ASYNC_RESTORE'])) restore_config = RestoreConfig( project=project, restore_user=restore_user, params=RestoreParams( sync_log_id=since, version=version, state_hash=state, include_item_count=items, app=app, device_id=device_id, ), cache_settings=RestoreCacheSettings(force_cache=force_cache or async_restore_enabled, cache_timeout=cache_timeout, overwrite_cache=overwrite_cache), async=async_restore_enabled) return restore_config.get_response(), restore_config.timing_context
def test_no_factor_set(self): self.user.loadtest_factor = None self.user.save() case = self.factory.create_case() restore_config = RestoreConfig(project=self.domain, user=self.user, params=RestoreParams(version=V2)) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(1, len(caseblocks)) self.assertEqual(caseblocks[0].get_case_id(), case._id)
def testCacheInvalidationAfterFileDelete(self): # first request should populate the cache original_payload = RestoreConfig(self.user, force_cache=True).get_payload() self.assertNotIsInstance(original_payload, CachedResponse) # Delete cached file os.remove(original_payload.get_filename()) # resyncing should recreate the cache next_file = RestoreConfig(self.user).get_payload() self.assertNotIsInstance(next_file, CachedResponse) self.assertNotEqual(original_payload.get_filename(), next_file.get_filename())
def generate_restore_response(project, user, restore_id="", version=V1, state_hash="", items=False): config = RestoreConfig( project=project, user=user, params=RestoreParams( sync_log_id=restore_id, version=version, state_hash=state_hash, include_item_count=items ) ) return config.get_response()
def test_return_412_between_bug_dates(self): log = SimplifiedSyncLog(user_id=self.restore_user.user_id, date=datetime(2016, 7, 19, 19, 20)) log.save() restore_config = RestoreConfig(project=self.project, restore_user=self.restore_user, params=RestoreParams( sync_log_id=log._id, version="2.0", ), cache_settings=RestoreCacheSettings()) response = restore_config.get_response() self.assertEqual(response.status_code, 412)
def generate_restore_payload(user, restore_id="", version=V1, state_hash="", items=False): """ Gets an XML payload suitable for OTA restore. user: who the payload is for restore_id: last sync token for this user version: the restore API version returns: the xml payload of the sync operation """ config = RestoreConfig(user, restore_id, version, state_hash, items=items) return config.get_payload().as_string()
def test_no_factor_set(self): self.user.loadtest_factor = None self.user.save() case = self.factory.create_case() restore_config = RestoreConfig( project=self.domain, restore_user=self.user.to_ota_restore_user(), params=RestoreParams(version=V2) ) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(1, len(caseblocks)) self.assertEqual(caseblocks[0].get_case_id(), case.case_id)
def setUp(self): delete_all_cases() delete_all_xforms() delete_all_sync_logs() self.user = User(user_id=USER_ID, username="******", password="******", date_joined=datetime(2011, 6, 9)) # this creates the initial blank sync token in the database restore_config = RestoreConfig(self.user) self.sync_log = synclog_from_restore_payload( restore_config.get_payload())
def test_simple_factor(self): self.user.loadtest_factor = 3 self.user.save() case1 = self.factory.create_case(case_name='case1') case2 = self.factory.create_case(case_name='case2') restore_config = RestoreConfig(self.user, version=V2, domain=self.domain) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(6, len(caseblocks)) self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == case1._id, caseblocks))) self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == case2._id, caseblocks))) self.assertEqual(3, len(filter(lambda cb: case1.name in cb.get_case_name(), caseblocks))) self.assertEqual(3, len(filter(lambda cb: case2.name in cb.get_case_name(), caseblocks)))
def test_build_id(self): app = Application(domain=self.domain) app.save() config = RestoreConfig( project=self.project, restore_user=self.restore_user, params=RestoreParams(app=app, ), ) config.get_payload() # this generates the sync log sync_log = SyncLog.last_for_user(self.restore_user.user_id) self.assertEqual(self.restore_user.user_id, sync_log.user_id) self.assertEqual(self.restore_user.domain, sync_log.domain) self.assertEqual(app._id, sync_log.build_id) self.addCleanup(app.delete)
def get_restore_response(domain, couch_user, app_id=None, since=None, version='1.0', state=None, items=False, force_cache=False, cache_timeout=None, overwrite_cache=False, force_restore_mode=None, as_user=None, has_data_cleanup_privelege=False, openrosa_version=OPENROSA_DEFAULT_VERSION): # not a view just a view util is_permitted, message = is_permitted_to_restore( domain, couch_user, as_user, has_data_cleanup_privelege, ) if not is_permitted: return HttpResponse(message, status=401), None if couch_user.is_commcare_user() and couch_user.is_demo_user: # if user is in demo-mode, return demo restore return demo_user_restore_response(couch_user), None restore_user = get_restore_user(domain, couch_user, as_user) if not restore_user: return HttpResponse('Could not find user', status=404), None project = Domain.get_by_name(domain) app = get_app(domain, app_id) if app_id else None async_restore_enabled = ( toggles.ASYNC_RESTORE.enabled(domain) and LooseVersion(openrosa_version) >= LooseVersion(OPENROSA_VERSION_MAP['ASYNC_RESTORE']) ) restore_config = RestoreConfig( project=project, restore_user=restore_user, params=RestoreParams( sync_log_id=since, version=version, state_hash=state, include_item_count=items, app=app, ), cache_settings=RestoreCacheSettings( force_cache=force_cache or async_restore_enabled, cache_timeout=cache_timeout, overwrite_cache=overwrite_cache ), async=async_restore_enabled ) return restore_config.get_response(), restore_config.timing_context
def get_restore_response(domain, couch_user, since=None, version='1.0', state=None): # not a view just a view util if not couch_user.is_commcare_user(): return HttpResponse("No linked chw found for %s" % couch_user.username, status=401) # Authentication Failure elif domain != couch_user.domain: return HttpResponse("%s was not in the domain %s" % (couch_user.username, domain), status=401) project = Domain.get_by_name(domain) restore_config = RestoreConfig( couch_user.to_casexml_user(), since, version, state, caching_enabled=project.ota_restore_caching, ) return restore_config.get_response()
def test_build_id(self): app = Application(domain=self.domain) app.save() config = RestoreConfig( project=self.project, restore_user=self.restore_user, params=RestoreParams( app=app, ), ) config.get_payload() # this generates the sync log sync_log = SyncLog.last_for_user(self.restore_user.user_id) self.assertEqual(self.restore_user.user_id, sync_log.user_id) self.assertEqual(self.restore_user.domain, sync_log.domain) self.assertEqual(app._id, sync_log.build_id) self.addCleanup(app.delete)
def reset_demo_user_restore(commcare_user, domain): """ Updates demo restore for the demo commcare_user """ assert commcare_user.domain == domain # if there is a restore already, delete it delete_demo_restore_for_user(commcare_user) # get latest restore restore = RestoreConfig( project=Domain.get_by_name(domain), restore_user=commcare_user.to_ota_restore_user(), params=RestoreParams(version=V2), ).get_payload().as_file() demo_restore = DemoUserRestore.create(commcare_user._id, restore, domain) # Set reference to new restore try: commcare_user.demo_restore_id = demo_restore.id commcare_user.save() except ResourceConflict: # If CommCareUser.report_metadata gets updated by sync log pillow, after a restore is # generated or for any other reason, there will be a DocumentUpdate conflict commcare_user = CommCareUser.get(commcare_user.get_id) commcare_user.is_demo_user = True commcare_user.demo_restore_id = demo_restore.id commcare_user.save()
def get_restore_config(project, user, restore_id="", version=V1, state_hash="", items=False, overwrite_cache=False, force_cache=False, device_id=None, case_sync=None, app=None): from casexml.apps.phone.restore import (RestoreCacheSettings, RestoreConfig, RestoreParams) return RestoreConfig(project=project, restore_user=user, case_sync=case_sync, params=RestoreParams( sync_log_id=restore_id, version=version, state_hash=state_hash, include_item_count=items, device_id=device_id, app=app, ), cache_settings=RestoreCacheSettings( overwrite_cache=overwrite_cache, force_cache=force_cache, ))
def _cached_restore(testcase, user, restore_id="", version=V2, purge_restore_cache=False): """DEPRECATED use <MockDevice>.sync().cases""" assert not hasattr(testcase, 'restore_config'), testcase assert not hasattr(testcase, 'payload_string'), testcase if restore_id and purge_restore_cache: RestorePayloadPathCache( domain=user.domain, user_id=user.user_id, sync_log_id=restore_id, device_id=None, ).invalidate() testcase.restore_config = RestoreConfig( project=user.project, restore_user=user, params=RestoreParams(restore_id, version=version), **getattr(testcase, 'restore_options', {})) testcase.payload_string = testcase.restore_config.get_payload().as_string() try: yield finally: del testcase.restore_config, testcase.payload_string
def test_simple_factor(self): self.user.loadtest_factor = 3 self.user.save() case1 = self.factory.create_case(case_name='case1') case2 = self.factory.create_case(case_name='case2') restore_config = RestoreConfig( project=self.domain, restore_user=self.user.to_ota_restore_user(), params=RestoreParams(version=V2), ) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(6, len(caseblocks)) self.assertEqual(1, len([cb for cb in caseblocks if cb.get_case_id() == case1.case_id])) self.assertEqual(1, len([cb for cb in caseblocks if cb.get_case_id() == case2.case_id])) self.assertEqual(3, len([cb for cb in caseblocks if case1.name in cb.get_case_name()])) self.assertEqual(3, len([cb for cb in caseblocks if case2.name in cb.get_case_name()]))
def tearDown(self): delete_all_xforms() delete_all_cases() delete_all_sync_logs() restore_config = RestoreConfig(project=self.project, restore_user=self.restore_user) restore_config.cache.delete(restore_config._initial_cache_key) super(BaseOtaRestoreTest, self).tearDown()
def testCaching(self): self.assertFalse(self.sync_log.has_cached_payload(V2)) # first request should populate the cache original_payload = RestoreConfig( self.user, version=V2, caching_enabled=True, restore_id=self.sync_log._id, ).get_payload() next_sync_log = synclog_from_restore_payload(original_payload) self.sync_log = SyncLog.get(self.sync_log._id) self.assertTrue(self.sync_log.has_cached_payload(V2)) # a second request with the same config should be exactly the same cached_payload = RestoreConfig( self.user, version=V2, caching_enabled=True, restore_id=self.sync_log._id, ).get_payload() self.assertEqual(original_payload, cached_payload) # a second request without caching should be different (generate a new id) uncached_payload = RestoreConfig( self.user, version=V2, caching_enabled=False, restore_id=self.sync_log._id, ).get_payload() self.assertNotEqual(original_payload, uncached_payload) uncached_sync_log = synclog_from_restore_payload(uncached_payload) self.assertNotEqual(next_sync_log._id, uncached_sync_log._id) # caching a different version should also produce something new versioned_payload = RestoreConfig( self.user, version=V1, caching_enabled=True, restore_id=self.sync_log._id, ).get_payload() self.assertNotEqual(original_payload, versioned_payload) versioned_sync_log = synclog_from_restore_payload(versioned_payload) self.assertNotEqual(next_sync_log._id, versioned_sync_log._id)
def test_parent_child(self): self.user.loadtest_factor = 3 self.user.save() child, parent = self.factory.create_or_update_case( CaseStructure( attrs={'case_name': 'parent'}, relationships=[ CaseRelationship(CaseStructure(attrs={'case_name': 'child'})), ] ) ) restore_config = RestoreConfig(self.user, version=V2, domain=self.domain) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(6, len(caseblocks)) self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == child._id, caseblocks))) self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == parent._id, caseblocks))) self.assertEqual(3, len(filter(lambda cb: child.name in cb.get_case_name(), caseblocks))) self.assertEqual(3, len(filter(lambda cb: parent.name in cb.get_case_name(), caseblocks)))
def prime_restore(username_or_id, domain, version, cache_timeout_hours, overwrite_cache, check_cache_only): couch_user = get_user(username_or_id, domain) try: project = couch_user.project restore_config = RestoreConfig( project=project, restore_user=couch_user.to_ota_restore_user(), params=RestoreParams( version=version, include_item_count=True, ), cache_settings=RestoreCacheSettings( force_cache=True, cache_timeout=cache_timeout_hours * 60 * 60, overwrite_cache=overwrite_cache ) ) if check_cache_only: cached_payload = _get_cached_payload(restore_config) ret = u'Restore cache {} for user: {}'.format( 'EXISTS' if cached_payload else 'does not exist', couch_user.human_friendly_name, ) else: restore_config.get_payload() cached_payload = _get_cached_payload(restore_config) if cached_payload: ret = u'Restore cached successfully for user: {}'.format( couch_user.human_friendly_name, ) else: raise PrimeRestoreException(u"Restore completed by cache still empty") except Exception as e: raise PrimeRestoreException(u'Error processing user: {}. Error was: {}'.format( couch_user.human_friendly_name, str(e) )) return {"messages": ret}
def setUp(self): super(CommTrackSyncTest, self).setUp() self.group = Group(domain=util.TEST_DOMAIN, name='commtrack-folks', users=[self.user._id], case_sharing=True) self.group._id = self.sp.owner_id self.group.save() self.restore_user = self.user.to_ota_restore_user() self.sp_block = CaseBlock( case_id=self.sp.case_id, ).as_xml() # get initial restore token restore_config = RestoreConfig( project=self.domain, restore_user=self.restore_user, params=RestoreParams(version=V2), ) self.sync_log_id = synclog_id_from_restore_payload(restore_config.get_payload().as_string())
def handle(self, username, **options): couch_user = CommCareUser.get_by_username(username) project = couch_user.project restore_config = RestoreConfig( project=project, restore_user=couch_user.to_ota_restore_user(), params=RestoreParams( version=V2, include_item_count=True, ), cache_settings=RestoreCacheSettings( force_cache=True, cache_timeout=1, overwrite_cache=False, )) with resident_set_size(): restore_config.get_payload()
def setUp(self): super(CommTrackSyncTest, self).setUp() self.group = Group(domain=util.TEST_DOMAIN, name='commtrack-folks', users=[self.user._id], case_sharing=True) self.group._id = self.sp.owner_id self.group.save() self.restore_user = self.user.to_ota_restore_user() self.sp_block = CaseBlock( case_id=self.sp.case_id, ).as_xml() # get initial restore token restore_config = RestoreConfig( project=self.domain, restore_user=self.restore_user, params=RestoreParams(version=V2), ) self.sync_log_id = deprecated_synclog_id_from_restore_payload( restore_config.get_payload().as_string())
def get_restore_response(domain, couch_user, since=None, version='1.0', state=None, items=False): # not a view just a view util if not couch_user.is_commcare_user(): return HttpResponse("No linked chw found for %s" % couch_user.username, status=401) # Authentication Failure elif domain != couch_user.domain: return HttpResponse("%s was not in the domain %s" % (couch_user.username, domain), status=401) project = Domain.get_by_name(domain) commtrack_settings = project.commtrack_settings stock_settings = commtrack_settings.get_ota_restore_settings() if commtrack_settings else None restore_config = RestoreConfig( couch_user.to_casexml_user(), since, version, state, items=items, stock_settings=stock_settings, domain=project ) return restore_config.get_response()
def handle(self, username, **options): couch_user = CommCareUser.get_by_username(username) project = couch_user.project restore_config = RestoreConfig( project=project, restore_user=couch_user.to_ota_restore_user(), params=RestoreParams( version=V2, include_item_count=True, ), cache_settings=RestoreCacheSettings( force_cache=True, cache_timeout=1, overwrite_cache=False, ) ) with resident_set_size(): restore_config.get_payload()
def test_cross_domain_assignments(self): good_domain = 'main-domain' domain = create_domain(good_domain) bad_domain = 'bad-domain' create_domain(bad_domain) user = CommCareUser.create(good_domain, format_username('user', good_domain), 'secret') def _submit_case(domain): case_id = uuid.uuid4().hex case_block = CaseBlock( create=True, case_id=case_id, case_name='donald', case_type='duck', user_id=user._id, owner_id=user._id, version=V2, ).as_xml() post_case_blocks([case_block], {'domain': domain}) return CommCareCase.get(case_id) good_case = _submit_case(good_domain) # create a case in the "wrong" domain # in the future this should actually fail completely bad_case = _submit_case(bad_domain) self.assertEqual(good_domain, good_case.domain) self.assertEqual(bad_domain, bad_case.domain) for case in (good_case, bad_case): self.assertEqual(user._id, case.user_id) self.assertEqual(user._id, case.owner_id) restore_config = RestoreConfig( project=domain, user=user.to_casexml_user(), params=RestoreParams(version=V2), ) payload = restore_config.get_payload().as_string() self.assertTrue(good_case._id in payload) self.assertFalse(bad_case._id in payload)
def get_restore_response(domain, couch_user, since=None, version='1.0', state=None, items=False, force_cache=False, cache_timeout=None, overwrite_cache=False): # not a view just a view util if not couch_user.is_commcare_user(): return HttpResponse("No linked chw found for %s" % couch_user.username, status=401) # Authentication Failure elif domain != couch_user.domain: return HttpResponse("%s was not in the domain %s" % (couch_user.username, domain), status=401) project = Domain.get_by_name(domain) restore_config = RestoreConfig( couch_user.to_casexml_user(), since, version, state, items=items, domain=project, force_cache=force_cache, cache_timeout=cache_timeout, overwrite_cache=overwrite_cache ) return restore_config.get_response()
def test_parent_child(self): self.user.loadtest_factor = 3 self.user.save() child, parent = self.factory.create_or_update_case( CaseStructure( attrs={'case_name': 'parent'}, indices=[ CaseIndex(CaseStructure(attrs={'case_name': 'child'})), ] ) ) restore_config = RestoreConfig( project=self.domain, restore_user=self.user.to_ota_restore_user(), params=RestoreParams(version=V2) ) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(6, len(caseblocks)) self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == child.case_id, caseblocks))) self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == parent.case_id, caseblocks))) self.assertEqual(3, len(filter(lambda cb: child.name in cb.get_case_name(), caseblocks))) self.assertEqual(3, len(filter(lambda cb: parent.name in cb.get_case_name(), caseblocks)))
def test_old_then_new_sync(self): restore_config = RestoreConfig(self.project, user=self.user) case = CaseFactory(domain=self.project.name, case_defaults={'owner_id': self.user_id}).create_case() restore_payload = restore_config.get_payload().as_string() self.assertTrue(case._id in restore_payload) sync_log = synclog_from_restore_payload(restore_payload) self.assertEqual(SyncLog, type(sync_log)) restore_config = RestoreConfig(self.project, user=self.user, params=RestoreParams(sync_log_id=sync_log._id)) original_payload_back = restore_config.get_payload().as_string() self.assertFalse(case._id in original_payload_back) self.assertEqual(SyncLog, type(synclog_from_restore_payload(original_payload_back))) OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=True, namespace='domain') restore_config = RestoreConfig(self.project, user=self.user, params=RestoreParams(sync_log_id=sync_log._id), cache_settings=RestoreCacheSettings(overwrite_cache=True)) migrated_payload_back = restore_config.get_payload().as_string() self.assertFalse(case._id in migrated_payload_back) self.assertEqual(SimplifiedSyncLog, type(synclog_from_restore_payload(migrated_payload_back))) OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=False, namespace='domain')