def handle(self, *args, **options): username = options['username'] if not username: print "You need a username!" print "Usage: ./manage.py mem_profile_restore --username [email protected]" return couch_user = CommCareUser.get_by_username(username) project = couch_user.project restore_config = RestoreConfig( project=project, restore_user=couch_user.to_ota_restore_user(), params=RestoreParams( version=V2, include_item_count=True, ), cache_settings=RestoreCacheSettings( force_cache=True, cache_timeout=1, overwrite_cache=False, ) ) with resident_set_size(): restore_config.get_payload()
def test_old_then_new_sync(self): restore_config = RestoreConfig(self.project, user=self.user) case = CaseFactory(domain=self.project.name, case_defaults={ 'owner_id': self.user_id }).create_case() restore_payload = restore_config.get_payload().as_string() self.assertTrue(case._id in restore_payload) sync_log = synclog_from_restore_payload(restore_payload) self.assertEqual(SyncLog, type(sync_log)) restore_config = RestoreConfig( self.project, user=self.user, params=RestoreParams(sync_log_id=sync_log._id)) original_payload_back = restore_config.get_payload().as_string() self.assertFalse(case._id in original_payload_back) self.assertEqual( SyncLog, type(synclog_from_restore_payload(original_payload_back))) OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=True, namespace='domain') restore_config = RestoreConfig( self.project, user=self.user, params=RestoreParams(sync_log_id=sync_log._id), cache_settings=RestoreCacheSettings(overwrite_cache=True)) migrated_payload_back = restore_config.get_payload().as_string() self.assertFalse(case._id in migrated_payload_back) self.assertEqual( SimplifiedSyncLog, type(synclog_from_restore_payload(migrated_payload_back))) OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=False, namespace='domain')
def handle(self, *args, **options): username = options['username'] if not username: print "You need a username!" print "Usage: ./manage.py mem_profile_restore --username [email protected]" return couch_user = CommCareUser.get_by_username(username) project = couch_user.project restore_config = RestoreConfig( project=project, restore_user=couch_user.to_ota_restore_user(), params=RestoreParams( version=V2, include_item_count=True, ), cache_settings=RestoreCacheSettings( force_cache=True, cache_timeout=1, overwrite_cache=False, )) with resident_set_size(): restore_config.get_payload()
def test_new_then_old_sync(self): OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=True, namespace="domain") restore_config = RestoreConfig(self.project, user=self.user) sync_log = synclog_from_restore_payload(restore_config.get_payload().as_string()) self.assertEqual(SimplifiedSyncLog, type(sync_log)) OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=False, namespace="domain") restore_config = RestoreConfig(self.project, user=self.user, params=RestoreParams(sync_log_id=sync_log._id)) with self.assertRaises(IncompatibleSyncLogType): restore_config.get_payload()
def test_initial_cache(self): restore_config = RestoreConfig(self.user, force_cache=True) original_payload = restore_config.get_payload() self.assertNotIsInstance(original_payload, CachedResponse) restore_config = RestoreConfig(self.user) cached_payload = restore_config.get_payload() self.assertIsInstance(cached_payload, CachedResponse) self.assertEqual(original_payload.as_string(), cached_payload.as_string())
def test_build_id(self): app = Application(domain=self.domain) app.save() config = RestoreConfig( project=self.project, restore_user=self.restore_user, params=RestoreParams(app=app, ), ) config.get_payload() # this generates the sync log sync_log = SyncLog.last_for_user(self.restore_user.user_id) self.assertEqual(self.restore_user.user_id, sync_log.user_id) self.assertEqual(self.restore_user.domain, sync_log.domain) self.assertEqual(app._id, sync_log.build_id) self.addCleanup(app.delete)
def setUp(self): super(CommTrackSyncTest, self).setUp() # reused stuff self.casexml_user = self.user.to_casexml_user() self.sp_block = CaseBlock( case_id=self.sp._id, version=V2, ).as_xml() # bootstrap ota stuff self.ct_settings.consumption_config = ConsumptionConfig( min_transactions=0, min_window=0, optimal_window=60, ) self.ct_settings.ota_restore_config = StockRestoreConfig( section_to_consumption_types={'stock': 'consumption'} ) set_default_monthly_consumption_for_domain(self.domain.name, 5) self.ota_settings = self.ct_settings.get_ota_restore_settings() # get initial restore token restore_config = RestoreConfig( self.casexml_user, version=V2, stock_settings=self.ota_settings, ) self.sync_log_id = synclog_id_from_restore_payload(restore_config.get_payload())
def test_build_id(self): app = Application(domain=self.domain) app.save() config = RestoreConfig( project=self.project, restore_user=self.restore_user, params=RestoreParams( app=app, ), ) config.get_payload() # this generates the sync log sync_log = SyncLog.last_for_user(self.restore_user.user_id) self.assertEqual(self.restore_user.user_id, sync_log.user_id) self.assertEqual(self.restore_user.domain, sync_log.domain) self.assertEqual(app._id, sync_log.build_id) self.addCleanup(app.delete)
def _get_ota_balance_blocks(project, user): restore_config = RestoreConfig( project=project, restore_user=user.to_ota_restore_user(), params=RestoreParams(version=V2), ) return extract_balance_xml(restore_config.get_payload().as_string())
def setUp(self): super(CommTrackSyncTest, self).setUp() # reused stuff self.restore_user = self.user.to_ota_restore_user() self.sp_block = CaseBlock(case_id=self.sp.case_id, ).as_xml() # bootstrap ota stuff self.ct_settings.consumption_config = ConsumptionConfig( min_transactions=0, min_window=0, optimal_window=60, ) self.ct_settings.ota_restore_config = StockRestoreConfig( section_to_consumption_types={'stock': 'consumption'}) set_default_monthly_consumption_for_domain(self.domain.name, 5) self.ota_settings = self.ct_settings.get_ota_restore_settings() # get initial restore token restore_config = RestoreConfig( project=self.domain, restore_user=self.restore_user, params=RestoreParams(version=V2), ) self.sync_log_id = synclog_id_from_restore_payload( restore_config.get_payload().as_string())
def _get_ota_balance_blocks(project, user): restore_config = RestoreConfig( project=project, restore_user=user.to_ota_restore_user(), params=RestoreParams(version=V2), ) return extract_balance_xml(restore_config.get_payload().as_string())
def generate_restore_payload(project, user, restore_id="", version=V1, state_hash="", items=False, overwrite_cache=False, force_cache=False): """ Gets an XML payload suitable for OTA restore. user: who the payload is for restore_id: last sync token for this user version: the restore API version returns: the xml payload of the sync operation """ config = RestoreConfig( project=project, user=user, params=RestoreParams( sync_log_id=restore_id, version=version, state_hash=state_hash, include_item_count=items ), cache_settings=RestoreCacheSettings( overwrite_cache=overwrite_cache, force_cache=force_cache, ) ) return config.get_payload().as_string()
def check_user_has_case(testcase, user, case_blocks, should_have=True, line_by_line=True, restore_id="", version=V2, purge_restore_cache=False, return_single=False): if restore_id and purge_restore_cache: SyncLog.get(restore_id).invalidate_cached_payloads() restore_config = RestoreConfig(project=user.project, restore_user=user, params=RestoreParams(restore_id, version=version)) payload_string = restore_config.get_payload().as_string() return check_payload_has_cases( testcase=testcase, payload_string=payload_string, username=user.username, case_blocks=case_blocks, should_have=should_have, line_by_line=line_by_line, version=version, return_single=return_single, restore_config=restore_config, )
def test_parent_child(self): self.user.loadtest_factor = 3 self.user.save() child, parent = self.factory.create_or_update_case( CaseStructure(attrs={'case_name': 'parent'}, relationships=[ CaseRelationship( CaseStructure(attrs={'case_name': 'child'})), ])) restore_config = RestoreConfig(project=self.domain, user=self.user, params=RestoreParams(version=V2)) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(6, len(caseblocks)) self.assertEqual( 1, len(filter(lambda cb: cb.get_case_id() == child._id, caseblocks))) self.assertEqual( 1, len(filter(lambda cb: cb.get_case_id() == parent._id, caseblocks))) self.assertEqual( 3, len(filter(lambda cb: child.name in cb.get_case_name(), caseblocks))) self.assertEqual( 3, len( filter(lambda cb: parent.name in cb.get_case_name(), caseblocks)))
def test_simple_factor(self): self.user.loadtest_factor = 3 self.user.save() case1 = self.factory.create_case(case_name='case1') case2 = self.factory.create_case(case_name='case2') restore_config = RestoreConfig( project=self.domain, restore_user=self.user.to_ota_restore_user(), params=RestoreParams(version=V2), ) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(6, len(caseblocks)) self.assertEqual( 1, len( filter(lambda cb: cb.get_case_id() == case1.case_id, caseblocks))) self.assertEqual( 1, len( filter(lambda cb: cb.get_case_id() == case2.case_id, caseblocks))) self.assertEqual( 3, len(filter(lambda cb: case1.name in cb.get_case_name(), caseblocks))) self.assertEqual( 3, len(filter(lambda cb: case2.name in cb.get_case_name(), caseblocks)))
def check_user_has_case(testcase, user, case_blocks, should_have=True, line_by_line=True, restore_id="", version=V2, purge_restore_cache=False, return_single=False): if not isinstance(case_blocks, list): case_blocks = [case_blocks] return_single = True XMLNS = NS_VERSION_MAP.get(version, 'http://openrosa.org/http/response') if restore_id and purge_restore_cache: SyncLog.get(restore_id).invalidate_cached_payloads() restore_config = RestoreConfig( project=user.project, restore_user=user, params=RestoreParams(restore_id, version=version) ) payload_string = restore_config.get_payload().as_string() blocks_from_restore = extract_caseblocks_from_xml(payload_string, version) def check_block(case_block): case_block.set('xmlns', XMLNS) case_block = RestoreCaseBlock(ElementTree.fromstring(ElementTree.tostring(case_block)), version=version) case_id = case_block.get_case_id() n = 0 def extra_info(): return "\n%s\n%s" % (case_block.to_string(), map(lambda b: b.to_string(), blocks_from_restore)) match = None for block in blocks_from_restore: if block.get_case_id() == case_id: if should_have: if line_by_line: check_xml_line_by_line( testcase, case_block.to_string(), block.to_string(), ) match = block n += 1 if n == 2: testcase.fail( "Block for case_id '%s' appears twice" " in ota restore for user '%s':%s" % (case_id, user.username, extra_info()) ) else: testcase.fail( "User '%s' gets case '%s' " "but shouldn't:%s" % (user.username, case_id, extra_info()) ) if not n and should_have: testcase.fail("Block for case_id '%s' doesn't appear in ota restore for user '%s':%s" % (case_id, user.username, extra_info())) return match matches = [check_block(case_block) for case_block in case_blocks] return restore_config, matches[0] if return_single else matches
def check_user_has_case(testcase, user, case_blocks, should_have=True, line_by_line=True, restore_id="", version=V2, purge_restore_cache=False, return_single=False): if not isinstance(case_blocks, list): case_blocks = [case_blocks] return_single = True XMLNS = NS_VERSION_MAP.get(version, 'http://openrosa.org/http/response') if restore_id and purge_restore_cache: SyncLog.get(restore_id).invalidate_cached_payloads() restore_config = RestoreConfig( project=user.project, restore_user=user, params=RestoreParams(restore_id, version=version) ) payload_string = restore_config.get_payload().as_string() blocks_from_restore = extract_caseblocks_from_xml(payload_string, version) def check_block(case_block): case_block.set('xmlns', XMLNS) case_block = RestoreCaseBlock(ElementTree.fromstring(ElementTree.tostring(case_block)), version=version) case_id = case_block.get_case_id() n = 0 def extra_info(): return "\n%s\n%s" % (case_block.to_string(), map(lambda b: b.to_string(), blocks_from_restore)) match = None for block in blocks_from_restore: if block.get_case_id() == case_id: if should_have: if line_by_line: check_xml_line_by_line( testcase, case_block.to_string(), block.to_string(), ) match = block n += 1 if n == 2: testcase.fail( "Block for case_id '%s' appears twice" " in ota restore for user '%s':%s" % (case_id, user.username, extra_info()) ) else: testcase.fail( "User '%s' gets case '%s' " "but shouldn't:%s" % (user.username, case_id, extra_info()) ) if not n and should_have: testcase.fail("Block for case_id '%s' doesn't appear in ota restore for user '%s':%s" % (case_id, user.username, extra_info())) return match matches = [check_block(case_block) for case_block in case_blocks] return restore_config, matches[0] if return_single else matches
def _get_ota_balance_blocks(ct_settings, user): ota_settings = ct_settings.get_ota_restore_settings() restore_config = RestoreConfig( user.to_casexml_user(), version=V2, stock_settings=ota_settings, ) return extract_balance_xml(restore_config.get_payload())
def test_new_then_old_sync(self): OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=True, namespace='domain') restore_config = RestoreConfig(self.project, user=self.user) sync_log = synclog_from_restore_payload( restore_config.get_payload().as_string()) self.assertEqual(SimplifiedSyncLog, type(sync_log)) OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=False, namespace='domain') restore_config = RestoreConfig( self.project, user=self.user, params=RestoreParams(sync_log_id=sync_log._id)) with self.assertRaises(IncompatibleSyncLogType): restore_config.get_payload()
def _get_ota_balance_blocks(ct_settings, user): ota_settings = ct_settings.get_ota_restore_settings() restore_config = RestoreConfig( user.to_casexml_user(), version=V2, stock_settings=ota_settings, ) return extract_balance_xml(restore_config.get_payload())
def test_no_factor_set(self): self.user.loadtest_factor = None self.user.save() case = self.factory.create_case() restore_config = RestoreConfig(self.user, version=V2) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(1, len(caseblocks)) self.assertEqual(caseblocks[0].get_case_id(), case._id)
def test_legacy_support_toggle(self): restore_config = RestoreConfig(self.project, restore_user=self.user) factory = CaseFactory(domain=self.project.name, case_defaults={'owner_id': self.user_id}) # create a parent and child case (with index) from one user parent_id, child_id = [uuid.uuid4().hex for i in range(2)] factory.create_or_update_cases([ CaseStructure( case_id=child_id, attrs={'create': True}, indices=[ CaseIndex( CaseStructure(case_id=parent_id, attrs={'create': True}), relationship='child', related_type='parent', ) ], ) ]) restore_payload = restore_config.get_payload().as_string().decode( 'utf-8') self.assertTrue(child_id in restore_payload) self.assertTrue(parent_id in restore_payload) sync_log = deprecated_synclog_from_restore_payload(restore_payload) self.assertEqual(SimplifiedSyncLog, type(sync_log)) # make both cases irrelevant by changing the owner ids factory.create_or_update_cases([ CaseStructure(case_id=parent_id, attrs={'owner_id': 'different'}), CaseStructure(case_id=child_id, attrs={'owner_id': 'different'}), ], form_extras={ 'last_sync_token': sync_log._id }) # doing it again should fail since they are no longer relevant # todo: add this back in when we add the assertion back. see SimplifiedSyncLog.prune_case # with self.assertRaises(SimplifiedSyncAssertionError): # factory.create_or_update_cases([ # CaseStructure(case_id=child_id, attrs={'owner_id': 'different'}), # CaseStructure(case_id=parent_id, attrs={'owner_id': 'different'}), # ], form_extras={'last_sync_token': sync_log._id}) # enabling the toggle should prevent the failure the second time # though we also need to hackily set the request object in the threadlocals LEGACY_SYNC_SUPPORT.set(self.domain, True, namespace=NAMESPACE_DOMAIN) request = JsonObject(domain=self.domain, path='testsubmit') set_request(request) factory.create_or_update_cases([ CaseStructure(case_id=child_id, attrs={'owner_id': 'different'}), CaseStructure(case_id=parent_id, attrs={'owner_id': 'different'}), ], form_extras={ 'last_sync_token': sync_log._id })
def prime_restore(username_or_id, domain, version, cache_timeout_hours, overwrite_cache, check_cache_only): couch_user = get_user(username_or_id, domain) try: project = couch_user.project restore_config = RestoreConfig( project=project, restore_user=couch_user.to_ota_restore_user(), params=RestoreParams( version=version, include_item_count=True, ), cache_settings=RestoreCacheSettings( force_cache=True, cache_timeout=cache_timeout_hours * 60 * 60, overwrite_cache=overwrite_cache ) ) if check_cache_only: cached_payload = _get_cached_payload(restore_config) ret = u'Restore cache {} for user: {}'.format( 'EXISTS' if cached_payload else 'does not exist', couch_user.human_friendly_name, ) else: restore_config.get_payload() cached_payload = _get_cached_payload(restore_config) if cached_payload: ret = u'Restore cached successfully for user: {}'.format( couch_user.human_friendly_name, ) else: raise PrimeRestoreException(u"Restore completed by cache still empty") except Exception as e: raise PrimeRestoreException(u'Error processing user: {}. Error was: {}'.format( couch_user.human_friendly_name, str(e) )) return {"messages": ret}
def handle(self, username, **options): couch_user = CommCareUser.get_by_username(username) project = couch_user.project restore_config = RestoreConfig( project=project, restore_user=couch_user.to_ota_restore_user(), params=RestoreParams( version=V2, include_item_count=True, ), cache_settings=RestoreCacheSettings( force_cache=True, cache_timeout=1, overwrite_cache=False, )) with resident_set_size(): restore_config.get_payload()
def setUp(self): delete_all_cases() delete_all_xforms() delete_all_sync_logs() self.user = User(user_id=USER_ID, username="******", password="******", date_joined=datetime(2011, 6, 9)) # this creates the initial blank sync token in the database restore_config = RestoreConfig(self.user) self.sync_log = synclog_from_restore_payload(restore_config.get_payload())
def prime_restore(username_or_id, domain, version, cache_timeout_hours, overwrite_cache, check_cache_only): couch_user = get_user(username_or_id, domain) try: project = couch_user.project restore_config = RestoreConfig( project=project, restore_user=couch_user.to_ota_restore_user(), params=RestoreParams( version=version, include_item_count=True, ), cache_settings=RestoreCacheSettings( force_cache=True, cache_timeout=cache_timeout_hours * 60 * 60, overwrite_cache=overwrite_cache ) ) if check_cache_only: cached_payload = _get_cached_payload(restore_config) ret = u'Restore cache {} for user: {}'.format( 'EXISTS' if cached_payload else 'does not exist', couch_user.human_friendly_name, ) else: restore_config.get_payload() cached_payload = _get_cached_payload(restore_config) if cached_payload: ret = u'Restore cached successfully for user: {}'.format( couch_user.human_friendly_name, ) else: raise PrimeRestoreException(u"Restore completed by cache still empty") except Exception as e: raise PrimeRestoreException(u'Error processing user: {}. Error was: {}'.format( couch_user.human_friendly_name, str(e) )) return {"messages": ret}
def test_legacy_support_toggle(self): restore_config = RestoreConfig(self.project, user=self.user) factory = CaseFactory(domain=self.project.name, case_defaults={"owner_id": self.user_id}) # create a parent and child case (with index) from one user parent_id, child_id = [uuid.uuid4().hex for i in range(2)] factory.create_or_update_cases( [ CaseStructure( case_id=child_id, attrs={"create": True}, indices=[ CaseIndex( CaseStructure(case_id=parent_id, attrs={"create": True}), relationship="child", related_type="parent", ) ], ) ] ) restore_payload = restore_config.get_payload().as_string() self.assertTrue(child_id in restore_payload) self.assertTrue(parent_id in restore_payload) sync_log = synclog_from_restore_payload(restore_payload) self.assertEqual(SimplifiedSyncLog, type(sync_log)) # make both cases irrelevant by changing the owner ids factory.create_or_update_cases( [ CaseStructure(case_id=parent_id, attrs={"owner_id": "different"}), CaseStructure(case_id=child_id, attrs={"owner_id": "different"}), ], form_extras={"last_sync_token": sync_log._id}, ) # doing it again should fail since they are no longer relevant # todo: add this back in when we add the assertion back. see SimplifiedSyncLog.prune_case # with self.assertRaises(SimplifiedSyncAssertionError): # factory.create_or_update_cases([ # CaseStructure(case_id=child_id, attrs={'owner_id': 'different'}), # CaseStructure(case_id=parent_id, attrs={'owner_id': 'different'}), # ], form_extras={'last_sync_token': sync_log._id}) # enabling the toggle should prevent the failure the second time # though we also need to hackily set the request object in the threadlocals LEGACY_SYNC_SUPPORT.set(self.domain, True, namespace="domain") request = JsonObject(domain=self.domain) set_request(request) factory.create_or_update_cases( [ CaseStructure(case_id=child_id, attrs={"owner_id": "different"}), CaseStructure(case_id=parent_id, attrs={"owner_id": "different"}), ], form_extras={"last_sync_token": sync_log._id}, )
def test_no_factor_set(self): self.user.loadtest_factor = None self.user.save() case = self.factory.create_case() restore_config = RestoreConfig(project=self.domain, user=self.user, params=RestoreParams(version=V2)) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(1, len(caseblocks)) self.assertEqual(caseblocks[0].get_case_id(), case._id)
def handle(self, username, **options): couch_user = CommCareUser.get_by_username(username) project = couch_user.project restore_config = RestoreConfig( project=project, restore_user=couch_user.to_ota_restore_user(), params=RestoreParams( version=V2, include_item_count=True, ), cache_settings=RestoreCacheSettings( force_cache=True, cache_timeout=1, overwrite_cache=False, ) ) with resident_set_size(): restore_config.get_payload()
def test_old_then_new_sync(self): restore_config = RestoreConfig(self.project, user=self.user) case = CaseFactory(domain=self.project.name, case_defaults={'owner_id': self.user_id}).create_case() restore_payload = restore_config.get_payload().as_string() self.assertTrue(case._id in restore_payload) sync_log = synclog_from_restore_payload(restore_payload) self.assertEqual(SyncLog, type(sync_log)) restore_config = RestoreConfig(self.project, user=self.user, params=RestoreParams(sync_log_id=sync_log._id)) original_payload_back = restore_config.get_payload().as_string() self.assertFalse(case._id in original_payload_back) self.assertEqual(SyncLog, type(synclog_from_restore_payload(original_payload_back))) OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=True, namespace='domain') restore_config = RestoreConfig(self.project, user=self.user, params=RestoreParams(sync_log_id=sync_log._id), cache_settings=RestoreCacheSettings(overwrite_cache=True)) migrated_payload_back = restore_config.get_payload().as_string() self.assertFalse(case._id in migrated_payload_back) self.assertEqual(SimplifiedSyncLog, type(synclog_from_restore_payload(migrated_payload_back))) OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=False, namespace='domain')
def generate_restore_payload(user, restore_id="", version=V1, state_hash="", items=False): """ Gets an XML payload suitable for OTA restore. user: who the payload is for restore_id: last sync token for this user version: the restore API version returns: the xml payload of the sync operation """ config = RestoreConfig(user, restore_id, version, state_hash, items=items) return config.get_payload().as_string()
def test_no_factor_set(self): self.user.loadtest_factor = None self.user.save() case = self.factory.create_case() restore_config = RestoreConfig( project=self.domain, restore_user=self.user.to_ota_restore_user(), params=RestoreParams(version=V2) ) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(1, len(caseblocks)) self.assertEqual(caseblocks[0].get_case_id(), case.case_id)
def test_simple_factor(self): self.user.loadtest_factor = 3 self.user.save() case1 = self.factory.create_case(case_name='case1') case2 = self.factory.create_case(case_name='case2') restore_config = RestoreConfig(self.user, version=V2, domain=self.domain) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(6, len(caseblocks)) self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == case1._id, caseblocks))) self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == case2._id, caseblocks))) self.assertEqual(3, len(filter(lambda cb: case1.name in cb.get_case_name(), caseblocks))) self.assertEqual(3, len(filter(lambda cb: case2.name in cb.get_case_name(), caseblocks)))
def setUp(self): delete_all_cases() delete_all_xforms() delete_all_sync_logs() self.user = User(user_id=USER_ID, username="******", password="******", date_joined=datetime(2011, 6, 9)) # this creates the initial blank sync token in the database restore_config = RestoreConfig(self.user) self.sync_log = synclog_from_restore_payload( restore_config.get_payload())
def test_simple_factor(self): self.user.loadtest_factor = 3 self.user.save() case1 = self.factory.create_case(case_name='case1') case2 = self.factory.create_case(case_name='case2') restore_config = RestoreConfig( project=self.domain, restore_user=self.user.to_ota_restore_user(), params=RestoreParams(version=V2), ) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(6, len(caseblocks)) self.assertEqual(1, len([cb for cb in caseblocks if cb.get_case_id() == case1.case_id])) self.assertEqual(1, len([cb for cb in caseblocks if cb.get_case_id() == case2.case_id])) self.assertEqual(3, len([cb for cb in caseblocks if case1.name in cb.get_case_name()])) self.assertEqual(3, len([cb for cb in caseblocks if case2.name in cb.get_case_name()]))
def setUp(self): super(CommTrackSyncTest, self).setUp() self.group = Group(domain=util.TEST_DOMAIN, name='commtrack-folks', users=[self.user._id], case_sharing=True) self.group._id = self.sp.owner_id self.group.save() self.restore_user = self.user.to_ota_restore_user() self.sp_block = CaseBlock( case_id=self.sp.case_id, ).as_xml() # get initial restore token restore_config = RestoreConfig( project=self.domain, restore_user=self.restore_user, params=RestoreParams(version=V2), ) self.sync_log_id = synclog_id_from_restore_payload(restore_config.get_payload().as_string())
def test_parent_child(self): self.user.loadtest_factor = 3 self.user.save() child, parent = self.factory.create_or_update_case( CaseStructure( attrs={'case_name': 'parent'}, relationships=[ CaseRelationship(CaseStructure(attrs={'case_name': 'child'})), ] ) ) restore_config = RestoreConfig(self.user, version=V2, domain=self.domain) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(6, len(caseblocks)) self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == child._id, caseblocks))) self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == parent._id, caseblocks))) self.assertEqual(3, len(filter(lambda cb: child.name in cb.get_case_name(), caseblocks))) self.assertEqual(3, len(filter(lambda cb: parent.name in cb.get_case_name(), caseblocks)))
def test_cross_domain_assignments(self): good_domain = 'main-domain' domain = create_domain(good_domain) bad_domain = 'bad-domain' create_domain(bad_domain) user = CommCareUser.create(good_domain, format_username('user', good_domain), 'secret', None, None) def _submit_case(domain): case_id = uuid.uuid4().hex case_block = CaseBlock( create=True, case_id=case_id, case_name='donald', case_type='duck', user_id=user._id, owner_id=user._id, ).as_xml() _, [case] = post_case_blocks([case_block], {'domain': domain}) return case good_case = _submit_case(good_domain) # create a case in the "wrong" domain # in the future this should actually fail completely bad_case = _submit_case(bad_domain) self.assertEqual(good_domain, good_case.domain) self.assertEqual(bad_domain, bad_case.domain) for case in (good_case, bad_case): self.assertEqual(user._id, case.user_id) self.assertEqual(user._id, case.owner_id) restore_config = RestoreConfig( project=domain, restore_user=user.to_ota_restore_user(), params=RestoreParams(version=V2), ) payload = restore_config.get_payload().as_string().decode('utf-8') self.assertIn(good_case.case_id, payload) self.assertNotIn(bad_case.case_id, payload)
def setUp(self): super(CommTrackSyncTest, self).setUp() self.group = Group(domain=util.TEST_DOMAIN, name='commtrack-folks', users=[self.user._id], case_sharing=True) self.group._id = self.sp.owner_id self.group.save() self.restore_user = self.user.to_ota_restore_user() self.sp_block = CaseBlock( case_id=self.sp.case_id, ).as_xml() # get initial restore token restore_config = RestoreConfig( project=self.domain, restore_user=self.restore_user, params=RestoreParams(version=V2), ) self.sync_log_id = deprecated_synclog_id_from_restore_payload( restore_config.get_payload().as_string())
def test_cross_domain_assignments(self): good_domain = 'main-domain' domain = create_domain(good_domain) bad_domain = 'bad-domain' create_domain(bad_domain) user = CommCareUser.create(good_domain, format_username('user', good_domain), 'secret') def _submit_case(domain): case_id = uuid.uuid4().hex case_block = CaseBlock( create=True, case_id=case_id, case_name='donald', case_type='duck', user_id=user._id, owner_id=user._id, version=V2, ).as_xml() post_case_blocks([case_block], {'domain': domain}) return CommCareCase.get(case_id) good_case = _submit_case(good_domain) # create a case in the "wrong" domain # in the future this should actually fail completely bad_case = _submit_case(bad_domain) self.assertEqual(good_domain, good_case.domain) self.assertEqual(bad_domain, bad_case.domain) for case in (good_case, bad_case): self.assertEqual(user._id, case.user_id) self.assertEqual(user._id, case.owner_id) restore_config = RestoreConfig( project=domain, user=user.to_casexml_user(), params=RestoreParams(version=V2), ) payload = restore_config.get_payload().as_string() self.assertTrue(good_case._id in payload) self.assertFalse(bad_case._id in payload)
def testCrossDomainAssignments(self): good_domain = 'main-domain' create_domain(good_domain) bad_domain = 'bad-domain' create_domain(bad_domain) user = CommCareUser.create(good_domain, format_username('user', good_domain), 'secret') def _submit_case(domain): case_id = uuid.uuid4().hex case_block = CaseBlock( create=True, case_id=case_id, case_name='donald', case_type='duck', user_id=user._id, owner_id=user._id, version=V2, ).as_xml(format_datetime=json_format_datetime) post_case_blocks([case_block], {'domain': domain}) return CommCareCase.get(case_id) good_case = _submit_case(good_domain) # create a case in the "wrong" domain # in the future this should actually fail completely bad_case = _submit_case(bad_domain) self.assertEqual(good_domain, good_case.domain) self.assertEqual(bad_domain, bad_case.domain) for case in (good_case, bad_case): self.assertEqual(user._id, case.user_id) self.assertEqual(user._id, case.owner_id) restore_config = RestoreConfig( user.to_casexml_user(), version=V2, ) payload = restore_config.get_payload() self.assertTrue(good_case._id in payload) self.assertFalse(bad_case._id in payload)
def test_parent_child(self): self.user.loadtest_factor = 3 self.user.save() child, parent = self.factory.create_or_update_case( CaseStructure(attrs={ 'case_name': 'parent', 'create': True }, indices=[ CaseIndex( CaseStructure(attrs={ 'case_name': 'child', 'create': True })), ])) restore_config = RestoreConfig( project=self.domain, restore_user=self.user.to_ota_restore_user(), params=RestoreParams(version=V2)) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(6, len(caseblocks)) self.assertEqual( 1, len([cb for cb in caseblocks if cb.get_case_id() == child.case_id])) self.assertEqual( 1, len([ cb for cb in caseblocks if cb.get_case_id() == parent.case_id ])) self.assertEqual( 3, len([cb for cb in caseblocks if child.name in cb.get_case_name()])) self.assertEqual( 3, len([cb for cb in caseblocks if parent.name in cb.get_case_name()]))
def test_parent_child(self): self.user.loadtest_factor = 3 self.user.save() child, parent = self.factory.create_or_update_case( CaseStructure( attrs={'case_name': 'parent'}, indices=[ CaseIndex(CaseStructure(attrs={'case_name': 'child'})), ] ) ) restore_config = RestoreConfig( project=self.domain, restore_user=self.user.to_ota_restore_user(), params=RestoreParams(version=V2) ) payload_string = restore_config.get_payload().as_string() caseblocks = extract_caseblocks_from_xml(payload_string) self.assertEqual(6, len(caseblocks)) self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == child.case_id, caseblocks))) self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == parent.case_id, caseblocks))) self.assertEqual(3, len(filter(lambda cb: child.name in cb.get_case_name(), caseblocks))) self.assertEqual(3, len(filter(lambda cb: parent.name in cb.get_case_name(), caseblocks)))
def prime_restore(domain, usernames_or_ids, version=V1, cache_timeout_hours=None, overwrite_cache=False, check_cache_only=False): """ Task to generate and cache a restore payload for each user passed in. :param domain: The domain name for the users :param usernames_or_ids: List of usernames or user IDs :param version: Restore format version :param cache_timeout_hours: Hours to cache the payload :param overwrite_cache: If True overwrite any existing cache :param check_cache_only: Don't generate the payload, just check if it is already cached """ total = len(usernames_or_ids) DownloadBase.set_progress(prime_restore, 0, total) ret = {'messages': []} for i, username_or_id in enumerate(usernames_or_ids): couch_user = get_user(username_or_id, domain) if not couch_user: ret['messages'].append('WARNING: User not found: {}'.format(username_or_id)) continue elif couch_user.domain != domain: ret['messages'].append("WARNING: User '{}' not from domain '{}'".format( username_or_id, domain )) continue try: project = couch_user.project commtrack_settings = project.commtrack_settings stock_settings = commtrack_settings.get_ota_restore_settings() if commtrack_settings else None restore_config = RestoreConfig( couch_user.to_casexml_user(), None, version, None, items=True, stock_settings=stock_settings, domain=project, force_cache=True, cache_timeout=cache_timeout_hours * 60 * 60, overwrite_cache=overwrite_cache ) if check_cache_only: cached_payload = _get_cached_payload(restore_config) ret['messages'].append(u'Restore cache {} for user: {}'.format( 'EXISTS' if cached_payload else 'does not exist', couch_user.human_friendly_name, )) else: restore_config.get_payload() cached_payload = _get_cached_payload(restore_config) if cached_payload: ret['messages'].append('SUCCESS: Restore cached successfully for user: {}'.format( couch_user.human_friendly_name, )) else: ret['messages'].append('ERROR: Restore completed by cache still empty for user: {}'.format( couch_user.human_friendly_name, )) except Exception as e: ret['messages'].append('ERROR: Error processing user: {}'.format(str(e))) DownloadBase.set_progress(prime_restore, i + 1, total) return ret
def check_user_has_case(testcase, user, case_blocks, should_have=True, line_by_line=True, restore_id="", version=V1, purge_restore_cache=False, return_single=False): if not isinstance(case_blocks, list): case_blocks = [case_blocks] return_single = True XMLNS = NS_VERSION_MAP.get(version, 'http://openrosa.org/http/response') if restore_id and purge_restore_cache: SyncLog.get(restore_id).invalidate_cached_payloads() restore_config = RestoreConfig(user, restore_id, version=version) payload_string = restore_config.get_payload().as_string() payload = ElementTree.fromstring(payload_string) blocks = payload.findall('{{{0}}}case'.format(XMLNS)) def get_case_id(block): if version == V1: return block.findtext('{{{0}}}case_id'.format(XMLNS)) else: return block.get('case_id') def check_block(case_block): case_block.set('xmlns', XMLNS) case_block = ElementTree.fromstring(ElementTree.tostring(case_block)) case_id = get_case_id(case_block) n = 0 def extra_info(): return "\n%s\n%s" % (ElementTree.tostring(case_block), map(ElementTree.tostring, blocks)) match = None for block in blocks: if get_case_id(block) == case_id: if should_have: if line_by_line: check_xml_line_by_line( testcase, ElementTree.tostring(case_block), ElementTree.tostring(block) ) match = block n += 1 if n == 2: testcase.fail( "Block for case_id '%s' appears twice" " in ota restore for user '%s':%s" % (case_id, user.username, extra_info()) ) else: testcase.fail( "User '%s' gets case '%s' " "but shouldn't:%s" % (user.username, case_id, extra_info()) ) if not n and should_have: testcase.fail("Block for case_id '%s' doesn't appear in ota restore for user '%s':%s" % (case_id, user.username, extra_info())) return match matches = [check_block(case_block) for case_block in case_blocks] return restore_config, matches[0] if return_single else matches
def prime_restore(domain, usernames_or_ids, version=V1, cache_timeout_hours=None, overwrite_cache=False, check_cache_only=False): """ Task to generate and cache a restore payload for each user passed in. :param domain: The domain name for the users :param usernames_or_ids: List of usernames or user IDs :param version: Restore format version :param cache_timeout_hours: Hours to cache the payload :param overwrite_cache: If True overwrite any existing cache :param check_cache_only: Don't generate the payload, just check if it is already cached """ total = len(usernames_or_ids) DownloadBase.set_progress(prime_restore, 0, total) ret = {'messages': []} for i, username_or_id in enumerate(usernames_or_ids): couch_user = get_user(username_or_id, domain) if not couch_user: ret['messages'].append('WARNING: User not found: {}'.format(username_or_id)) continue elif couch_user.domain != domain: ret['messages'].append("WARNING: User '{}' not from domain '{}'".format( username_or_id, domain )) continue try: project = couch_user.project restore_config = RestoreConfig( project=project, user=couch_user.to_casexml_user(), params=RestoreParams( version=version, include_item_count=True, ), cache_settings=RestoreCacheSettings( force_cache=True, cache_timeout=cache_timeout_hours * 60 * 60, overwrite_cache=overwrite_cache ) ) if check_cache_only: cached_payload = _get_cached_payload(restore_config) ret['messages'].append(u'Restore cache {} for user: {}'.format( 'EXISTS' if cached_payload else 'does not exist', couch_user.human_friendly_name, )) else: restore_config.get_payload() cached_payload = _get_cached_payload(restore_config) if cached_payload: ret['messages'].append('SUCCESS: Restore cached successfully for user: {}'.format( couch_user.human_friendly_name, )) else: ret['messages'].append('ERROR: Restore completed by cache still empty for user: {}'.format( couch_user.human_friendly_name, )) except Exception as e: ret['messages'].append('ERROR: Error processing user: {}'.format(str(e))) DownloadBase.set_progress(prime_restore, i + 1, total) return ret