def handle(self, *args, **options):
username = options['username']
if not username:
print "You need a username!"
print "Usage: ./manage.py mem_profile_restore --username [email protected]"
return
couch_user = CommCareUser.get_by_username(username)
project = couch_user.project
restore_config = RestoreConfig(
project=project,
restore_user=couch_user.to_ota_restore_user(),
params=RestoreParams(
version=V2,
include_item_count=True,
),
cache_settings=RestoreCacheSettings(
force_cache=True,
cache_timeout=1,
overwrite_cache=False,
)
)
with resident_set_size():
restore_config.get_payload()
def test_old_then_new_sync(self):
restore_config = RestoreConfig(self.project, user=self.user)
case = CaseFactory(domain=self.project.name,
case_defaults={
'owner_id': self.user_id
}).create_case()
restore_payload = restore_config.get_payload().as_string()
self.assertTrue(case._id in restore_payload)
sync_log = synclog_from_restore_payload(restore_payload)
self.assertEqual(SyncLog, type(sync_log))
restore_config = RestoreConfig(
self.project,
user=self.user,
params=RestoreParams(sync_log_id=sync_log._id))
original_payload_back = restore_config.get_payload().as_string()
self.assertFalse(case._id in original_payload_back)
self.assertEqual(
SyncLog, type(synclog_from_restore_payload(original_payload_back)))
OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain,
enabled=True,
namespace='domain')
restore_config = RestoreConfig(
self.project,
user=self.user,
params=RestoreParams(sync_log_id=sync_log._id),
cache_settings=RestoreCacheSettings(overwrite_cache=True))
migrated_payload_back = restore_config.get_payload().as_string()
self.assertFalse(case._id in migrated_payload_back)
self.assertEqual(
SimplifiedSyncLog,
type(synclog_from_restore_payload(migrated_payload_back)))
OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain,
enabled=False,
namespace='domain')
def handle(self, *args, **options):
username = options['username']
if not username:
print "You need a username!"
print "Usage: ./manage.py mem_profile_restore --username [email protected]"
return
couch_user = CommCareUser.get_by_username(username)
project = couch_user.project
restore_config = RestoreConfig(
project=project,
restore_user=couch_user.to_ota_restore_user(),
params=RestoreParams(
version=V2,
include_item_count=True,
),
cache_settings=RestoreCacheSettings(
force_cache=True,
cache_timeout=1,
overwrite_cache=False,
))
with resident_set_size():
restore_config.get_payload()
def test_new_then_old_sync(self):
OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=True, namespace="domain")
restore_config = RestoreConfig(self.project, user=self.user)
sync_log = synclog_from_restore_payload(restore_config.get_payload().as_string())
self.assertEqual(SimplifiedSyncLog, type(sync_log))
OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=False, namespace="domain")
restore_config = RestoreConfig(self.project, user=self.user, params=RestoreParams(sync_log_id=sync_log._id))
with self.assertRaises(IncompatibleSyncLogType):
restore_config.get_payload()
def test_initial_cache(self):
restore_config = RestoreConfig(self.user, force_cache=True)
original_payload = restore_config.get_payload()
self.assertNotIsInstance(original_payload, CachedResponse)
restore_config = RestoreConfig(self.user)
cached_payload = restore_config.get_payload()
self.assertIsInstance(cached_payload, CachedResponse)
self.assertEqual(original_payload.as_string(), cached_payload.as_string())
def test_build_id(self):
app = Application(domain=self.domain)
app.save()
config = RestoreConfig(
project=self.project,
restore_user=self.restore_user,
params=RestoreParams(app=app, ),
)
config.get_payload() # this generates the sync log
sync_log = SyncLog.last_for_user(self.restore_user.user_id)
self.assertEqual(self.restore_user.user_id, sync_log.user_id)
self.assertEqual(self.restore_user.domain, sync_log.domain)
self.assertEqual(app._id, sync_log.build_id)
self.addCleanup(app.delete)
def setUp(self):
super(CommTrackSyncTest, self).setUp()
# reused stuff
self.casexml_user = self.user.to_casexml_user()
self.sp_block = CaseBlock(
case_id=self.sp._id,
version=V2,
).as_xml()
# bootstrap ota stuff
self.ct_settings.consumption_config = ConsumptionConfig(
min_transactions=0,
min_window=0,
optimal_window=60,
)
self.ct_settings.ota_restore_config = StockRestoreConfig(
section_to_consumption_types={'stock': 'consumption'}
)
set_default_monthly_consumption_for_domain(self.domain.name, 5)
self.ota_settings = self.ct_settings.get_ota_restore_settings()
# get initial restore token
restore_config = RestoreConfig(
self.casexml_user,
version=V2,
stock_settings=self.ota_settings,
)
self.sync_log_id = synclog_id_from_restore_payload(restore_config.get_payload())
def test_build_id(self):
app = Application(domain=self.domain)
app.save()
config = RestoreConfig(
project=self.project,
restore_user=self.restore_user,
params=RestoreParams(
app=app,
),
)
config.get_payload() # this generates the sync log
sync_log = SyncLog.last_for_user(self.restore_user.user_id)
self.assertEqual(self.restore_user.user_id, sync_log.user_id)
self.assertEqual(self.restore_user.domain, sync_log.domain)
self.assertEqual(app._id, sync_log.build_id)
self.addCleanup(app.delete)
def _get_ota_balance_blocks(project, user):
restore_config = RestoreConfig(
project=project,
restore_user=user.to_ota_restore_user(),
params=RestoreParams(version=V2),
)
return extract_balance_xml(restore_config.get_payload().as_string())
def setUp(self):
super(CommTrackSyncTest, self).setUp()
# reused stuff
self.restore_user = self.user.to_ota_restore_user()
self.sp_block = CaseBlock(case_id=self.sp.case_id, ).as_xml()
# bootstrap ota stuff
self.ct_settings.consumption_config = ConsumptionConfig(
min_transactions=0,
min_window=0,
optimal_window=60,
)
self.ct_settings.ota_restore_config = StockRestoreConfig(
section_to_consumption_types={'stock': 'consumption'})
set_default_monthly_consumption_for_domain(self.domain.name, 5)
self.ota_settings = self.ct_settings.get_ota_restore_settings()
# get initial restore token
restore_config = RestoreConfig(
project=self.domain,
restore_user=self.restore_user,
params=RestoreParams(version=V2),
)
self.sync_log_id = synclog_id_from_restore_payload(
restore_config.get_payload().as_string())
def _get_ota_balance_blocks(project, user):
restore_config = RestoreConfig(
project=project,
restore_user=user.to_ota_restore_user(),
params=RestoreParams(version=V2),
)
return extract_balance_xml(restore_config.get_payload().as_string())
def generate_restore_payload(project, user, restore_id="", version=V1, state_hash="",
items=False, overwrite_cache=False, force_cache=False):
"""
Gets an XML payload suitable for OTA restore.
user: who the payload is for
restore_id: last sync token for this user
version: the restore API version
returns: the xml payload of the sync operation
"""
config = RestoreConfig(
project=project,
user=user,
params=RestoreParams(
sync_log_id=restore_id,
version=version,
state_hash=state_hash,
include_item_count=items
),
cache_settings=RestoreCacheSettings(
overwrite_cache=overwrite_cache,
force_cache=force_cache,
)
)
return config.get_payload().as_string()
def check_user_has_case(testcase,
user,
case_blocks,
should_have=True,
line_by_line=True,
restore_id="",
version=V2,
purge_restore_cache=False,
return_single=False):
if restore_id and purge_restore_cache:
SyncLog.get(restore_id).invalidate_cached_payloads()
restore_config = RestoreConfig(project=user.project,
restore_user=user,
params=RestoreParams(restore_id,
version=version))
payload_string = restore_config.get_payload().as_string()
return check_payload_has_cases(
testcase=testcase,
payload_string=payload_string,
username=user.username,
case_blocks=case_blocks,
should_have=should_have,
line_by_line=line_by_line,
version=version,
return_single=return_single,
restore_config=restore_config,
)
def test_parent_child(self):
self.user.loadtest_factor = 3
self.user.save()
child, parent = self.factory.create_or_update_case(
CaseStructure(attrs={'case_name': 'parent'},
relationships=[
CaseRelationship(
CaseStructure(attrs={'case_name': 'child'})),
]))
restore_config = RestoreConfig(project=self.domain,
user=self.user,
params=RestoreParams(version=V2))
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(6, len(caseblocks))
self.assertEqual(
1, len(filter(lambda cb: cb.get_case_id() == child._id,
caseblocks)))
self.assertEqual(
1,
len(filter(lambda cb: cb.get_case_id() == parent._id, caseblocks)))
self.assertEqual(
3,
len(filter(lambda cb: child.name in cb.get_case_name(),
caseblocks)))
self.assertEqual(
3,
len(
filter(lambda cb: parent.name in cb.get_case_name(),
caseblocks)))
def test_simple_factor(self):
self.user.loadtest_factor = 3
self.user.save()
case1 = self.factory.create_case(case_name='case1')
case2 = self.factory.create_case(case_name='case2')
restore_config = RestoreConfig(
project=self.domain,
restore_user=self.user.to_ota_restore_user(),
params=RestoreParams(version=V2),
)
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(6, len(caseblocks))
self.assertEqual(
1,
len(
filter(lambda cb: cb.get_case_id() == case1.case_id,
caseblocks)))
self.assertEqual(
1,
len(
filter(lambda cb: cb.get_case_id() == case2.case_id,
caseblocks)))
self.assertEqual(
3,
len(filter(lambda cb: case1.name in cb.get_case_name(),
caseblocks)))
self.assertEqual(
3,
len(filter(lambda cb: case2.name in cb.get_case_name(),
caseblocks)))
def check_user_has_case(testcase, user, case_blocks, should_have=True,
line_by_line=True, restore_id="", version=V2,
purge_restore_cache=False, return_single=False):
if not isinstance(case_blocks, list):
case_blocks = [case_blocks]
return_single = True
XMLNS = NS_VERSION_MAP.get(version, 'http://openrosa.org/http/response')
if restore_id and purge_restore_cache:
SyncLog.get(restore_id).invalidate_cached_payloads()
restore_config = RestoreConfig(
project=user.project,
restore_user=user, params=RestoreParams(restore_id, version=version)
)
payload_string = restore_config.get_payload().as_string()
blocks_from_restore = extract_caseblocks_from_xml(payload_string, version)
def check_block(case_block):
case_block.set('xmlns', XMLNS)
case_block = RestoreCaseBlock(ElementTree.fromstring(ElementTree.tostring(case_block)), version=version)
case_id = case_block.get_case_id()
n = 0
def extra_info():
return "\n%s\n%s" % (case_block.to_string(), map(lambda b: b.to_string(), blocks_from_restore))
match = None
for block in blocks_from_restore:
if block.get_case_id() == case_id:
if should_have:
if line_by_line:
check_xml_line_by_line(
testcase,
case_block.to_string(),
block.to_string(),
)
match = block
n += 1
if n == 2:
testcase.fail(
"Block for case_id '%s' appears twice"
" in ota restore for user '%s':%s" % (case_id, user.username, extra_info())
)
else:
testcase.fail(
"User '%s' gets case '%s' "
"but shouldn't:%s" % (user.username, case_id, extra_info())
)
if not n and should_have:
testcase.fail("Block for case_id '%s' doesn't appear in ota restore for user '%s':%s"
% (case_id, user.username, extra_info()))
return match
matches = [check_block(case_block) for case_block in case_blocks]
return restore_config, matches[0] if return_single else matches
def check_user_has_case(testcase, user, case_blocks, should_have=True,
line_by_line=True, restore_id="", version=V2,
purge_restore_cache=False, return_single=False):
if not isinstance(case_blocks, list):
case_blocks = [case_blocks]
return_single = True
XMLNS = NS_VERSION_MAP.get(version, 'http://openrosa.org/http/response')
if restore_id and purge_restore_cache:
SyncLog.get(restore_id).invalidate_cached_payloads()
restore_config = RestoreConfig(
project=user.project,
restore_user=user, params=RestoreParams(restore_id, version=version)
)
payload_string = restore_config.get_payload().as_string()
blocks_from_restore = extract_caseblocks_from_xml(payload_string, version)
def check_block(case_block):
case_block.set('xmlns', XMLNS)
case_block = RestoreCaseBlock(ElementTree.fromstring(ElementTree.tostring(case_block)), version=version)
case_id = case_block.get_case_id()
n = 0
def extra_info():
return "\n%s\n%s" % (case_block.to_string(), map(lambda b: b.to_string(), blocks_from_restore))
match = None
for block in blocks_from_restore:
if block.get_case_id() == case_id:
if should_have:
if line_by_line:
check_xml_line_by_line(
testcase,
case_block.to_string(),
block.to_string(),
)
match = block
n += 1
if n == 2:
testcase.fail(
"Block for case_id '%s' appears twice"
" in ota restore for user '%s':%s" % (case_id, user.username, extra_info())
)
else:
testcase.fail(
"User '%s' gets case '%s' "
"but shouldn't:%s" % (user.username, case_id, extra_info())
)
if not n and should_have:
testcase.fail("Block for case_id '%s' doesn't appear in ota restore for user '%s':%s"
% (case_id, user.username, extra_info()))
return match
matches = [check_block(case_block) for case_block in case_blocks]
return restore_config, matches[0] if return_single else matches
def _get_ota_balance_blocks(ct_settings, user):
ota_settings = ct_settings.get_ota_restore_settings()
restore_config = RestoreConfig(
user.to_casexml_user(),
version=V2,
stock_settings=ota_settings,
)
return extract_balance_xml(restore_config.get_payload())
def test_new_then_old_sync(self):
OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain,
enabled=True,
namespace='domain')
restore_config = RestoreConfig(self.project, user=self.user)
sync_log = synclog_from_restore_payload(
restore_config.get_payload().as_string())
self.assertEqual(SimplifiedSyncLog, type(sync_log))
OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain,
enabled=False,
namespace='domain')
restore_config = RestoreConfig(
self.project,
user=self.user,
params=RestoreParams(sync_log_id=sync_log._id))
with self.assertRaises(IncompatibleSyncLogType):
restore_config.get_payload()
def _get_ota_balance_blocks(ct_settings, user):
ota_settings = ct_settings.get_ota_restore_settings()
restore_config = RestoreConfig(
user.to_casexml_user(),
version=V2,
stock_settings=ota_settings,
)
return extract_balance_xml(restore_config.get_payload())
def test_no_factor_set(self):
self.user.loadtest_factor = None
self.user.save()
case = self.factory.create_case()
restore_config = RestoreConfig(self.user, version=V2)
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(1, len(caseblocks))
self.assertEqual(caseblocks[0].get_case_id(), case._id)
def test_legacy_support_toggle(self):
restore_config = RestoreConfig(self.project, restore_user=self.user)
factory = CaseFactory(domain=self.project.name,
case_defaults={'owner_id': self.user_id})
# create a parent and child case (with index) from one user
parent_id, child_id = [uuid.uuid4().hex for i in range(2)]
factory.create_or_update_cases([
CaseStructure(
case_id=child_id,
attrs={'create': True},
indices=[
CaseIndex(
CaseStructure(case_id=parent_id,
attrs={'create': True}),
relationship='child',
related_type='parent',
)
],
)
])
restore_payload = restore_config.get_payload().as_string().decode(
'utf-8')
self.assertTrue(child_id in restore_payload)
self.assertTrue(parent_id in restore_payload)
sync_log = deprecated_synclog_from_restore_payload(restore_payload)
self.assertEqual(SimplifiedSyncLog, type(sync_log))
# make both cases irrelevant by changing the owner ids
factory.create_or_update_cases([
CaseStructure(case_id=parent_id, attrs={'owner_id': 'different'}),
CaseStructure(case_id=child_id, attrs={'owner_id': 'different'}),
],
form_extras={
'last_sync_token': sync_log._id
})
# doing it again should fail since they are no longer relevant
# todo: add this back in when we add the assertion back. see SimplifiedSyncLog.prune_case
# with self.assertRaises(SimplifiedSyncAssertionError):
# factory.create_or_update_cases([
# CaseStructure(case_id=child_id, attrs={'owner_id': 'different'}),
# CaseStructure(case_id=parent_id, attrs={'owner_id': 'different'}),
# ], form_extras={'last_sync_token': sync_log._id})
# enabling the toggle should prevent the failure the second time
# though we also need to hackily set the request object in the threadlocals
LEGACY_SYNC_SUPPORT.set(self.domain, True, namespace=NAMESPACE_DOMAIN)
request = JsonObject(domain=self.domain, path='testsubmit')
set_request(request)
factory.create_or_update_cases([
CaseStructure(case_id=child_id, attrs={'owner_id': 'different'}),
CaseStructure(case_id=parent_id, attrs={'owner_id': 'different'}),
],
form_extras={
'last_sync_token': sync_log._id
})
def prime_restore(username_or_id, domain, version, cache_timeout_hours,
overwrite_cache, check_cache_only):
couch_user = get_user(username_or_id, domain)
try:
project = couch_user.project
restore_config = RestoreConfig(
project=project,
restore_user=couch_user.to_ota_restore_user(),
params=RestoreParams(
version=version,
include_item_count=True,
),
cache_settings=RestoreCacheSettings(
force_cache=True,
cache_timeout=cache_timeout_hours * 60 * 60,
overwrite_cache=overwrite_cache
)
)
if check_cache_only:
cached_payload = _get_cached_payload(restore_config)
ret = u'Restore cache {} for user: {}'.format(
'EXISTS' if cached_payload else 'does not exist',
couch_user.human_friendly_name,
)
else:
restore_config.get_payload()
cached_payload = _get_cached_payload(restore_config)
if cached_payload:
ret = u'Restore cached successfully for user: {}'.format(
couch_user.human_friendly_name,
)
else:
raise PrimeRestoreException(u"Restore completed by cache still empty")
except Exception as e:
raise PrimeRestoreException(u'Error processing user: {}. Error was: {}'.format(
couch_user.human_friendly_name, str(e)
))
return {"messages": ret}
def handle(self, username, **options):
couch_user = CommCareUser.get_by_username(username)
project = couch_user.project
restore_config = RestoreConfig(
project=project,
restore_user=couch_user.to_ota_restore_user(),
params=RestoreParams(
version=V2,
include_item_count=True,
),
cache_settings=RestoreCacheSettings(
force_cache=True,
cache_timeout=1,
overwrite_cache=False,
))
with resident_set_size():
restore_config.get_payload()
def setUp(self):
delete_all_cases()
delete_all_xforms()
delete_all_sync_logs()
self.user = User(user_id=USER_ID, username="******",
password="******", date_joined=datetime(2011, 6, 9))
# this creates the initial blank sync token in the database
restore_config = RestoreConfig(self.user)
self.sync_log = synclog_from_restore_payload(restore_config.get_payload())
def prime_restore(username_or_id, domain, version, cache_timeout_hours,
overwrite_cache, check_cache_only):
couch_user = get_user(username_or_id, domain)
try:
project = couch_user.project
restore_config = RestoreConfig(
project=project,
restore_user=couch_user.to_ota_restore_user(),
params=RestoreParams(
version=version,
include_item_count=True,
),
cache_settings=RestoreCacheSettings(
force_cache=True,
cache_timeout=cache_timeout_hours * 60 * 60,
overwrite_cache=overwrite_cache
)
)
if check_cache_only:
cached_payload = _get_cached_payload(restore_config)
ret = u'Restore cache {} for user: {}'.format(
'EXISTS' if cached_payload else 'does not exist',
couch_user.human_friendly_name,
)
else:
restore_config.get_payload()
cached_payload = _get_cached_payload(restore_config)
if cached_payload:
ret = u'Restore cached successfully for user: {}'.format(
couch_user.human_friendly_name,
)
else:
raise PrimeRestoreException(u"Restore completed by cache still empty")
except Exception as e:
raise PrimeRestoreException(u'Error processing user: {}. Error was: {}'.format(
couch_user.human_friendly_name, str(e)
))
return {"messages": ret}
def test_legacy_support_toggle(self):
restore_config = RestoreConfig(self.project, user=self.user)
factory = CaseFactory(domain=self.project.name, case_defaults={"owner_id": self.user_id})
# create a parent and child case (with index) from one user
parent_id, child_id = [uuid.uuid4().hex for i in range(2)]
factory.create_or_update_cases(
[
CaseStructure(
case_id=child_id,
attrs={"create": True},
indices=[
CaseIndex(
CaseStructure(case_id=parent_id, attrs={"create": True}),
relationship="child",
related_type="parent",
)
],
)
]
)
restore_payload = restore_config.get_payload().as_string()
self.assertTrue(child_id in restore_payload)
self.assertTrue(parent_id in restore_payload)
sync_log = synclog_from_restore_payload(restore_payload)
self.assertEqual(SimplifiedSyncLog, type(sync_log))
# make both cases irrelevant by changing the owner ids
factory.create_or_update_cases(
[
CaseStructure(case_id=parent_id, attrs={"owner_id": "different"}),
CaseStructure(case_id=child_id, attrs={"owner_id": "different"}),
],
form_extras={"last_sync_token": sync_log._id},
)
# doing it again should fail since they are no longer relevant
# todo: add this back in when we add the assertion back. see SimplifiedSyncLog.prune_case
# with self.assertRaises(SimplifiedSyncAssertionError):
# factory.create_or_update_cases([
# CaseStructure(case_id=child_id, attrs={'owner_id': 'different'}),
# CaseStructure(case_id=parent_id, attrs={'owner_id': 'different'}),
# ], form_extras={'last_sync_token': sync_log._id})
# enabling the toggle should prevent the failure the second time
# though we also need to hackily set the request object in the threadlocals
LEGACY_SYNC_SUPPORT.set(self.domain, True, namespace="domain")
request = JsonObject(domain=self.domain)
set_request(request)
factory.create_or_update_cases(
[
CaseStructure(case_id=child_id, attrs={"owner_id": "different"}),
CaseStructure(case_id=parent_id, attrs={"owner_id": "different"}),
],
form_extras={"last_sync_token": sync_log._id},
)
def test_no_factor_set(self):
self.user.loadtest_factor = None
self.user.save()
case = self.factory.create_case()
restore_config = RestoreConfig(project=self.domain,
user=self.user,
params=RestoreParams(version=V2))
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(1, len(caseblocks))
self.assertEqual(caseblocks[0].get_case_id(), case._id)
def handle(self, username, **options):
couch_user = CommCareUser.get_by_username(username)
project = couch_user.project
restore_config = RestoreConfig(
project=project,
restore_user=couch_user.to_ota_restore_user(),
params=RestoreParams(
version=V2,
include_item_count=True,
),
cache_settings=RestoreCacheSettings(
force_cache=True,
cache_timeout=1,
overwrite_cache=False,
)
)
with resident_set_size():
restore_config.get_payload()
def test_old_then_new_sync(self):
restore_config = RestoreConfig(self.project, user=self.user)
case = CaseFactory(domain=self.project.name, case_defaults={'owner_id': self.user_id}).create_case()
restore_payload = restore_config.get_payload().as_string()
self.assertTrue(case._id in restore_payload)
sync_log = synclog_from_restore_payload(restore_payload)
self.assertEqual(SyncLog, type(sync_log))
restore_config = RestoreConfig(self.project, user=self.user,
params=RestoreParams(sync_log_id=sync_log._id))
original_payload_back = restore_config.get_payload().as_string()
self.assertFalse(case._id in original_payload_back)
self.assertEqual(SyncLog, type(synclog_from_restore_payload(original_payload_back)))
OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=True, namespace='domain')
restore_config = RestoreConfig(self.project, user=self.user,
params=RestoreParams(sync_log_id=sync_log._id),
cache_settings=RestoreCacheSettings(overwrite_cache=True))
migrated_payload_back = restore_config.get_payload().as_string()
self.assertFalse(case._id in migrated_payload_back)
self.assertEqual(SimplifiedSyncLog, type(synclog_from_restore_payload(migrated_payload_back)))
OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=False, namespace='domain')
def generate_restore_payload(user, restore_id="", version=V1, state_hash="",
items=False):
"""
Gets an XML payload suitable for OTA restore.
user: who the payload is for
restore_id: last sync token for this user
version: the restore API version
returns: the xml payload of the sync operation
"""
config = RestoreConfig(user, restore_id, version, state_hash, items=items)
return config.get_payload().as_string()
def test_no_factor_set(self):
self.user.loadtest_factor = None
self.user.save()
case = self.factory.create_case()
restore_config = RestoreConfig(
project=self.domain,
restore_user=self.user.to_ota_restore_user(),
params=RestoreParams(version=V2)
)
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(1, len(caseblocks))
self.assertEqual(caseblocks[0].get_case_id(), case.case_id)
def test_simple_factor(self):
self.user.loadtest_factor = 3
self.user.save()
case1 = self.factory.create_case(case_name='case1')
case2 = self.factory.create_case(case_name='case2')
restore_config = RestoreConfig(self.user, version=V2, domain=self.domain)
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(6, len(caseblocks))
self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == case1._id, caseblocks)))
self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == case2._id, caseblocks)))
self.assertEqual(3, len(filter(lambda cb: case1.name in cb.get_case_name(), caseblocks)))
self.assertEqual(3, len(filter(lambda cb: case2.name in cb.get_case_name(), caseblocks)))
def setUp(self):
delete_all_cases()
delete_all_xforms()
delete_all_sync_logs()
self.user = User(user_id=USER_ID,
username="******",
password="******",
date_joined=datetime(2011, 6, 9))
# this creates the initial blank sync token in the database
restore_config = RestoreConfig(self.user)
self.sync_log = synclog_from_restore_payload(
restore_config.get_payload())
def test_simple_factor(self):
self.user.loadtest_factor = 3
self.user.save()
case1 = self.factory.create_case(case_name='case1')
case2 = self.factory.create_case(case_name='case2')
restore_config = RestoreConfig(
project=self.domain,
restore_user=self.user.to_ota_restore_user(),
params=RestoreParams(version=V2),
)
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(6, len(caseblocks))
self.assertEqual(1, len([cb for cb in caseblocks if cb.get_case_id() == case1.case_id]))
self.assertEqual(1, len([cb for cb in caseblocks if cb.get_case_id() == case2.case_id]))
self.assertEqual(3, len([cb for cb in caseblocks if case1.name in cb.get_case_name()]))
self.assertEqual(3, len([cb for cb in caseblocks if case2.name in cb.get_case_name()]))
def setUp(self):
super(CommTrackSyncTest, self).setUp()
self.group = Group(domain=util.TEST_DOMAIN, name='commtrack-folks',
users=[self.user._id], case_sharing=True)
self.group._id = self.sp.owner_id
self.group.save()
self.restore_user = self.user.to_ota_restore_user()
self.sp_block = CaseBlock(
case_id=self.sp.case_id,
).as_xml()
# get initial restore token
restore_config = RestoreConfig(
project=self.domain,
restore_user=self.restore_user,
params=RestoreParams(version=V2),
)
self.sync_log_id = synclog_id_from_restore_payload(restore_config.get_payload().as_string())
def test_parent_child(self):
self.user.loadtest_factor = 3
self.user.save()
child, parent = self.factory.create_or_update_case(
CaseStructure(
attrs={'case_name': 'parent'},
relationships=[
CaseRelationship(CaseStructure(attrs={'case_name': 'child'})),
]
)
)
restore_config = RestoreConfig(self.user, version=V2, domain=self.domain)
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(6, len(caseblocks))
self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == child._id, caseblocks)))
self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == parent._id, caseblocks)))
self.assertEqual(3, len(filter(lambda cb: child.name in cb.get_case_name(), caseblocks)))
self.assertEqual(3, len(filter(lambda cb: parent.name in cb.get_case_name(), caseblocks)))
def test_cross_domain_assignments(self):
good_domain = 'main-domain'
domain = create_domain(good_domain)
bad_domain = 'bad-domain'
create_domain(bad_domain)
user = CommCareUser.create(good_domain,
format_username('user', good_domain),
'secret', None, None)
def _submit_case(domain):
case_id = uuid.uuid4().hex
case_block = CaseBlock(
create=True,
case_id=case_id,
case_name='donald',
case_type='duck',
user_id=user._id,
owner_id=user._id,
).as_xml()
_, [case] = post_case_blocks([case_block], {'domain': domain})
return case
good_case = _submit_case(good_domain)
# create a case in the "wrong" domain
# in the future this should actually fail completely
bad_case = _submit_case(bad_domain)
self.assertEqual(good_domain, good_case.domain)
self.assertEqual(bad_domain, bad_case.domain)
for case in (good_case, bad_case):
self.assertEqual(user._id, case.user_id)
self.assertEqual(user._id, case.owner_id)
restore_config = RestoreConfig(
project=domain,
restore_user=user.to_ota_restore_user(),
params=RestoreParams(version=V2),
)
payload = restore_config.get_payload().as_string().decode('utf-8')
self.assertIn(good_case.case_id, payload)
self.assertNotIn(bad_case.case_id, payload)
def setUp(self):
super(CommTrackSyncTest, self).setUp()
self.group = Group(domain=util.TEST_DOMAIN, name='commtrack-folks',
users=[self.user._id], case_sharing=True)
self.group._id = self.sp.owner_id
self.group.save()
self.restore_user = self.user.to_ota_restore_user()
self.sp_block = CaseBlock(
case_id=self.sp.case_id,
).as_xml()
# get initial restore token
restore_config = RestoreConfig(
project=self.domain,
restore_user=self.restore_user,
params=RestoreParams(version=V2),
)
self.sync_log_id = deprecated_synclog_id_from_restore_payload(
restore_config.get_payload().as_string())
def test_cross_domain_assignments(self):
good_domain = 'main-domain'
domain = create_domain(good_domain)
bad_domain = 'bad-domain'
create_domain(bad_domain)
user = CommCareUser.create(good_domain, format_username('user', good_domain), 'secret')
def _submit_case(domain):
case_id = uuid.uuid4().hex
case_block = CaseBlock(
create=True,
case_id=case_id,
case_name='donald',
case_type='duck',
user_id=user._id,
owner_id=user._id,
version=V2,
).as_xml()
post_case_blocks([case_block], {'domain': domain})
return CommCareCase.get(case_id)
good_case = _submit_case(good_domain)
# create a case in the "wrong" domain
# in the future this should actually fail completely
bad_case = _submit_case(bad_domain)
self.assertEqual(good_domain, good_case.domain)
self.assertEqual(bad_domain, bad_case.domain)
for case in (good_case, bad_case):
self.assertEqual(user._id, case.user_id)
self.assertEqual(user._id, case.owner_id)
restore_config = RestoreConfig(
project=domain,
user=user.to_casexml_user(),
params=RestoreParams(version=V2),
)
payload = restore_config.get_payload().as_string()
self.assertTrue(good_case._id in payload)
self.assertFalse(bad_case._id in payload)
def testCrossDomainAssignments(self):
good_domain = 'main-domain'
create_domain(good_domain)
bad_domain = 'bad-domain'
create_domain(bad_domain)
user = CommCareUser.create(good_domain, format_username('user', good_domain), 'secret')
def _submit_case(domain):
case_id = uuid.uuid4().hex
case_block = CaseBlock(
create=True,
case_id=case_id,
case_name='donald',
case_type='duck',
user_id=user._id,
owner_id=user._id,
version=V2,
).as_xml(format_datetime=json_format_datetime)
post_case_blocks([case_block], {'domain': domain})
return CommCareCase.get(case_id)
good_case = _submit_case(good_domain)
# create a case in the "wrong" domain
# in the future this should actually fail completely
bad_case = _submit_case(bad_domain)
self.assertEqual(good_domain, good_case.domain)
self.assertEqual(bad_domain, bad_case.domain)
for case in (good_case, bad_case):
self.assertEqual(user._id, case.user_id)
self.assertEqual(user._id, case.owner_id)
restore_config = RestoreConfig(
user.to_casexml_user(), version=V2,
)
payload = restore_config.get_payload()
self.assertTrue(good_case._id in payload)
self.assertFalse(bad_case._id in payload)
def test_parent_child(self):
self.user.loadtest_factor = 3
self.user.save()
child, parent = self.factory.create_or_update_case(
CaseStructure(attrs={
'case_name': 'parent',
'create': True
},
indices=[
CaseIndex(
CaseStructure(attrs={
'case_name': 'child',
'create': True
})),
]))
restore_config = RestoreConfig(
project=self.domain,
restore_user=self.user.to_ota_restore_user(),
params=RestoreParams(version=V2))
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(6, len(caseblocks))
self.assertEqual(
1,
len([cb for cb in caseblocks
if cb.get_case_id() == child.case_id]))
self.assertEqual(
1,
len([
cb for cb in caseblocks if cb.get_case_id() == parent.case_id
]))
self.assertEqual(
3,
len([cb for cb in caseblocks if child.name in cb.get_case_name()]))
self.assertEqual(
3,
len([cb for cb in caseblocks
if parent.name in cb.get_case_name()]))
def test_parent_child(self):
self.user.loadtest_factor = 3
self.user.save()
child, parent = self.factory.create_or_update_case(
CaseStructure(
attrs={'case_name': 'parent'},
indices=[
CaseIndex(CaseStructure(attrs={'case_name': 'child'})),
]
)
)
restore_config = RestoreConfig(
project=self.domain,
restore_user=self.user.to_ota_restore_user(),
params=RestoreParams(version=V2)
)
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(6, len(caseblocks))
self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == child.case_id, caseblocks)))
self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == parent.case_id, caseblocks)))
self.assertEqual(3, len(filter(lambda cb: child.name in cb.get_case_name(), caseblocks)))
self.assertEqual(3, len(filter(lambda cb: parent.name in cb.get_case_name(), caseblocks)))
def prime_restore(domain, usernames_or_ids, version=V1, cache_timeout_hours=None,
overwrite_cache=False, check_cache_only=False):
"""
Task to generate and cache a restore payload for each user passed in.
:param domain: The domain name for the users
:param usernames_or_ids: List of usernames or user IDs
:param version: Restore format version
:param cache_timeout_hours: Hours to cache the payload
:param overwrite_cache: If True overwrite any existing cache
:param check_cache_only: Don't generate the payload, just check if it is already cached
"""
total = len(usernames_or_ids)
DownloadBase.set_progress(prime_restore, 0, total)
ret = {'messages': []}
for i, username_or_id in enumerate(usernames_or_ids):
couch_user = get_user(username_or_id, domain)
if not couch_user:
ret['messages'].append('WARNING: User not found: {}'.format(username_or_id))
continue
elif couch_user.domain != domain:
ret['messages'].append("WARNING: User '{}' not from domain '{}'".format(
username_or_id,
domain
))
continue
try:
project = couch_user.project
commtrack_settings = project.commtrack_settings
stock_settings = commtrack_settings.get_ota_restore_settings() if commtrack_settings else None
restore_config = RestoreConfig(
couch_user.to_casexml_user(), None, version, None,
items=True,
stock_settings=stock_settings,
domain=project,
force_cache=True,
cache_timeout=cache_timeout_hours * 60 * 60,
overwrite_cache=overwrite_cache
)
if check_cache_only:
cached_payload = _get_cached_payload(restore_config)
ret['messages'].append(u'Restore cache {} for user: {}'.format(
'EXISTS' if cached_payload else 'does not exist',
couch_user.human_friendly_name,
))
else:
restore_config.get_payload()
cached_payload = _get_cached_payload(restore_config)
if cached_payload:
ret['messages'].append('SUCCESS: Restore cached successfully for user: {}'.format(
couch_user.human_friendly_name,
))
else:
ret['messages'].append('ERROR: Restore completed by cache still empty for user: {}'.format(
couch_user.human_friendly_name,
))
except Exception as e:
ret['messages'].append('ERROR: Error processing user: {}'.format(str(e)))
DownloadBase.set_progress(prime_restore, i + 1, total)
return ret
def check_user_has_case(testcase, user, case_blocks, should_have=True,
line_by_line=True, restore_id="", version=V1,
purge_restore_cache=False, return_single=False):
if not isinstance(case_blocks, list):
case_blocks = [case_blocks]
return_single = True
XMLNS = NS_VERSION_MAP.get(version, 'http://openrosa.org/http/response')
if restore_id and purge_restore_cache:
SyncLog.get(restore_id).invalidate_cached_payloads()
restore_config = RestoreConfig(user, restore_id, version=version)
payload_string = restore_config.get_payload().as_string()
payload = ElementTree.fromstring(payload_string)
blocks = payload.findall('{{{0}}}case'.format(XMLNS))
def get_case_id(block):
if version == V1:
return block.findtext('{{{0}}}case_id'.format(XMLNS))
else:
return block.get('case_id')
def check_block(case_block):
case_block.set('xmlns', XMLNS)
case_block = ElementTree.fromstring(ElementTree.tostring(case_block))
case_id = get_case_id(case_block)
n = 0
def extra_info():
return "\n%s\n%s" % (ElementTree.tostring(case_block), map(ElementTree.tostring, blocks))
match = None
for block in blocks:
if get_case_id(block) == case_id:
if should_have:
if line_by_line:
check_xml_line_by_line(
testcase,
ElementTree.tostring(case_block),
ElementTree.tostring(block)
)
match = block
n += 1
if n == 2:
testcase.fail(
"Block for case_id '%s' appears twice"
" in ota restore for user '%s':%s" % (case_id, user.username, extra_info())
)
else:
testcase.fail(
"User '%s' gets case '%s' "
"but shouldn't:%s" % (user.username, case_id, extra_info())
)
if not n and should_have:
testcase.fail("Block for case_id '%s' doesn't appear in ota restore for user '%s':%s"
% (case_id, user.username, extra_info()))
return match
matches = [check_block(case_block) for case_block in case_blocks]
return restore_config, matches[0] if return_single else matches
def prime_restore(domain, usernames_or_ids, version=V1, cache_timeout_hours=None,
overwrite_cache=False, check_cache_only=False):
"""
Task to generate and cache a restore payload for each user passed in.
:param domain: The domain name for the users
:param usernames_or_ids: List of usernames or user IDs
:param version: Restore format version
:param cache_timeout_hours: Hours to cache the payload
:param overwrite_cache: If True overwrite any existing cache
:param check_cache_only: Don't generate the payload, just check if it is already cached
"""
total = len(usernames_or_ids)
DownloadBase.set_progress(prime_restore, 0, total)
ret = {'messages': []}
for i, username_or_id in enumerate(usernames_or_ids):
couch_user = get_user(username_or_id, domain)
if not couch_user:
ret['messages'].append('WARNING: User not found: {}'.format(username_or_id))
continue
elif couch_user.domain != domain:
ret['messages'].append("WARNING: User '{}' not from domain '{}'".format(
username_or_id,
domain
))
continue
try:
project = couch_user.project
restore_config = RestoreConfig(
project=project,
user=couch_user.to_casexml_user(),
params=RestoreParams(
version=version,
include_item_count=True,
),
cache_settings=RestoreCacheSettings(
force_cache=True,
cache_timeout=cache_timeout_hours * 60 * 60,
overwrite_cache=overwrite_cache
)
)
if check_cache_only:
cached_payload = _get_cached_payload(restore_config)
ret['messages'].append(u'Restore cache {} for user: {}'.format(
'EXISTS' if cached_payload else 'does not exist',
couch_user.human_friendly_name,
))
else:
restore_config.get_payload()
cached_payload = _get_cached_payload(restore_config)
if cached_payload:
ret['messages'].append('SUCCESS: Restore cached successfully for user: {}'.format(
couch_user.human_friendly_name,
))
else:
ret['messages'].append('ERROR: Restore completed by cache still empty for user: {}'.format(
couch_user.human_friendly_name,
))
except Exception as e:
ret['messages'].append('ERROR: Error processing user: {}'.format(str(e)))
DownloadBase.set_progress(prime_restore, i + 1, total)
return ret
