def get_restore_response(domain, couch_user, app_id=None, since=None, version='1.0',
state=None, items=False, force_cache=False,
cache_timeout=None, overwrite_cache=False,
force_restore_mode=None):
# not a view just a view util
if not couch_user.is_commcare_user():
return HttpResponse("No linked chw found for %s" % couch_user.username,
status=401) # Authentication Failure
elif domain != couch_user.domain:
return HttpResponse("%s was not in the domain %s" % (couch_user.username, domain),
status=401)
project = Domain.get_by_name(domain)
app = get_app(domain, app_id) if app_id else None
restore_config = RestoreConfig(
project=project,
user=couch_user.to_casexml_user(),
params=RestoreParams(
sync_log_id=since,
version=version,
state_hash=state,
include_item_count=items,
force_restore_mode=force_restore_mode,
app=app,
),
cache_settings=RestoreCacheSettings(
force_cache=force_cache,
cache_timeout=cache_timeout,
overwrite_cache=overwrite_cache
),
)
return restore_config.get_response()
def handle(self, *args, **options):
username = options['username']
if not username:
print "You need a username!"
print "Usage: ./manage.py mem_profile_restore --username [email protected]"
return
couch_user = CommCareUser.get_by_username(username)
project = couch_user.project
restore_config = RestoreConfig(
project=project,
restore_user=couch_user.to_ota_restore_user(),
params=RestoreParams(
version=V2,
include_item_count=True,
),
cache_settings=RestoreCacheSettings(
force_cache=True,
cache_timeout=1,
overwrite_cache=False,
))
with resident_set_size():
restore_config.get_payload()
def testCacheNonInvalidation(self):
original_payload = RestoreConfig(
self.user,
version=V2,
caching_enabled=True,
restore_id=self.sync_log._id,
).get_payload()
self.sync_log = SyncLog.get(self.sync_log._id)
self.assertTrue(self.sync_log.has_cached_payload(V2))
# posting a case associated with this sync token should invalidate the cache
# submitting a case not with the token will not touch the cache for that token
case_id = "cache_noninvalidation"
post_case_blocks([
CaseBlock(
create=True,
case_id=case_id,
user_id=self.user.user_id,
owner_id=self.user.user_id,
case_type=PARENT_TYPE,
version=V2,
).as_xml()
])
next_payload = RestoreConfig(
self.user,
version=V2,
caching_enabled=True,
restore_id=self.sync_log._id,
).get_payload()
self.assertEqual(original_payload, next_payload)
self.assertFalse(case_id in next_payload)
def setUp(self):
super(CommTrackSyncTest, self).setUp()
# reused stuff
self.restore_user = self.user.to_ota_restore_user()
self.sp_block = CaseBlock(case_id=self.sp.case_id, ).as_xml()
# bootstrap ota stuff
self.ct_settings.consumption_config = ConsumptionConfig(
min_transactions=0,
min_window=0,
optimal_window=60,
)
self.ct_settings.ota_restore_config = StockRestoreConfig(
section_to_consumption_types={'stock': 'consumption'})
set_default_monthly_consumption_for_domain(self.domain.name, 5)
self.ota_settings = self.ct_settings.get_ota_restore_settings()
# get initial restore token
restore_config = RestoreConfig(
project=self.domain,
restore_user=self.restore_user,
params=RestoreParams(version=V2),
)
self.sync_log_id = synclog_id_from_restore_payload(
restore_config.get_payload().as_string())
def testCacheInvalidation(self):
original_payload = RestoreConfig(
self.user,
version=V2,
caching_enabled=True,
restore_id=self.sync_log._id,
).get_payload()
self.sync_log = SyncLog.get(self.sync_log._id)
self.assertTrue(self.sync_log.has_cached_payload(V2))
# posting a case associated with this sync token should invalidate the cache
case_id = "cache_invalidation"
self._createCaseStubs([case_id])
self.sync_log = SyncLog.get(self.sync_log._id)
self.assertFalse(self.sync_log.has_cached_payload(V2))
# resyncing should recreate the cache
next_payload = RestoreConfig(
self.user,
version=V2,
caching_enabled=True,
restore_id=self.sync_log._id,
).get_payload()
self.sync_log = SyncLog.get(self.sync_log._id)
self.assertTrue(self.sync_log.has_cached_payload(V2))
self.assertNotEqual(original_payload, next_payload)
self.assertFalse(case_id in original_payload)
self.assertTrue(case_id in next_payload)
def _get_ota_balance_blocks(project, user):
restore_config = RestoreConfig(
project=project,
restore_user=user.to_ota_restore_user(),
params=RestoreParams(version=V2),
)
return extract_balance_xml(restore_config.get_payload().as_string())
def setUp(self):
super(CommTrackSyncTest, self).setUp()
# reused stuff
self.casexml_user = self.user.to_casexml_user()
self.sp_block = CaseBlock(
case_id=self.sp._id,
version=V2,
).as_xml()
# bootstrap ota stuff
self.ct_settings.consumption_config = ConsumptionConfig(
min_transactions=0,
min_window=0,
optimal_window=60,
)
self.ct_settings.ota_restore_config = StockRestoreConfig(
section_to_consumption_types={'stock': 'consumption'}
)
set_default_monthly_consumption_for_domain(self.domain.name, 5)
self.ota_settings = self.ct_settings.get_ota_restore_settings()
# get initial restore token
restore_config = RestoreConfig(
self.casexml_user,
version=V2,
stock_settings=self.ota_settings,
)
self.sync_log_id = synclog_id_from_restore_payload(restore_config.get_payload())
def get_restore_response(domain, couch_user, since=None, version='1.0',
state=None, items=False, force_cache=False,
cache_timeout=None, overwrite_cache=False,
force_restore_mode=None):
# not a view just a view util
if not couch_user.is_commcare_user():
return HttpResponse("No linked chw found for %s" % couch_user.username,
status=401) # Authentication Failure
elif domain != couch_user.domain:
return HttpResponse("%s was not in the domain %s" % (couch_user.username, domain),
status=401)
project = Domain.get_by_name(domain)
restore_config = RestoreConfig(
project=project,
user=couch_user.to_casexml_user(),
params=RestoreParams(
sync_log_id=since,
version=version,
state_hash=state,
include_item_count=items,
force_restore_mode=force_restore_mode,
),
cache_settings=RestoreCacheSettings(
force_cache=force_cache,
cache_timeout=cache_timeout,
overwrite_cache=overwrite_cache
),
)
return restore_config.get_response()
def handle(self, *args, **options):
username = options['username']
if not username:
print "You need a username!"
print "Usage: ./manage.py mem_profile_restore --username [email protected]"
return
couch_user = CommCareUser.get_by_username(username)
project = couch_user.project
restore_config = RestoreConfig(
project=project,
restore_user=couch_user.to_ota_restore_user(),
params=RestoreParams(
version=V2,
include_item_count=True,
),
cache_settings=RestoreCacheSettings(
force_cache=True,
cache_timeout=1,
overwrite_cache=False,
)
)
with resident_set_size():
restore_config.get_payload()
def generate_restore_payload(project, user, restore_id="", version=V1, state_hash="",
items=False, overwrite_cache=False, force_cache=False):
"""
Gets an XML payload suitable for OTA restore.
user: who the payload is for
restore_id: last sync token for this user
version: the restore API version
returns: the xml payload of the sync operation
"""
config = RestoreConfig(
project=project,
user=user,
params=RestoreParams(
sync_log_id=restore_id,
version=version,
state_hash=state_hash,
include_item_count=items
),
cache_settings=RestoreCacheSettings(
overwrite_cache=overwrite_cache,
force_cache=force_cache,
)
)
return config.get_payload().as_string()
def _get_ota_balance_blocks(project, user):
restore_config = RestoreConfig(
project=project,
restore_user=user.to_ota_restore_user(),
params=RestoreParams(version=V2),
)
return extract_balance_xml(restore_config.get_payload().as_string())
def test_simple_factor(self):
self.user.loadtest_factor = 3
self.user.save()
case1 = self.factory.create_case(case_name='case1')
case2 = self.factory.create_case(case_name='case2')
restore_config = RestoreConfig(
project=self.domain,
restore_user=self.user.to_ota_restore_user(),
params=RestoreParams(version=V2),
)
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(6, len(caseblocks))
self.assertEqual(
1,
len(
filter(lambda cb: cb.get_case_id() == case1.case_id,
caseblocks)))
self.assertEqual(
1,
len(
filter(lambda cb: cb.get_case_id() == case2.case_id,
caseblocks)))
self.assertEqual(
3,
len(filter(lambda cb: case1.name in cb.get_case_name(),
caseblocks)))
self.assertEqual(
3,
len(filter(lambda cb: case2.name in cb.get_case_name(),
caseblocks)))
def get_restore_response(domain,
couch_user,
since=None,
version='1.0',
state=None,
items=False):
# not a view just a view util
if not couch_user.is_commcare_user():
return HttpResponse("No linked chw found for %s" % couch_user.username,
status=401) # Authentication Failure
elif domain != couch_user.domain:
return HttpResponse("%s was not in the domain %s" %
(couch_user.username, domain),
status=401)
project = Domain.get_by_name(domain)
commtrack_settings = project.commtrack_settings
stock_settings = commtrack_settings.get_ota_restore_settings(
) if commtrack_settings else None
restore_config = RestoreConfig(
couch_user.to_casexml_user(),
since,
version,
state,
caching_enabled=project.ota_restore_caching,
items=items,
stock_settings=stock_settings,
)
return restore_config.get_response()
def test_synced_during_and_after_bug_resolution_returns_200(self):
during = SimplifiedSyncLog(
user_id=self.restore_user.user_id,
date=datetime(2016, 7, 19, 20, 0) # during bug
)
during.save()
after = SimplifiedSyncLog(
user_id=self.restore_user.user_id,
previous_log_id=during._id,
date=datetime(2016, 7, 21, 19, 0) # after resolution
)
after.save()
restore_config = RestoreConfig(
project=self.project,
restore_user=self.restore_user,
params=RestoreParams(
sync_log_id=after._id,
version="2.0",
),
cache_settings=RestoreCacheSettings()
)
response = restore_config.get_response()
self.assertEqual(response.status_code, 200)
def check_user_has_case(testcase,
user,
case_blocks,
should_have=True,
line_by_line=True,
restore_id="",
version=V2,
purge_restore_cache=False,
return_single=False):
if restore_id and purge_restore_cache:
SyncLog.get(restore_id).invalidate_cached_payloads()
restore_config = RestoreConfig(project=user.project,
restore_user=user,
params=RestoreParams(restore_id,
version=version))
payload_string = restore_config.get_payload().as_string()
return check_payload_has_cases(
testcase=testcase,
payload_string=payload_string,
username=user.username,
case_blocks=case_blocks,
should_have=should_have,
line_by_line=line_by_line,
version=version,
return_single=return_single,
restore_config=restore_config,
)
def test_parent_child(self):
self.user.loadtest_factor = 3
self.user.save()
child, parent = self.factory.create_or_update_case(
CaseStructure(attrs={'case_name': 'parent'},
relationships=[
CaseRelationship(
CaseStructure(attrs={'case_name': 'child'})),
]))
restore_config = RestoreConfig(project=self.domain,
user=self.user,
params=RestoreParams(version=V2))
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(6, len(caseblocks))
self.assertEqual(
1, len(filter(lambda cb: cb.get_case_id() == child._id,
caseblocks)))
self.assertEqual(
1,
len(filter(lambda cb: cb.get_case_id() == parent._id, caseblocks)))
self.assertEqual(
3,
len(filter(lambda cb: child.name in cb.get_case_name(),
caseblocks)))
self.assertEqual(
3,
len(
filter(lambda cb: parent.name in cb.get_case_name(),
caseblocks)))
def test_synced_after_bug_date_but_not_fixed(self):
before = SimplifiedSyncLog(
user_id=self.restore_user.user_id,
date=datetime(2016, 7, 19, 18,
0) # synced before bug was introduced
)
before.save()
during = SimplifiedSyncLog(
user_id=self.restore_user.user_id,
previous_log_id=before._id,
date=datetime(2016, 7, 19, 20, 0) # during bug
)
during.save()
after = SimplifiedSyncLog(
user_id=self.restore_user.user_id,
previous_log_id=during._id,
date=datetime(2016, 7, 20, 19, 0) # after bug, before resolution
)
after.save()
restore_config = RestoreConfig(project=self.project,
restore_user=self.restore_user,
params=RestoreParams(
sync_log_id=after._id,
version="2.0",
),
cache_settings=RestoreCacheSettings())
response = restore_config.get_response()
self.assertEqual(response.status_code, 412)
def check_user_has_case(testcase, user, case_blocks, should_have=True,
line_by_line=True, restore_id="", version=V2,
purge_restore_cache=False, return_single=False):
if not isinstance(case_blocks, list):
case_blocks = [case_blocks]
return_single = True
XMLNS = NS_VERSION_MAP.get(version, 'http://openrosa.org/http/response')
if restore_id and purge_restore_cache:
SyncLog.get(restore_id).invalidate_cached_payloads()
restore_config = RestoreConfig(
project=user.project,
restore_user=user, params=RestoreParams(restore_id, version=version)
)
payload_string = restore_config.get_payload().as_string()
blocks_from_restore = extract_caseblocks_from_xml(payload_string, version)
def check_block(case_block):
case_block.set('xmlns', XMLNS)
case_block = RestoreCaseBlock(ElementTree.fromstring(ElementTree.tostring(case_block)), version=version)
case_id = case_block.get_case_id()
n = 0
def extra_info():
return "\n%s\n%s" % (case_block.to_string(), map(lambda b: b.to_string(), blocks_from_restore))
match = None
for block in blocks_from_restore:
if block.get_case_id() == case_id:
if should_have:
if line_by_line:
check_xml_line_by_line(
testcase,
case_block.to_string(),
block.to_string(),
)
match = block
n += 1
if n == 2:
testcase.fail(
"Block for case_id '%s' appears twice"
" in ota restore for user '%s':%s" % (case_id, user.username, extra_info())
)
else:
testcase.fail(
"User '%s' gets case '%s' "
"but shouldn't:%s" % (user.username, case_id, extra_info())
)
if not n and should_have:
testcase.fail("Block for case_id '%s' doesn't appear in ota restore for user '%s':%s"
% (case_id, user.username, extra_info()))
return match
matches = [check_block(case_block) for case_block in case_blocks]
return restore_config, matches[0] if return_single else matches
def check_user_has_case(testcase, user, case_blocks, should_have=True,
line_by_line=True, restore_id="", version=V2,
purge_restore_cache=False, return_single=False):
if not isinstance(case_blocks, list):
case_blocks = [case_blocks]
return_single = True
XMLNS = NS_VERSION_MAP.get(version, 'http://openrosa.org/http/response')
if restore_id and purge_restore_cache:
SyncLog.get(restore_id).invalidate_cached_payloads()
restore_config = RestoreConfig(
project=user.project,
restore_user=user, params=RestoreParams(restore_id, version=version)
)
payload_string = restore_config.get_payload().as_string()
blocks_from_restore = extract_caseblocks_from_xml(payload_string, version)
def check_block(case_block):
case_block.set('xmlns', XMLNS)
case_block = RestoreCaseBlock(ElementTree.fromstring(ElementTree.tostring(case_block)), version=version)
case_id = case_block.get_case_id()
n = 0
def extra_info():
return "\n%s\n%s" % (case_block.to_string(), map(lambda b: b.to_string(), blocks_from_restore))
match = None
for block in blocks_from_restore:
if block.get_case_id() == case_id:
if should_have:
if line_by_line:
check_xml_line_by_line(
testcase,
case_block.to_string(),
block.to_string(),
)
match = block
n += 1
if n == 2:
testcase.fail(
"Block for case_id '%s' appears twice"
" in ota restore for user '%s':%s" % (case_id, user.username, extra_info())
)
else:
testcase.fail(
"User '%s' gets case '%s' "
"but shouldn't:%s" % (user.username, case_id, extra_info())
)
if not n and should_have:
testcase.fail("Block for case_id '%s' doesn't appear in ota restore for user '%s':%s"
% (case_id, user.username, extra_info()))
return match
matches = [check_block(case_block) for case_block in case_blocks]
return restore_config, matches[0] if return_single else matches
def _get_ota_balance_blocks(ct_settings, user):
ota_settings = ct_settings.get_ota_restore_settings()
restore_config = RestoreConfig(
user.to_casexml_user(),
version=V2,
stock_settings=ota_settings,
)
return extract_balance_xml(restore_config.get_payload())
def _get_ota_balance_blocks(ct_settings, user):
ota_settings = ct_settings.get_ota_restore_settings()
restore_config = RestoreConfig(
user.to_casexml_user(),
version=V2,
stock_settings=ota_settings,
)
return extract_balance_xml(restore_config.get_payload())
def test_new_then_old_sync(self):
OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=True, namespace="domain")
restore_config = RestoreConfig(self.project, user=self.user)
sync_log = synclog_from_restore_payload(restore_config.get_payload().as_string())
self.assertEqual(SimplifiedSyncLog, type(sync_log))
OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=False, namespace="domain")
restore_config = RestoreConfig(self.project, user=self.user, params=RestoreParams(sync_log_id=sync_log._id))
with self.assertRaises(IncompatibleSyncLogType):
restore_config.get_payload()
def test_no_factor_set(self):
self.user.loadtest_factor = None
self.user.save()
case = self.factory.create_case()
restore_config = RestoreConfig(self.user, version=V2)
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(1, len(caseblocks))
self.assertEqual(caseblocks[0].get_case_id(), case._id)
def test_legacy_support_toggle(self):
restore_config = RestoreConfig(self.project, restore_user=self.user)
factory = CaseFactory(domain=self.project.name,
case_defaults={'owner_id': self.user_id})
# create a parent and child case (with index) from one user
parent_id, child_id = [uuid.uuid4().hex for i in range(2)]
factory.create_or_update_cases([
CaseStructure(
case_id=child_id,
attrs={'create': True},
indices=[
CaseIndex(
CaseStructure(case_id=parent_id,
attrs={'create': True}),
relationship='child',
related_type='parent',
)
],
)
])
restore_payload = restore_config.get_payload().as_string().decode(
'utf-8')
self.assertTrue(child_id in restore_payload)
self.assertTrue(parent_id in restore_payload)
sync_log = deprecated_synclog_from_restore_payload(restore_payload)
self.assertEqual(SimplifiedSyncLog, type(sync_log))
# make both cases irrelevant by changing the owner ids
factory.create_or_update_cases([
CaseStructure(case_id=parent_id, attrs={'owner_id': 'different'}),
CaseStructure(case_id=child_id, attrs={'owner_id': 'different'}),
],
form_extras={
'last_sync_token': sync_log._id
})
# doing it again should fail since they are no longer relevant
# todo: add this back in when we add the assertion back. see SimplifiedSyncLog.prune_case
# with self.assertRaises(SimplifiedSyncAssertionError):
# factory.create_or_update_cases([
# CaseStructure(case_id=child_id, attrs={'owner_id': 'different'}),
# CaseStructure(case_id=parent_id, attrs={'owner_id': 'different'}),
# ], form_extras={'last_sync_token': sync_log._id})
# enabling the toggle should prevent the failure the second time
# though we also need to hackily set the request object in the threadlocals
LEGACY_SYNC_SUPPORT.set(self.domain, True, namespace=NAMESPACE_DOMAIN)
request = JsonObject(domain=self.domain, path='testsubmit')
set_request(request)
factory.create_or_update_cases([
CaseStructure(case_id=child_id, attrs={'owner_id': 'different'}),
CaseStructure(case_id=parent_id, attrs={'owner_id': 'different'}),
],
form_extras={
'last_sync_token': sync_log._id
})
def setUp(self):
delete_all_cases()
delete_all_xforms()
delete_all_sync_logs()
self.user = User(user_id=USER_ID, username="******",
password="******", date_joined=datetime(2011, 6, 9))
# this creates the initial blank sync token in the database
restore_config = RestoreConfig(self.user)
self.sync_log = synclog_from_restore_payload(restore_config.get_payload())
def test_initial_cache(self):
restore_config = RestoreConfig(self.user, force_cache=True)
original_payload = restore_config.get_payload()
self.assertNotIsInstance(original_payload, CachedResponse)
restore_config = RestoreConfig(self.user)
cached_payload = restore_config.get_payload()
self.assertIsInstance(cached_payload, CachedResponse)
self.assertEqual(original_payload.as_string(), cached_payload.as_string())
def test_old_then_new_sync(self):
restore_config = RestoreConfig(self.project, user=self.user)
case = CaseFactory(domain=self.project.name,
case_defaults={
'owner_id': self.user_id
}).create_case()
restore_payload = restore_config.get_payload().as_string()
self.assertTrue(case._id in restore_payload)
sync_log = synclog_from_restore_payload(restore_payload)
self.assertEqual(SyncLog, type(sync_log))
restore_config = RestoreConfig(
self.project,
user=self.user,
params=RestoreParams(sync_log_id=sync_log._id))
original_payload_back = restore_config.get_payload().as_string()
self.assertFalse(case._id in original_payload_back)
self.assertEqual(
SyncLog, type(synclog_from_restore_payload(original_payload_back)))
OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain,
enabled=True,
namespace='domain')
restore_config = RestoreConfig(
self.project,
user=self.user,
params=RestoreParams(sync_log_id=sync_log._id),
cache_settings=RestoreCacheSettings(overwrite_cache=True))
migrated_payload_back = restore_config.get_payload().as_string()
self.assertFalse(case._id in migrated_payload_back)
self.assertEqual(
SimplifiedSyncLog,
type(synclog_from_restore_payload(migrated_payload_back)))
OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain,
enabled=False,
namespace='domain')
def test_legacy_support_toggle(self):
restore_config = RestoreConfig(self.project, user=self.user)
factory = CaseFactory(domain=self.project.name, case_defaults={"owner_id": self.user_id})
# create a parent and child case (with index) from one user
parent_id, child_id = [uuid.uuid4().hex for i in range(2)]
factory.create_or_update_cases(
[
CaseStructure(
case_id=child_id,
attrs={"create": True},
indices=[
CaseIndex(
CaseStructure(case_id=parent_id, attrs={"create": True}),
relationship="child",
related_type="parent",
)
],
)
]
)
restore_payload = restore_config.get_payload().as_string()
self.assertTrue(child_id in restore_payload)
self.assertTrue(parent_id in restore_payload)
sync_log = synclog_from_restore_payload(restore_payload)
self.assertEqual(SimplifiedSyncLog, type(sync_log))
# make both cases irrelevant by changing the owner ids
factory.create_or_update_cases(
[
CaseStructure(case_id=parent_id, attrs={"owner_id": "different"}),
CaseStructure(case_id=child_id, attrs={"owner_id": "different"}),
],
form_extras={"last_sync_token": sync_log._id},
)
# doing it again should fail since they are no longer relevant
# todo: add this back in when we add the assertion back. see SimplifiedSyncLog.prune_case
# with self.assertRaises(SimplifiedSyncAssertionError):
# factory.create_or_update_cases([
# CaseStructure(case_id=child_id, attrs={'owner_id': 'different'}),
# CaseStructure(case_id=parent_id, attrs={'owner_id': 'different'}),
# ], form_extras={'last_sync_token': sync_log._id})
# enabling the toggle should prevent the failure the second time
# though we also need to hackily set the request object in the threadlocals
LEGACY_SYNC_SUPPORT.set(self.domain, True, namespace="domain")
request = JsonObject(domain=self.domain)
set_request(request)
factory.create_or_update_cases(
[
CaseStructure(case_id=child_id, attrs={"owner_id": "different"}),
CaseStructure(case_id=parent_id, attrs={"owner_id": "different"}),
],
form_extras={"last_sync_token": sync_log._id},
)
def get_restore_response(domain,
couch_user,
app_id=None,
since=None,
version='1.0',
state=None,
items=False,
force_cache=False,
cache_timeout=None,
overwrite_cache=False,
force_restore_mode=None,
as_user=None,
device_id=None,
has_data_cleanup_privelege=False,
openrosa_version=OPENROSA_DEFAULT_VERSION):
# not a view just a view util
is_permitted, message = is_permitted_to_restore(
domain,
couch_user,
as_user,
has_data_cleanup_privelege,
)
if not is_permitted:
return HttpResponse(message, status=401), None
if couch_user.is_commcare_user() and couch_user.is_demo_user:
# if user is in demo-mode, return demo restore
return demo_user_restore_response(couch_user), None
restore_user = get_restore_user(domain, couch_user, as_user)
if not restore_user:
return HttpResponse('Could not find user', status=404), None
project = Domain.get_by_name(domain)
app = get_app(domain, app_id) if app_id else None
async_restore_enabled = (toggles.ASYNC_RESTORE.enabled(domain) and
LooseVersion(openrosa_version) >= LooseVersion(
OPENROSA_VERSION_MAP['ASYNC_RESTORE']))
restore_config = RestoreConfig(
project=project,
restore_user=restore_user,
params=RestoreParams(
sync_log_id=since,
version=version,
state_hash=state,
include_item_count=items,
app=app,
device_id=device_id,
),
cache_settings=RestoreCacheSettings(force_cache=force_cache
or async_restore_enabled,
cache_timeout=cache_timeout,
overwrite_cache=overwrite_cache),
async=async_restore_enabled)
return restore_config.get_response(), restore_config.timing_context
def test_no_factor_set(self):
self.user.loadtest_factor = None
self.user.save()
case = self.factory.create_case()
restore_config = RestoreConfig(project=self.domain,
user=self.user,
params=RestoreParams(version=V2))
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(1, len(caseblocks))
self.assertEqual(caseblocks[0].get_case_id(), case._id)
def testCacheInvalidationAfterFileDelete(self):
# first request should populate the cache
original_payload = RestoreConfig(self.user, force_cache=True).get_payload()
self.assertNotIsInstance(original_payload, CachedResponse)
# Delete cached file
os.remove(original_payload.get_filename())
# resyncing should recreate the cache
next_file = RestoreConfig(self.user).get_payload()
self.assertNotIsInstance(next_file, CachedResponse)
self.assertNotEqual(original_payload.get_filename(), next_file.get_filename())
def generate_restore_response(project, user, restore_id="", version=V1, state_hash="", items=False):
config = RestoreConfig(
project=project,
user=user,
params=RestoreParams(
sync_log_id=restore_id,
version=version,
state_hash=state_hash,
include_item_count=items
)
)
return config.get_response()
def test_return_412_between_bug_dates(self):
log = SimplifiedSyncLog(user_id=self.restore_user.user_id,
date=datetime(2016, 7, 19, 19, 20))
log.save()
restore_config = RestoreConfig(project=self.project,
restore_user=self.restore_user,
params=RestoreParams(
sync_log_id=log._id,
version="2.0",
),
cache_settings=RestoreCacheSettings())
response = restore_config.get_response()
self.assertEqual(response.status_code, 412)
def generate_restore_payload(user, restore_id="", version=V1, state_hash="",
items=False):
"""
Gets an XML payload suitable for OTA restore.
user: who the payload is for
restore_id: last sync token for this user
version: the restore API version
returns: the xml payload of the sync operation
"""
config = RestoreConfig(user, restore_id, version, state_hash, items=items)
return config.get_payload().as_string()
def test_no_factor_set(self):
self.user.loadtest_factor = None
self.user.save()
case = self.factory.create_case()
restore_config = RestoreConfig(
project=self.domain,
restore_user=self.user.to_ota_restore_user(),
params=RestoreParams(version=V2)
)
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(1, len(caseblocks))
self.assertEqual(caseblocks[0].get_case_id(), case.case_id)
def setUp(self):
delete_all_cases()
delete_all_xforms()
delete_all_sync_logs()
self.user = User(user_id=USER_ID,
username="******",
password="******",
date_joined=datetime(2011, 6, 9))
# this creates the initial blank sync token in the database
restore_config = RestoreConfig(self.user)
self.sync_log = synclog_from_restore_payload(
restore_config.get_payload())
def test_simple_factor(self):
self.user.loadtest_factor = 3
self.user.save()
case1 = self.factory.create_case(case_name='case1')
case2 = self.factory.create_case(case_name='case2')
restore_config = RestoreConfig(self.user, version=V2, domain=self.domain)
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(6, len(caseblocks))
self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == case1._id, caseblocks)))
self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == case2._id, caseblocks)))
self.assertEqual(3, len(filter(lambda cb: case1.name in cb.get_case_name(), caseblocks)))
self.assertEqual(3, len(filter(lambda cb: case2.name in cb.get_case_name(), caseblocks)))
def test_build_id(self):
app = Application(domain=self.domain)
app.save()
config = RestoreConfig(
project=self.project,
restore_user=self.restore_user,
params=RestoreParams(app=app, ),
)
config.get_payload() # this generates the sync log
sync_log = SyncLog.last_for_user(self.restore_user.user_id)
self.assertEqual(self.restore_user.user_id, sync_log.user_id)
self.assertEqual(self.restore_user.domain, sync_log.domain)
self.assertEqual(app._id, sync_log.build_id)
self.addCleanup(app.delete)
def get_restore_response(domain, couch_user, app_id=None, since=None, version='1.0',
state=None, items=False, force_cache=False,
cache_timeout=None, overwrite_cache=False,
force_restore_mode=None,
as_user=None,
has_data_cleanup_privelege=False,
openrosa_version=OPENROSA_DEFAULT_VERSION):
# not a view just a view util
is_permitted, message = is_permitted_to_restore(
domain,
couch_user,
as_user,
has_data_cleanup_privelege,
)
if not is_permitted:
return HttpResponse(message, status=401), None
if couch_user.is_commcare_user() and couch_user.is_demo_user:
# if user is in demo-mode, return demo restore
return demo_user_restore_response(couch_user), None
restore_user = get_restore_user(domain, couch_user, as_user)
if not restore_user:
return HttpResponse('Could not find user', status=404), None
project = Domain.get_by_name(domain)
app = get_app(domain, app_id) if app_id else None
async_restore_enabled = (
toggles.ASYNC_RESTORE.enabled(domain) and
LooseVersion(openrosa_version) >= LooseVersion(OPENROSA_VERSION_MAP['ASYNC_RESTORE'])
)
restore_config = RestoreConfig(
project=project,
restore_user=restore_user,
params=RestoreParams(
sync_log_id=since,
version=version,
state_hash=state,
include_item_count=items,
app=app,
),
cache_settings=RestoreCacheSettings(
force_cache=force_cache or async_restore_enabled,
cache_timeout=cache_timeout,
overwrite_cache=overwrite_cache
),
async=async_restore_enabled
)
return restore_config.get_response(), restore_config.timing_context
def get_restore_response(domain, couch_user, since=None, version='1.0', state=None):
# not a view just a view util
if not couch_user.is_commcare_user():
return HttpResponse("No linked chw found for %s" % couch_user.username,
status=401) # Authentication Failure
elif domain != couch_user.domain:
return HttpResponse("%s was not in the domain %s" % (couch_user.username, domain),
status=401)
project = Domain.get_by_name(domain)
restore_config = RestoreConfig(
couch_user.to_casexml_user(), since, version, state,
caching_enabled=project.ota_restore_caching,
)
return restore_config.get_response()
def test_build_id(self):
app = Application(domain=self.domain)
app.save()
config = RestoreConfig(
project=self.project,
restore_user=self.restore_user,
params=RestoreParams(
app=app,
),
)
config.get_payload() # this generates the sync log
sync_log = SyncLog.last_for_user(self.restore_user.user_id)
self.assertEqual(self.restore_user.user_id, sync_log.user_id)
self.assertEqual(self.restore_user.domain, sync_log.domain)
self.assertEqual(app._id, sync_log.build_id)
self.addCleanup(app.delete)
def reset_demo_user_restore(commcare_user, domain):
"""
Updates demo restore for the demo commcare_user
"""
assert commcare_user.domain == domain
# if there is a restore already, delete it
delete_demo_restore_for_user(commcare_user)
# get latest restore
restore = RestoreConfig(
project=Domain.get_by_name(domain),
restore_user=commcare_user.to_ota_restore_user(),
params=RestoreParams(version=V2),
).get_payload().as_file()
demo_restore = DemoUserRestore.create(commcare_user._id, restore, domain)
# Set reference to new restore
try:
commcare_user.demo_restore_id = demo_restore.id
commcare_user.save()
except ResourceConflict:
# If CommCareUser.report_metadata gets updated by sync log pillow, after a restore is
# generated or for any other reason, there will be a DocumentUpdate conflict
commcare_user = CommCareUser.get(commcare_user.get_id)
commcare_user.is_demo_user = True
commcare_user.demo_restore_id = demo_restore.id
commcare_user.save()
def get_restore_config(project,
user,
restore_id="",
version=V1,
state_hash="",
items=False,
overwrite_cache=False,
force_cache=False,
device_id=None,
case_sync=None,
app=None):
from casexml.apps.phone.restore import (RestoreCacheSettings,
RestoreConfig, RestoreParams)
return RestoreConfig(project=project,
restore_user=user,
case_sync=case_sync,
params=RestoreParams(
sync_log_id=restore_id,
version=version,
state_hash=state_hash,
include_item_count=items,
device_id=device_id,
app=app,
),
cache_settings=RestoreCacheSettings(
overwrite_cache=overwrite_cache,
force_cache=force_cache,
))
def _cached_restore(testcase,
user,
restore_id="",
version=V2,
purge_restore_cache=False):
"""DEPRECATED use <MockDevice>.sync().cases"""
assert not hasattr(testcase, 'restore_config'), testcase
assert not hasattr(testcase, 'payload_string'), testcase
if restore_id and purge_restore_cache:
RestorePayloadPathCache(
domain=user.domain,
user_id=user.user_id,
sync_log_id=restore_id,
device_id=None,
).invalidate()
testcase.restore_config = RestoreConfig(
project=user.project,
restore_user=user,
params=RestoreParams(restore_id, version=version),
**getattr(testcase, 'restore_options', {}))
testcase.payload_string = testcase.restore_config.get_payload().as_string()
try:
yield
finally:
del testcase.restore_config, testcase.payload_string
def test_simple_factor(self):
self.user.loadtest_factor = 3
self.user.save()
case1 = self.factory.create_case(case_name='case1')
case2 = self.factory.create_case(case_name='case2')
restore_config = RestoreConfig(
project=self.domain,
restore_user=self.user.to_ota_restore_user(),
params=RestoreParams(version=V2),
)
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(6, len(caseblocks))
self.assertEqual(1, len([cb for cb in caseblocks if cb.get_case_id() == case1.case_id]))
self.assertEqual(1, len([cb for cb in caseblocks if cb.get_case_id() == case2.case_id]))
self.assertEqual(3, len([cb for cb in caseblocks if case1.name in cb.get_case_name()]))
self.assertEqual(3, len([cb for cb in caseblocks if case2.name in cb.get_case_name()]))
def tearDown(self):
delete_all_xforms()
delete_all_cases()
delete_all_sync_logs()
restore_config = RestoreConfig(project=self.project,
restore_user=self.restore_user)
restore_config.cache.delete(restore_config._initial_cache_key)
super(BaseOtaRestoreTest, self).tearDown()
def testCaching(self):
self.assertFalse(self.sync_log.has_cached_payload(V2))
# first request should populate the cache
original_payload = RestoreConfig(
self.user,
version=V2,
caching_enabled=True,
restore_id=self.sync_log._id,
).get_payload()
next_sync_log = synclog_from_restore_payload(original_payload)
self.sync_log = SyncLog.get(self.sync_log._id)
self.assertTrue(self.sync_log.has_cached_payload(V2))
# a second request with the same config should be exactly the same
cached_payload = RestoreConfig(
self.user,
version=V2,
caching_enabled=True,
restore_id=self.sync_log._id,
).get_payload()
self.assertEqual(original_payload, cached_payload)
# a second request without caching should be different (generate a new id)
uncached_payload = RestoreConfig(
self.user,
version=V2,
caching_enabled=False,
restore_id=self.sync_log._id,
).get_payload()
self.assertNotEqual(original_payload, uncached_payload)
uncached_sync_log = synclog_from_restore_payload(uncached_payload)
self.assertNotEqual(next_sync_log._id, uncached_sync_log._id)
# caching a different version should also produce something new
versioned_payload = RestoreConfig(
self.user,
version=V1,
caching_enabled=True,
restore_id=self.sync_log._id,
).get_payload()
self.assertNotEqual(original_payload, versioned_payload)
versioned_sync_log = synclog_from_restore_payload(versioned_payload)
self.assertNotEqual(next_sync_log._id, versioned_sync_log._id)
def test_parent_child(self):
self.user.loadtest_factor = 3
self.user.save()
child, parent = self.factory.create_or_update_case(
CaseStructure(
attrs={'case_name': 'parent'},
relationships=[
CaseRelationship(CaseStructure(attrs={'case_name': 'child'})),
]
)
)
restore_config = RestoreConfig(self.user, version=V2, domain=self.domain)
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(6, len(caseblocks))
self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == child._id, caseblocks)))
self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == parent._id, caseblocks)))
self.assertEqual(3, len(filter(lambda cb: child.name in cb.get_case_name(), caseblocks)))
self.assertEqual(3, len(filter(lambda cb: parent.name in cb.get_case_name(), caseblocks)))
def prime_restore(username_or_id, domain, version, cache_timeout_hours,
overwrite_cache, check_cache_only):
couch_user = get_user(username_or_id, domain)
try:
project = couch_user.project
restore_config = RestoreConfig(
project=project,
restore_user=couch_user.to_ota_restore_user(),
params=RestoreParams(
version=version,
include_item_count=True,
),
cache_settings=RestoreCacheSettings(
force_cache=True,
cache_timeout=cache_timeout_hours * 60 * 60,
overwrite_cache=overwrite_cache
)
)
if check_cache_only:
cached_payload = _get_cached_payload(restore_config)
ret = u'Restore cache {} for user: {}'.format(
'EXISTS' if cached_payload else 'does not exist',
couch_user.human_friendly_name,
)
else:
restore_config.get_payload()
cached_payload = _get_cached_payload(restore_config)
if cached_payload:
ret = u'Restore cached successfully for user: {}'.format(
couch_user.human_friendly_name,
)
else:
raise PrimeRestoreException(u"Restore completed by cache still empty")
except Exception as e:
raise PrimeRestoreException(u'Error processing user: {}. Error was: {}'.format(
couch_user.human_friendly_name, str(e)
))
return {"messages": ret}
def setUp(self):
super(CommTrackSyncTest, self).setUp()
self.group = Group(domain=util.TEST_DOMAIN, name='commtrack-folks',
users=[self.user._id], case_sharing=True)
self.group._id = self.sp.owner_id
self.group.save()
self.restore_user = self.user.to_ota_restore_user()
self.sp_block = CaseBlock(
case_id=self.sp.case_id,
).as_xml()
# get initial restore token
restore_config = RestoreConfig(
project=self.domain,
restore_user=self.restore_user,
params=RestoreParams(version=V2),
)
self.sync_log_id = synclog_id_from_restore_payload(restore_config.get_payload().as_string())
def handle(self, username, **options):
couch_user = CommCareUser.get_by_username(username)
project = couch_user.project
restore_config = RestoreConfig(
project=project,
restore_user=couch_user.to_ota_restore_user(),
params=RestoreParams(
version=V2,
include_item_count=True,
),
cache_settings=RestoreCacheSettings(
force_cache=True,
cache_timeout=1,
overwrite_cache=False,
))
with resident_set_size():
restore_config.get_payload()
def setUp(self):
super(CommTrackSyncTest, self).setUp()
self.group = Group(domain=util.TEST_DOMAIN, name='commtrack-folks',
users=[self.user._id], case_sharing=True)
self.group._id = self.sp.owner_id
self.group.save()
self.restore_user = self.user.to_ota_restore_user()
self.sp_block = CaseBlock(
case_id=self.sp.case_id,
).as_xml()
# get initial restore token
restore_config = RestoreConfig(
project=self.domain,
restore_user=self.restore_user,
params=RestoreParams(version=V2),
)
self.sync_log_id = deprecated_synclog_id_from_restore_payload(
restore_config.get_payload().as_string())
def get_restore_response(domain, couch_user, since=None, version='1.0',
state=None, items=False):
# not a view just a view util
if not couch_user.is_commcare_user():
return HttpResponse("No linked chw found for %s" % couch_user.username,
status=401) # Authentication Failure
elif domain != couch_user.domain:
return HttpResponse("%s was not in the domain %s" % (couch_user.username, domain),
status=401)
project = Domain.get_by_name(domain)
commtrack_settings = project.commtrack_settings
stock_settings = commtrack_settings.get_ota_restore_settings() if commtrack_settings else None
restore_config = RestoreConfig(
couch_user.to_casexml_user(), since, version, state,
items=items,
stock_settings=stock_settings,
domain=project
)
return restore_config.get_response()
def handle(self, username, **options):
couch_user = CommCareUser.get_by_username(username)
project = couch_user.project
restore_config = RestoreConfig(
project=project,
restore_user=couch_user.to_ota_restore_user(),
params=RestoreParams(
version=V2,
include_item_count=True,
),
cache_settings=RestoreCacheSettings(
force_cache=True,
cache_timeout=1,
overwrite_cache=False,
)
)
with resident_set_size():
restore_config.get_payload()
def test_cross_domain_assignments(self):
good_domain = 'main-domain'
domain = create_domain(good_domain)
bad_domain = 'bad-domain'
create_domain(bad_domain)
user = CommCareUser.create(good_domain, format_username('user', good_domain), 'secret')
def _submit_case(domain):
case_id = uuid.uuid4().hex
case_block = CaseBlock(
create=True,
case_id=case_id,
case_name='donald',
case_type='duck',
user_id=user._id,
owner_id=user._id,
version=V2,
).as_xml()
post_case_blocks([case_block], {'domain': domain})
return CommCareCase.get(case_id)
good_case = _submit_case(good_domain)
# create a case in the "wrong" domain
# in the future this should actually fail completely
bad_case = _submit_case(bad_domain)
self.assertEqual(good_domain, good_case.domain)
self.assertEqual(bad_domain, bad_case.domain)
for case in (good_case, bad_case):
self.assertEqual(user._id, case.user_id)
self.assertEqual(user._id, case.owner_id)
restore_config = RestoreConfig(
project=domain,
user=user.to_casexml_user(),
params=RestoreParams(version=V2),
)
payload = restore_config.get_payload().as_string()
self.assertTrue(good_case._id in payload)
self.assertFalse(bad_case._id in payload)
def get_restore_response(domain, couch_user, since=None, version='1.0',
state=None, items=False, force_cache=False,
cache_timeout=None, overwrite_cache=False):
# not a view just a view util
if not couch_user.is_commcare_user():
return HttpResponse("No linked chw found for %s" % couch_user.username,
status=401) # Authentication Failure
elif domain != couch_user.domain:
return HttpResponse("%s was not in the domain %s" % (couch_user.username, domain),
status=401)
project = Domain.get_by_name(domain)
restore_config = RestoreConfig(
couch_user.to_casexml_user(), since, version, state,
items=items,
domain=project,
force_cache=force_cache,
cache_timeout=cache_timeout,
overwrite_cache=overwrite_cache
)
return restore_config.get_response()
def test_parent_child(self):
self.user.loadtest_factor = 3
self.user.save()
child, parent = self.factory.create_or_update_case(
CaseStructure(
attrs={'case_name': 'parent'},
indices=[
CaseIndex(CaseStructure(attrs={'case_name': 'child'})),
]
)
)
restore_config = RestoreConfig(
project=self.domain,
restore_user=self.user.to_ota_restore_user(),
params=RestoreParams(version=V2)
)
payload_string = restore_config.get_payload().as_string()
caseblocks = extract_caseblocks_from_xml(payload_string)
self.assertEqual(6, len(caseblocks))
self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == child.case_id, caseblocks)))
self.assertEqual(1, len(filter(lambda cb: cb.get_case_id() == parent.case_id, caseblocks)))
self.assertEqual(3, len(filter(lambda cb: child.name in cb.get_case_name(), caseblocks)))
self.assertEqual(3, len(filter(lambda cb: parent.name in cb.get_case_name(), caseblocks)))
def test_old_then_new_sync(self):
restore_config = RestoreConfig(self.project, user=self.user)
case = CaseFactory(domain=self.project.name, case_defaults={'owner_id': self.user_id}).create_case()
restore_payload = restore_config.get_payload().as_string()
self.assertTrue(case._id in restore_payload)
sync_log = synclog_from_restore_payload(restore_payload)
self.assertEqual(SyncLog, type(sync_log))
restore_config = RestoreConfig(self.project, user=self.user,
params=RestoreParams(sync_log_id=sync_log._id))
original_payload_back = restore_config.get_payload().as_string()
self.assertFalse(case._id in original_payload_back)
self.assertEqual(SyncLog, type(synclog_from_restore_payload(original_payload_back)))
OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=True, namespace='domain')
restore_config = RestoreConfig(self.project, user=self.user,
params=RestoreParams(sync_log_id=sync_log._id),
cache_settings=RestoreCacheSettings(overwrite_cache=True))
migrated_payload_back = restore_config.get_payload().as_string()
self.assertFalse(case._id in migrated_payload_back)
self.assertEqual(SimplifiedSyncLog, type(synclog_from_restore_payload(migrated_payload_back)))
OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain, enabled=False, namespace='domain')
