def test_synced_before_and_after_bug_resolution_200(self):
before = SimplifiedSyncLog(
user_id=self.restore_user.user_id,
date=datetime(2016, 7, 19, 18,
0) # synced before bug was introduced
)
before.save()
restore_config = RestoreConfig(project=self.project,
restore_user=self.restore_user,
params=RestoreParams(
sync_log_id=before._id,
version="2.0",
),
cache_settings=RestoreCacheSettings())
response = restore_config.get_response()
self.assertEqual(response.status_code, 200)
after = SimplifiedSyncLog(
user_id=self.restore_user.user_id,
previous_log_id=before._id,
date=datetime(2016, 7, 21, 19, 0) # after resolution
)
after.save()
restore_config = RestoreConfig(project=self.project,
restore_user=self.restore_user,
params=RestoreParams(
sync_log_id=after._id,
version="2.0",
),
cache_settings=RestoreCacheSettings())
response = restore_config.get_response()
self.assertEqual(response.status_code, 200)
def test_synced_during_and_after_bug_resolution_returns_200(self):
during = SimplifiedSyncLog(
user_id=self.restore_user.user_id,
date=datetime(2016, 7, 19, 20, 0) # during bug
)
during.save()
after = SimplifiedSyncLog(
user_id=self.restore_user.user_id,
previous_log_id=during._id,
date=datetime(2016, 7, 21, 19, 0) # after resolution
)
after.save()
restore_config = RestoreConfig(
project=self.project,
restore_user=self.restore_user,
params=RestoreParams(
sync_log_id=after._id,
version="2.0",
),
cache_settings=RestoreCacheSettings()
)
response = restore_config.get_response()
self.assertEqual(response.status_code, 200)
def handle(self, *args, **options):
username = options['username']
if not username:
print "You need a username!"
print "Usage: ./manage.py mem_profile_restore --username [email protected]"
return
couch_user = CommCareUser.get_by_username(username)
project = couch_user.project
restore_config = RestoreConfig(
project=project,
restore_user=couch_user.to_ota_restore_user(),
params=RestoreParams(
version=V2,
include_item_count=True,
),
cache_settings=RestoreCacheSettings(
force_cache=True,
cache_timeout=1,
overwrite_cache=False,
))
with resident_set_size():
restore_config.get_payload()
def get_restore_response(domain, couch_user, since=None, version='1.0',
state=None, items=False, force_cache=False,
cache_timeout=None, overwrite_cache=False,
force_restore_mode=None):
# not a view just a view util
if not couch_user.is_commcare_user():
return HttpResponse("No linked chw found for %s" % couch_user.username,
status=401) # Authentication Failure
elif domain != couch_user.domain:
return HttpResponse("%s was not in the domain %s" % (couch_user.username, domain),
status=401)
project = Domain.get_by_name(domain)
restore_config = RestoreConfig(
project=project,
user=couch_user.to_casexml_user(),
params=RestoreParams(
sync_log_id=since,
version=version,
state_hash=state,
include_item_count=items,
force_restore_mode=force_restore_mode,
),
cache_settings=RestoreCacheSettings(
force_cache=force_cache,
cache_timeout=cache_timeout,
overwrite_cache=overwrite_cache
),
)
return restore_config.get_response()
def test_old_then_new_sync(self):
restore_config = RestoreConfig(self.project, user=self.user)
case = CaseFactory(domain=self.project.name,
case_defaults={
'owner_id': self.user_id
}).create_case()
restore_payload = restore_config.get_payload().as_string()
self.assertTrue(case._id in restore_payload)
sync_log = synclog_from_restore_payload(restore_payload)
self.assertEqual(SyncLog, type(sync_log))
restore_config = RestoreConfig(
self.project,
user=self.user,
params=RestoreParams(sync_log_id=sync_log._id))
original_payload_back = restore_config.get_payload().as_string()
self.assertFalse(case._id in original_payload_back)
self.assertEqual(
SyncLog, type(synclog_from_restore_payload(original_payload_back)))
OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain,
enabled=True,
namespace='domain')
restore_config = RestoreConfig(
self.project,
user=self.user,
params=RestoreParams(sync_log_id=sync_log._id),
cache_settings=RestoreCacheSettings(overwrite_cache=True))
migrated_payload_back = restore_config.get_payload().as_string()
self.assertFalse(case._id in migrated_payload_back)
self.assertEqual(
SimplifiedSyncLog,
type(synclog_from_restore_payload(migrated_payload_back)))
OWNERSHIP_CLEANLINESS_RESTORE.set(self.domain,
enabled=False,
namespace='domain')
def generate_restore_payload(project, user, restore_id="", version=V1, state_hash="",
items=False, overwrite_cache=False, force_cache=False):
"""
Gets an XML payload suitable for OTA restore.
user: who the payload is for
restore_id: last sync token for this user
version: the restore API version
returns: the xml payload of the sync operation
"""
config = RestoreConfig(
project=project,
user=user,
params=RestoreParams(
sync_log_id=restore_id,
version=version,
state_hash=state_hash,
include_item_count=items
),
cache_settings=RestoreCacheSettings(
overwrite_cache=overwrite_cache,
force_cache=force_cache,
)
)
return config.get_payload().as_string()
def get_restore_config(project,
user,
restore_id="",
version=V1,
state_hash="",
items=False,
overwrite_cache=False,
force_cache=False,
device_id=None,
case_sync=None,
app=None):
from casexml.apps.phone.restore import (RestoreCacheSettings,
RestoreConfig, RestoreParams)
return RestoreConfig(project=project,
restore_user=user,
case_sync=case_sync,
params=RestoreParams(
sync_log_id=restore_id,
version=version,
state_hash=state_hash,
include_item_count=items,
device_id=device_id,
app=app,
),
cache_settings=RestoreCacheSettings(
overwrite_cache=overwrite_cache,
force_cache=force_cache,
))
def get_restore_response(domain,
couch_user,
app_id=None,
since=None,
version='1.0',
state=None,
items=False,
force_cache=False,
cache_timeout=None,
overwrite_cache=False,
force_restore_mode=None,
as_user=None,
device_id=None,
has_data_cleanup_privelege=False,
openrosa_version=OPENROSA_DEFAULT_VERSION):
# not a view just a view util
is_permitted, message = is_permitted_to_restore(
domain,
couch_user,
as_user,
has_data_cleanup_privelege,
)
if not is_permitted:
return HttpResponse(message, status=401), None
if couch_user.is_commcare_user() and couch_user.is_demo_user:
# if user is in demo-mode, return demo restore
return demo_user_restore_response(couch_user), None
restore_user = get_restore_user(domain, couch_user, as_user)
if not restore_user:
return HttpResponse('Could not find user', status=404), None
project = Domain.get_by_name(domain)
app = get_app(domain, app_id) if app_id else None
async_restore_enabled = (toggles.ASYNC_RESTORE.enabled(domain) and
LooseVersion(openrosa_version) >= LooseVersion(
OPENROSA_VERSION_MAP['ASYNC_RESTORE']))
restore_config = RestoreConfig(
project=project,
restore_user=restore_user,
params=RestoreParams(
sync_log_id=since,
version=version,
state_hash=state,
include_item_count=items,
app=app,
device_id=device_id,
),
cache_settings=RestoreCacheSettings(force_cache=force_cache
or async_restore_enabled,
cache_timeout=cache_timeout,
overwrite_cache=overwrite_cache),
async=async_restore_enabled)
return restore_config.get_response(), restore_config.timing_context
def _restore_config(self, is_async=True, sync_log_id='', overwrite_cache=False):
restore_config = RestoreConfig(
project=self.project,
restore_user=self.user,
params=RestoreParams(sync_log_id=sync_log_id, version=V2),
cache_settings=RestoreCacheSettings(
overwrite_cache=overwrite_cache
),
is_async=is_async
)
self.addCleanup(get_redis_default_cache().clear)
return restore_config
def test_return_412_between_bug_dates(self):
log = SimplifiedSyncLog(user_id=self.restore_user.user_id,
date=datetime(2016, 7, 19, 19, 20))
log.save()
restore_config = RestoreConfig(project=self.project,
restore_user=self.restore_user,
params=RestoreParams(
sync_log_id=log._id,
version="2.0",
),
cache_settings=RestoreCacheSettings())
response = restore_config.get_response()
self.assertEqual(response.status_code, 412)
def get_restore_config(project, user, restore_id="", version=V1, state_hash="",
items=False, overwrite_cache=False, force_cache=False, device_id=None):
return RestoreConfig(
project=project,
restore_user=user,
params=RestoreParams(
sync_log_id=restore_id,
version=version,
state_hash=state_hash,
include_item_count=items,
device_id=device_id,
),
cache_settings=RestoreCacheSettings(
overwrite_cache=overwrite_cache,
force_cache=force_cache,
)
)
def handle(self, username, **options):
couch_user = CommCareUser.get_by_username(username)
project = couch_user.project
restore_config = RestoreConfig(
project=project,
restore_user=couch_user.to_ota_restore_user(),
params=RestoreParams(
version=V2,
include_item_count=True,
),
cache_settings=RestoreCacheSettings(
force_cache=True,
cache_timeout=1,
overwrite_cache=False,
))
with resident_set_size():
restore_config.get_payload()
def prime_restore(username_or_id, domain, version, cache_timeout_hours,
overwrite_cache, check_cache_only):
couch_user = get_user(username_or_id, domain)
try:
project = couch_user.project
restore_config = RestoreConfig(
project=project,
restore_user=couch_user.to_ota_restore_user(),
params=RestoreParams(
version=version,
include_item_count=True,
),
cache_settings=RestoreCacheSettings(
force_cache=True,
cache_timeout=cache_timeout_hours * 60 * 60,
overwrite_cache=overwrite_cache
)
)
if check_cache_only:
cached_payload = _get_cached_payload(restore_config)
ret = u'Restore cache {} for user: {}'.format(
'EXISTS' if cached_payload else 'does not exist',
couch_user.human_friendly_name,
)
else:
restore_config.get_payload()
cached_payload = _get_cached_payload(restore_config)
if cached_payload:
ret = u'Restore cached successfully for user: {}'.format(
couch_user.human_friendly_name,
)
else:
raise PrimeRestoreException(u"Restore completed by cache still empty")
except Exception as e:
raise PrimeRestoreException(u'Error processing user: {}. Error was: {}'.format(
couch_user.human_friendly_name, str(e)
))
return {"messages": ret}
def get_restore_response(domain,
couch_user,
app_id=None,
since=None,
version='1.0',
state=None,
items=False,
force_cache=False,
cache_timeout=None,
overwrite_cache=False,
as_user=None,
device_id=None,
user_id=None,
openrosa_version=None,
case_sync=None):
"""
:param domain: Domain being restored from
:param couch_user: User performing restore
:param app_id: App ID of the app making the request
:param since: ID of current sync log used to generate incremental sync
:param version: Version of the sync response required
:param state: Hash value of the current database of cases on the device for consistency checking
:param items: Include item count if True
:param force_cache: Force response to be cached
:param cache_timeout: Override the default cache timeout of 1 hour.
:param overwrite_cache: Ignore cached response if True
:param as_user: Username of user to generate restore for (if different from current user)
:param device_id: ID of device performing restore
:param user_id: ID of user performing restore (used in case of deleted user with same username)
:param openrosa_version:
:param case_sync: Override default case sync algorithm
:return: Tuple of (http response, timing context or None)
"""
if user_id and user_id != couch_user.user_id:
# sync with a user that has been deleted but a new
# user was created with the same username and password
from couchforms.openrosa_response import get_simple_response_xml
from couchforms.openrosa_response import ResponseNature
response = get_simple_response_xml(
'Attempt to sync with invalid user.',
ResponseNature.OTA_RESTORE_ERROR)
return HttpResponse(response,
content_type="text/xml; charset=utf-8",
status=412), None
is_demo_restore = couch_user.is_commcare_user() and couch_user.is_demo_user
if is_demo_restore:
# if user is in demo-mode, return demo restore
return demo_user_restore_response(couch_user), None
uses_login_as = bool(as_user)
as_user_obj = CouchUser.get_by_username(as_user) if uses_login_as else None
if uses_login_as and not as_user_obj:
msg = _('Invalid restore as user {}').format(as_user)
return HttpResponse(msg, status=401), None
is_permitted, message = is_permitted_to_restore(
domain,
couch_user,
as_user_obj,
)
if not is_permitted:
return HttpResponse(message, status=401), None
restore_user = get_restore_user(domain, couch_user, as_user_obj)
if not restore_user:
return HttpResponse('Could not find user', status=404), None
project = Domain.get_by_name(domain)
async_restore_enabled = (toggles.ASYNC_RESTORE.enabled(domain)
and openrosa_version and
LooseVersion(openrosa_version) >= LooseVersion(
OPENROSA_VERSION_MAP['ASYNC_RESTORE']))
app = get_app_cached(domain, app_id) if app_id else None
restore_config = RestoreConfig(
project=project,
restore_user=restore_user,
params=RestoreParams(
sync_log_id=since,
version=version,
state_hash=state,
include_item_count=items,
app=app,
device_id=device_id,
openrosa_version=openrosa_version,
),
cache_settings=RestoreCacheSettings(force_cache=force_cache
or async_restore_enabled,
cache_timeout=cache_timeout,
overwrite_cache=overwrite_cache),
is_async=async_restore_enabled,
case_sync=case_sync,
)
return restore_config.get_response(), restore_config.timing_context
def tearDownClass(cls):
cls.project.delete()
delete_all_cases()
delete_all_sync_logs()
delete_all_users()
super(BaseAsyncRestoreTest, cls).tearDownClass()
def _restore_config(self,
async=True,
sync_log_id='',
overwrite_cache=False):
restore_config = RestoreConfig(
project=self.project,
restore_user=self.user,
params=RestoreParams(sync_log_id=sync_log_id, version=V2),
cache_settings=RestoreCacheSettings(
overwrite_cache=overwrite_cache),
is_async=async)
self.addCleanup(get_redis_default_cache().clear)
return restore_config
class AsyncRestoreTestCouchOnly(BaseAsyncRestoreTest):
@mock.patch('casexml.apps.phone.restore.get_async_restore_payload')
def test_regular_restore_doesnt_start_task(self, task):
"""
when the feature flag is off, the celery task does not get called
"""
self._restore_config(async=False).get_payload()
self.assertFalse(task.delay.called)
@mock.patch('casexml.apps.phone.restore.get_async_restore_payload')
def get_restore_response(domain,
couch_user,
app_id=None,
since=None,
version='1.0',
state=None,
items=False,
force_cache=False,
cache_timeout=None,
overwrite_cache=False,
force_restore_mode=None,
as_user=None,
device_id=None,
user_id=None,
has_data_cleanup_privelege=False,
openrosa_version=OPENROSA_DEFAULT_VERSION):
if user_id and user_id != couch_user.user_id:
# sync with a user that has been deleted but a new
# user was created with the same username and password
from couchforms.openrosa_response import get_simple_response_xml
from couchforms.openrosa_response import ResponseNature
response = get_simple_response_xml(
'Attempt to sync with invalid user.',
ResponseNature.OTA_RESTORE_ERROR)
return HttpResponse(response,
content_type="text/xml; charset=utf-8",
status=412), None
# not a view just a view util
is_permitted, message = is_permitted_to_restore(
domain,
couch_user,
as_user,
has_data_cleanup_privelege,
)
if not is_permitted:
return HttpResponse(message, status=401), None
is_demo_restore = couch_user.is_commcare_user() and couch_user.is_demo_user
is_enikshay = toggles.ENIKSHAY.enabled(domain)
if is_enikshay:
couch_restore_user = couch_user
if not is_demo_restore and as_user is not None:
couch_restore_user = CouchUser.get_by_username(as_user)
update_device_id(couch_restore_user, device_id)
if is_demo_restore:
# if user is in demo-mode, return demo restore
return demo_user_restore_response(couch_user), None
restore_user = get_restore_user(domain, couch_user, as_user)
if not restore_user:
return HttpResponse('Could not find user', status=404), None
project = Domain.get_by_name(domain)
app = get_app(domain, app_id) if app_id else None
async_restore_enabled = (toggles.ASYNC_RESTORE.enabled(domain) and
LooseVersion(openrosa_version) >= LooseVersion(
OPENROSA_VERSION_MAP['ASYNC_RESTORE']))
restore_config = RestoreConfig(
project=project,
restore_user=restore_user,
params=RestoreParams(
sync_log_id=since,
version=version,
state_hash=state,
include_item_count=items,
app=app,
device_id=device_id,
),
cache_settings=RestoreCacheSettings(force_cache=force_cache
or async_restore_enabled,
cache_timeout=cache_timeout,
overwrite_cache=overwrite_cache),
async=async_restore_enabled)
return restore_config.get_response(), restore_config.timing_context
def get_restore_response(domain,
couch_user,
app_id=None,
since=None,
version='1.0',
state=None,
items=False,
force_cache=False,
cache_timeout=None,
overwrite_cache=False,
as_user=None,
device_id=None,
user_id=None,
openrosa_version=None,
case_sync=None):
if user_id and user_id != couch_user.user_id:
# sync with a user that has been deleted but a new
# user was created with the same username and password
from couchforms.openrosa_response import get_simple_response_xml
from couchforms.openrosa_response import ResponseNature
response = get_simple_response_xml(
'Attempt to sync with invalid user.',
ResponseNature.OTA_RESTORE_ERROR)
return HttpResponse(response,
content_type="text/xml; charset=utf-8",
status=412), None
is_demo_restore = couch_user.is_commcare_user() and couch_user.is_demo_user
if is_demo_restore:
# if user is in demo-mode, return demo restore
return demo_user_restore_response(couch_user), None
uses_login_as = bool(as_user)
as_user_obj = CouchUser.get_by_username(as_user) if uses_login_as else None
if uses_login_as and not as_user_obj:
msg = _('Invalid restore as user {}').format(as_user)
return HttpResponse(msg, status=401), None
is_permitted, message = is_permitted_to_restore(
domain,
couch_user,
as_user_obj,
)
if not is_permitted:
return HttpResponse(message, status=401), None
restore_user = get_restore_user(domain, couch_user, as_user_obj)
if not restore_user:
return HttpResponse('Could not find user', status=404), None
project = Domain.get_by_name(domain)
async_restore_enabled = (toggles.ASYNC_RESTORE.enabled(domain)
and openrosa_version and
LooseVersion(openrosa_version) >= LooseVersion(
OPENROSA_VERSION_MAP['ASYNC_RESTORE']))
app = get_app_cached(domain, app_id) if app_id else None
restore_config = RestoreConfig(
project=project,
restore_user=restore_user,
params=RestoreParams(
sync_log_id=since,
version=version,
state_hash=state,
include_item_count=items,
app=app,
device_id=device_id,
openrosa_version=openrosa_version,
),
cache_settings=RestoreCacheSettings(force_cache=force_cache
or async_restore_enabled,
cache_timeout=cache_timeout,
overwrite_cache=overwrite_cache),
async=async_restore_enabled,
case_sync=case_sync,
)
return restore_config.get_response(), restore_config.timing_context
def prime_restore(domain, usernames_or_ids, version=V1, cache_timeout_hours=None,
overwrite_cache=False, check_cache_only=False):
"""
Task to generate and cache a restore payload for each user passed in.
:param domain: The domain name for the users
:param usernames_or_ids: List of usernames or user IDs
:param version: Restore format version
:param cache_timeout_hours: Hours to cache the payload
:param overwrite_cache: If True overwrite any existing cache
:param check_cache_only: Don't generate the payload, just check if it is already cached
"""
total = len(usernames_or_ids)
DownloadBase.set_progress(prime_restore, 0, total)
ret = {'messages': []}
for i, username_or_id in enumerate(usernames_or_ids):
couch_user = get_user(username_or_id, domain)
if not couch_user:
ret['messages'].append('WARNING: User not found: {}'.format(username_or_id))
continue
elif couch_user.domain != domain:
ret['messages'].append("WARNING: User '{}' not from domain '{}'".format(
username_or_id,
domain
))
continue
try:
project = couch_user.project
restore_config = RestoreConfig(
project=project,
user=couch_user.to_casexml_user(),
params=RestoreParams(
version=version,
include_item_count=True,
),
cache_settings=RestoreCacheSettings(
force_cache=True,
cache_timeout=cache_timeout_hours * 60 * 60,
overwrite_cache=overwrite_cache
)
)
if check_cache_only:
cached_payload = _get_cached_payload(restore_config)
ret['messages'].append(u'Restore cache {} for user: {}'.format(
'EXISTS' if cached_payload else 'does not exist',
couch_user.human_friendly_name,
))
else:
restore_config.get_payload()
cached_payload = _get_cached_payload(restore_config)
if cached_payload:
ret['messages'].append('SUCCESS: Restore cached successfully for user: {}'.format(
couch_user.human_friendly_name,
))
else:
ret['messages'].append('ERROR: Restore completed by cache still empty for user: {}'.format(
couch_user.human_friendly_name,
))
except Exception as e:
ret['messages'].append('ERROR: Error processing user: {}'.format(str(e)))
DownloadBase.set_progress(prime_restore, i + 1, total)
return ret
