def last_sync_log(self):
if self.params.sync_log_id:
try:
sync_log = get_properly_wrapped_sync_log(
self.params.sync_log_id)
except ResourceNotFound:
# if we are in loose mode, return an HTTP 412 so that the phone will
# just force a fresh sync
raise MissingSyncLog('No sync log with ID {} found'.format(
self.params.sync_log_id))
if sync_log.doc_type != 'SyncLog':
raise InvalidSyncLogException(
'Bad sync log doc type for {}'.format(
self.params.sync_log_id))
elif sync_log.user_id != self.user.user_id:
raise SyncLogUserMismatch(
'Sync log {} does not match user id {} (was {})'.format(
self.params.sync_log_id, self.user.user_id,
sync_log.user_id))
# convert to the right type if necessary
if not isinstance(sync_log, self.sync_log_class):
# this call can fail with an IncompatibleSyncLogType error
sync_log = self.sync_log_class.from_other_format(sync_log)
return sync_log
else:
return None
def last_sync_log(self):
if self._last_sync_log is Ellipsis:
if self.params.sync_log_id:
# if we are in loose mode, return an HTTP 412 so that the phone will
# just force a fresh sync
# This raises MissingSyncLog exception if synclog not found
sync_log = get_properly_wrapped_sync_log(
self.params.sync_log_id)
if sync_log.doc_type not in ('SyncLog', 'SimplifiedSyncLog'):
raise InvalidSyncLogException(
'Bad sync log doc type for {}'.format(
self.params.sync_log_id))
elif sync_log.user_id != self.restore_user.user_id:
raise SyncLogUserMismatch(
'Sync log {} does not match user id {} (was {})'.
format(self.params.sync_log_id,
self.restore_user.user_id, sync_log.user_id))
# convert to the right type if necessary
if not isinstance(sync_log, SimplifiedSyncLog):
# this call can fail with an IncompatibleSyncLogType error
sync_log = SimplifiedSyncLog.from_other_format(sync_log)
self._last_sync_log = sync_log
else:
self._last_sync_log = None
return self._last_sync_log
def testMismatch(self):
self.assertEqual(CaseStateHash(EMPTY_HASH), self.sync_log.get_state_hash())
c1 = CaseBlock(case_id="abc123", create=True,
owner_id=self.user.user_id).as_xml()
c2 = CaseBlock(case_id="123abc", create=True,
owner_id=self.user.user_id).as_xml()
post_case_blocks([c1, c2],
form_extras={"last_sync_token": self.sync_log.get_id})
self.sync_log = get_properly_wrapped_sync_log(self.sync_log.get_id)
real_hash = CaseStateHash("409c5c597fa2c2a693b769f0d2ad432b")
bad_hash = CaseStateHash("thisisntright")
self.assertEqual(real_hash, self.sync_log.get_state_hash())
generate_restore_payload(
self.project, self.user, self.sync_log.get_id,
version=V2, state_hash=str(real_hash)
)
try:
generate_restore_payload(self.project, self.user, self.sync_log.get_id,
version=V2, state_hash=str(bad_hash))
self.fail("Call to generate a payload with a bad hash should fail!")
except BadStateException, e:
self.assertEqual(real_hash, e.server_hash)
self.assertEqual(bad_hash, e.phone_hash)
self.assertEqual(2, len(e.case_ids))
self.assertTrue("abc123" in e.case_ids)
self.assertTrue("123abc" in e.case_ids)
def update_sync_log_with_checks(sync_log, xform, cases, case_db,
case_id_blacklist=None):
assert case_db is not None
from casexml.apps.case.xform import CaseProcessingConfig
case_id_blacklist = case_id_blacklist or []
try:
sync_log.update_phone_lists(xform, cases)
except SyncLogAssertionError as e:
if e.case_id and e.case_id not in case_id_blacklist:
form_ids = get_case_xform_ids(e.case_id)
case_id_blacklist.append(e.case_id)
for form_id in form_ids:
if form_id != xform._id:
form = XFormInstance.get(form_id)
if form.doc_type == 'XFormInstance':
from casexml.apps.case.xform import process_cases_with_casedb
process_cases_with_casedb(
[form],
case_db,
CaseProcessingConfig(
strict_asserts=True,
case_id_blacklist=case_id_blacklist
)
)
updated_log = get_properly_wrapped_sync_log(sync_log._id)
update_sync_log_with_checks(updated_log, xform, cases, case_db,
case_id_blacklist=case_id_blacklist)
def update_sync_log_with_checks(sync_log, xform, cases, case_db,
case_id_blacklist=None):
assert case_db is not None
from casexml.apps.case.xform import CaseProcessingConfig
case_id_blacklist = case_id_blacklist or []
try:
sync_log.update_phone_lists(xform, cases)
except SyncLogAssertionError, e:
if e.case_id and e.case_id not in case_id_blacklist:
form_ids = get_case_xform_ids(e.case_id)
case_id_blacklist.append(e.case_id)
for form_id in form_ids:
if form_id != xform._id:
form = XFormInstance.get(form_id)
if form.doc_type == 'XFormInstance':
from casexml.apps.case.xform import process_cases_with_casedb
process_cases_with_casedb(
[form],
case_db,
CaseProcessingConfig(
strict_asserts=True,
case_id_blacklist=case_id_blacklist
)
)
updated_log = get_properly_wrapped_sync_log(sync_log._id)
update_sync_log_with_checks(updated_log, xform, cases, case_db,
case_id_blacklist=case_id_blacklist)
def get_sync_token(self):
from casexml.apps.phone.models import get_properly_wrapped_sync_log
if self.last_sync_token:
try:
return get_properly_wrapped_sync_log(self.last_sync_token)
except ResourceNotFound:
pass
return None
def get_sync_token(self):
from casexml.apps.phone.models import get_properly_wrapped_sync_log
if self.last_sync_token:
try:
return get_properly_wrapped_sync_log(self.last_sync_token)
except MissingSyncLog:
pass
return None
def phone_holds_all_cases(request):
try:
synclog = get_properly_wrapped_sync_log(request.last_sync_token)
missing_case_ids_on_phone = set(
case_ids) - synclog.case_ids_on_phone
return not missing_case_ids_on_phone
except MissingSyncLog:
return False
def get_sync_token(self):
from casexml.apps.phone.models import get_properly_wrapped_sync_log
if self.last_sync_token:
try:
return get_properly_wrapped_sync_log(self.last_sync_token)
except MissingSyncLog:
pass
return None
def test(self):
if skip:
self.skipTest(skip)
self.build_case_structures(test_name)
desired_cases = self._get_test(test_name).get('outcome', [])
undesired_cases = [case for case in self.ALL_CASES if case not in desired_cases]
sync_log = get_properly_wrapped_sync_log(self.sync_log._id)
self.assertEqual(sync_log.case_ids_on_phone, set(desired_cases))
assert_user_has_cases(self, self.user, desired_cases)
assert_user_doesnt_have_cases(self, self.user, undesired_cases)
def test_previous_log_purged(self):
initial_synclog_id = synclog_id_from_restore_payload(
generate_restore_payload(self.project,
self.restore_user,
items=True))
# form submission success when there is no previous sync log
form_xml = get_simple_form_xml(uuid.uuid4().hex)
submit_form_locally(form_xml,
self.domain,
last_sync_token=initial_synclog_id)
# second sync
synclog_id = synclog_id_from_restore_payload(
generate_restore_payload(self.project,
self.restore_user,
restore_id=initial_synclog_id))
synclog = get_properly_wrapped_sync_log(synclog_id)
self.assertEqual(synclog.previous_log_id, initial_synclog_id)
self.assertFalse(synclog.previous_log_removed)
# form submission after second sync should remove first synclog
form_xml = get_simple_form_xml(uuid.uuid4().hex)
submit_form_locally(form_xml, self.domain, last_sync_token=synclog_id)
synclog = get_properly_wrapped_sync_log(synclog_id)
self.assertEqual(synclog.previous_log_id, initial_synclog_id)
self.assertTrue(synclog.previous_log_removed)
with self.assertRaises(ResourceNotFound):
get_properly_wrapped_sync_log(initial_synclog_id)
# form submissions after purge don't fail
form_xml = get_simple_form_xml(uuid.uuid4().hex)
submit_form_locally(form_xml, self.domain, last_sync_token=synclog_id)
# restores after purge don't fail
response = generate_restore_response(self.project,
self.restore_user,
restore_id=synclog_id)
self.assertEqual(response.status_code, 200)
def get_sync_token(self):
from casexml.apps.phone.models import get_properly_wrapped_sync_log
if self.last_sync_token:
try:
return get_properly_wrapped_sync_log(self.last_sync_token)
except ResourceNotFound:
logging.exception('No sync token with ID {} found. Form is {} in domain {}'.format(
self.last_sync_token, self.form_id, self.domain,
))
raise
return None
def get_sync_token(self):
from casexml.apps.phone.models import get_properly_wrapped_sync_log
if self.last_sync_token:
try:
return get_properly_wrapped_sync_log(self.last_sync_token)
except ResourceNotFound:
logging.exception('No sync token with ID {} found. Form is {} in domain {}'.format(
self.last_sync_token, self.form_id, self.domain,
))
raise
return None
def get_log(self):
"""Get the latest sync log from the database
Unlike the `log` property, this method does not cache its
result. A sync log is updated when new cases are processed as
part of a form submission referencing the sync log. Therefore
a sync log returned by this method and the one returned by the
`log` property may reference different cases. See
`casexml.apps.case.xform.process_cases_with_casedb` and
`casexml.apps.case.util.update_sync_log_with_checks`.
"""
return get_properly_wrapped_sync_log(self.restore_id)
def get_log(self):
"""Get the latest sync log from the database
Unlike the `log` property, this method does not cache its
result. A sync log is updated when new cases are processed as
part of a form submission referencing the sync log. Therefore
a sync log returned by this method and the one returned by the
`log` property may reference different cases. See
`casexml.apps.case.xform.process_cases_with_casedb` and
`casexml.apps.case.util.update_sync_log_with_checks`.
"""
return get_properly_wrapped_sync_log(self.restore_id)
def test(self):
if skip:
self.skipTest(skip)
self.build_case_structures(test_name)
desired_cases = self._get_test(test_name).get('outcome', [])
undesired_cases = [
case for case in self.ALL_CASES if case not in desired_cases
]
sync_log = get_properly_wrapped_sync_log(self.sync_log._id)
self.assertEqual(sync_log.case_ids_on_phone, set(desired_cases))
assert_user_has_cases(self, self.user, desired_cases)
assert_user_doesnt_have_cases(self, self.user, undesired_cases)
test.__name__ = get_test_name(test_name)
return test
def copy_payload_and_synclog_and_get_new_file(filelike_payload):
"""
Given a restore payload, extracts the sync log id and sync log from the payload,
makes a copy of the sync log, and then returns a new FileReference with the same contents
except using the new sync log ID.
"""
synclog_id, end_position = extract_synclog_id_from_filelike_payload(filelike_payload)
old_sync_log = get_properly_wrapped_sync_log(synclog_id)
new_sync_log_doc = old_sync_log.to_json()
new_sync_log_id = uuid.uuid4().hex
new_sync_log_doc['_id'] = new_sync_log_id
del new_sync_log_doc['_rev']
old_sync_log.get_db().save_doc(new_sync_log_doc)
return replace_sync_log_id_in_filelike_payload(
filelike_payload, old_sync_log._id, new_sync_log_id, end_position
)
def last_sync_log(self):
if self._last_sync_log is Ellipsis:
if self.params.sync_log_id:
# if we are in loose mode, return an HTTP 412 so that the phone will
# just force a fresh sync
# This raises MissingSyncLog exception if synclog not found
sync_log = get_properly_wrapped_sync_log(self.params.sync_log_id)
if sync_log.doc_type != 'SimplifiedSyncLog':
raise InvalidSyncLogException('Bad sync log doc type for {}'.format(self.params.sync_log_id))
elif sync_log.user_id != self.restore_user.user_id:
raise SyncLogUserMismatch('Sync log {} does not match user id {} (was {})'.format(
self.params.sync_log_id, self.restore_user.user_id, sync_log.user_id
))
self._last_sync_log = sync_log
else:
self._last_sync_log = None
return self._last_sync_log
def test_copy_payload(self):
sync_log = SimplifiedSyncLog(case_ids_on_phone=set(["case-1", "case-2"]))
sync_log.save()
payload = dummy_restore_xml(sync_log._id).strip()
fd, path = tempfile.mkstemp()
with os.fdopen(fd, "wb") as f:
f.write(payload)
with open(path, "r") as f:
updated_fileref = copy_payload_and_synclog_and_get_new_file(f)
updated_payload = updated_fileref.file.read()
updated_id = synclog_id_from_restore_payload(updated_payload)
self.assertNotEqual(sync_log._id, updated_id)
self.assertTrue(_restore_id_block(updated_id) in updated_payload)
self.assertFalse(sync_log._id in updated_payload)
updated_log = get_properly_wrapped_sync_log(updated_id)
self.assertEqual(updated_log.case_ids_on_phone, sync_log.case_ids_on_phone)
def test_get_and_save_legacy_synclog_with_attachm(self):
legacy_sync_log = self.legacy_sync_logs[0]
attachment_name = 'test_attach'
get_db(None).put_attachment(legacy_sync_log._doc, 'test', attachment_name, 'text/plain')
sync_log = get_properly_wrapped_sync_log(legacy_sync_log._id)
self.assertIn(attachment_name, sync_log._attachments)
# this used to fail for docs with attachments
sync_log.save()
# cleanup
def del_attachment():
get_db(None).delete_attachment(legacy_sync_log._doc, attachment_name)
del legacy_sync_log._attachments
self.addCleanup(del_attachment)
self.addCleanup(sync_log.delete)
def test_copy_payload(self):
sync_log = SimplifiedSyncLog(case_ids_on_phone=set(['case-1', 'case-2']))
sync_log.save()
payload = dummy_restore_xml(sync_log._id).strip()
fd, path = tempfile.mkstemp()
with os.fdopen(fd, 'wb') as f:
f.write(payload)
with open(path, 'r') as f:
updated_fileref = copy_payload_and_synclog_and_get_new_file(f)
updated_payload = updated_fileref.file.read()
updated_id = synclog_id_from_restore_payload(updated_payload)
self.assertNotEqual(sync_log._id, updated_id)
self.assertTrue(_restore_id_block(updated_id) in updated_payload)
self.assertFalse(sync_log._id in updated_payload)
updated_log = get_properly_wrapped_sync_log(updated_id)
self.assertEqual(updated_log.case_ids_on_phone, sync_log.case_ids_on_phone)
def copy_payload_and_synclog_and_get_new_file(filelike_payload):
"""
Given a restore payload, extracts the sync log id and sync log from the payload,
makes a copy of the sync log, and then returns a new FileReference with the same contents
except using the new sync log ID.
"""
synclog_id, end_position = extract_synclog_id_from_filelike_payload(
filelike_payload)
old_sync_log = get_properly_wrapped_sync_log(synclog_id)
new_sync_log_doc = old_sync_log.to_json()
new_sync_log_id = uuid.uuid4().hex
new_sync_log_doc['_id'] = new_sync_log_id
del new_sync_log_doc['_rev']
old_sync_log.get_db().save_doc(new_sync_log_doc)
return replace_sync_log_id_in_filelike_payload(filelike_payload,
old_sync_log._id,
new_sync_log_id,
end_position)
def test_get_and_save_legacy_synclog_with_attachm(self):
legacy_sync_log = self.legacy_sync_logs[0]
attachment_name = 'test_attach'
get_db(None).put_attachment(legacy_sync_log._doc, 'test', attachment_name, 'text/plain')
sync_log = get_properly_wrapped_sync_log(legacy_sync_log._id)
self.assertIn(attachment_name, sync_log._attachments)
# this used to fail for docs with attachments
sync_log.save()
# cleanup
def del_attachment():
get_db(None).delete_attachment(legacy_sync_log._doc, attachment_name)
del legacy_sync_log._attachments
self.addCleanup(del_attachment)
self.addCleanup(sync_log.delete)
def last_sync_log(self):
if self.params.sync_log_id:
try:
sync_log = get_properly_wrapped_sync_log(self.params.sync_log_id)
except ResourceNotFound:
# if we are in loose mode, return an HTTP 412 so that the phone will
# just force a fresh sync
raise MissingSyncLog('No sync log with ID {} found'.format(self.params.sync_log_id))
if sync_log.doc_type != 'SyncLog':
raise InvalidSyncLogException('Bad sync log doc type for {}'.format(self.params.sync_log_id))
elif sync_log.user_id != self.user.user_id:
raise SyncLogUserMismatch('Sync log {} does not match user id {} (was {})'.format(
self.params.sync_log_id, self.user.user_id, sync_log.user_id
))
return sync_log
else:
return None
def last_sync_log(self):
if self.params.sync_log_id:
try:
sync_log = get_properly_wrapped_sync_log(self.params.sync_log_id)
except ResourceNotFound:
# if we are in loose mode, return an HTTP 412 so that the phone will
# just force a fresh sync
raise MissingSyncLog('No sync log with ID {} found'.format(self.params.sync_log_id))
if sync_log.doc_type != 'SyncLog':
raise InvalidSyncLogException('Bad sync log doc type for {}'.format(self.params.sync_log_id))
elif sync_log.user_id != self.user.user_id:
raise SyncLogUserMismatch('Sync log {} does not match user id {} (was {})'.format(
self.params.sync_log_id, self.user.user_id, sync_log.user_id
))
# convert to the right type if necessary
if not isinstance(sync_log, self.sync_log_class):
# this call can fail with an IncompatibleSyncLogType error
sync_log = self.sync_log_class.from_other_format(sync_log)
return sync_log
else:
return None
def last_sync_log(self):
if self._last_sync_log is Ellipsis:
if self.params.sync_log_id:
# if we are in loose mode, return an HTTP 412 so that the phone will
# just force a fresh sync
# This raises MissingSyncLog exception if synclog not found
sync_log = get_properly_wrapped_sync_log(self.params.sync_log_id)
if sync_log.doc_type not in ('SyncLog', 'SimplifiedSyncLog'):
raise InvalidSyncLogException('Bad sync log doc type for {}'.format(self.params.sync_log_id))
elif sync_log.user_id != self.restore_user.user_id:
raise SyncLogUserMismatch('Sync log {} does not match user id {} (was {})'.format(
self.params.sync_log_id, self.restore_user.user_id, sync_log.user_id
))
# convert to the right type if necessary
if not isinstance(sync_log, SimplifiedSyncLog):
# this call can fail with an IncompatibleSyncLogType error
sync_log = SimplifiedSyncLog.from_other_format(sync_log)
self._last_sync_log = sync_log
else:
self._last_sync_log = None
return self._last_sync_log
def update_sync_log_with_checks(sync_log, xform, cases, case_db,
case_id_blacklist=None):
assert case_db is not None
from casexml.apps.case.xform import CaseProcessingConfig
case_id_blacklist = case_id_blacklist or []
try:
sync_log.update_phone_lists(xform, cases)
except SyncLogAssertionError as e:
soft_assert('@'.join(['skelly', 'dimagi.com']))(
False,
'SyncLogAssertionError raised while updating phone lists',
{
'form_id': xform.form_id,
'cases': [case.case_id for case in cases]
}
)
if e.case_id and e.case_id not in case_id_blacklist:
form_ids = get_case_xform_ids(e.case_id)
case_id_blacklist.append(e.case_id)
for form_id in form_ids:
if form_id != xform._id:
form = XFormInstance.get(form_id)
if form.doc_type == 'XFormInstance':
from casexml.apps.case.xform import process_cases_with_casedb
process_cases_with_casedb(
[form],
case_db,
CaseProcessingConfig(
strict_asserts=True,
case_id_blacklist=case_id_blacklist
)
)
updated_log = get_properly_wrapped_sync_log(sync_log._id)
update_sync_log_with_checks(updated_log, xform, cases, case_db,
case_id_blacklist=case_id_blacklist)
def deprecated_synclog_from_restore_payload(restore_payload):
"""DEPRECATED use <MockDevice>.sync().get_log()"""
return get_properly_wrapped_sync_log(
deprecated_synclog_id_from_restore_payload(restore_payload))
def synclog_from_restore_payload(restore_payload):
return get_properly_wrapped_sync_log(synclog_id_from_restore_payload(restore_payload))
def deprecated_synclog_from_restore_payload(restore_payload):
"""DEPRECATED use <MockDevice>.sync().get_log()"""
return get_properly_wrapped_sync_log(
deprecated_synclog_id_from_restore_payload(restore_payload))
'cases': [case.case_id for case in cases]
})
if e.case_id and e.case_id not in case_id_blacklist:
form_ids = get_case_xform_ids(e.case_id)
case_id_blacklist.append(e.case_id)
for form_id in form_ids:
if form_id != xform._id:
form = XFormInstance.get(form_id)
if form.doc_type == 'XFormInstance':
from casexml.apps.case.xform import process_cases_with_casedb
process_cases_with_casedb(
[form], case_db,
CaseProcessingConfig(
strict_asserts=True,
case_id_blacklist=case_id_blacklist))
updated_log = get_properly_wrapped_sync_log(sync_log._id)
update_sync_log_with_checks(updated_log,
xform,
cases,
case_db,
case_id_blacklist=case_id_blacklist)
def get_indexed_cases(domain, case_ids):
"""
Given a base list of cases, gets all wrapped cases that they reference
(parent cases).
"""
from casexml.apps.case.models import CommCareCase
return [
