def testCachingResponse(self):
log = SyncLog()
log.save()
self.assertFalse(log.has_cached_payload(V1))
self.assertFalse(log.has_cached_payload(V2))
self.assertEqual(None, log.get_cached_payload(V1))
self.assertEqual(None, log.get_cached_payload(V2))
log.invalidate_cached_payloads()
payload = "<node>melting hippo</node>"
log.set_cached_payload(payload, V1)
self.assertTrue(log.has_cached_payload(V1))
self.assertFalse(log.has_cached_payload(V2))
self.assertEqual(payload, log.get_cached_payload(V1))
self.assertEqual(None, log.get_cached_payload(V2))
v2_payload = "<node>melting hippo 2.0</node>"
log.set_cached_payload(v2_payload, V2)
self.assertTrue(log.has_cached_payload(V1))
self.assertTrue(log.has_cached_payload(V2))
self.assertEqual(payload, log.get_cached_payload(V1))
self.assertEqual(v2_payload, log.get_cached_payload(V2))
log.invalidate_cached_payloads()
self.assertFalse(log.has_cached_payload(V1))
self.assertFalse(log.has_cached_payload(V2))
self.assertEqual(None, log.get_cached_payload(V1))
self.assertEqual(None, log.get_cached_payload(V2))
def test_pillow(self):
from corehq.apps.change_feed.topics import get_topic_offset
from corehq.pillows.synclog import get_user_sync_history_pillow
consumer = get_test_kafka_consumer(topics.SYNCLOG_SQL)
# get the seq id before the change is published
kafka_seq = get_topic_offset(topics.SYNCLOG_SQL)
# make sure user has empty reporting-metadata before a sync
self.assertEqual(self.ccuser.reporting_metadata.last_syncs, [])
# do a sync
synclog = SyncLog(domain=self.domain.name, user_id=self.ccuser._id,
date=datetime.datetime(2015, 7, 1, 0, 0))
synclog.save()
# make sure kafka change updates the user with latest sync info
message = next(consumer)
change_meta = change_meta_from_kafka_message(message.value)
synclog = self._get_latest_synclog()
self.assertEqual(change_meta.document_id, synclog._id)
self.assertEqual(change_meta.domain, self.domain.name)
# make sure processor updates the user correctly
pillow = get_user_sync_history_pillow()
pillow.process_changes(since=kafka_seq, forever=False)
ccuser = CommCareUser.get(self.ccuser._id)
self.assertEqual(len(ccuser.reporting_metadata.last_syncs), 1)
self.assertEqual(ccuser.reporting_metadata.last_syncs[0].sync_date, synclog.date)
self.assertEqual(ccuser.reporting_metadata.last_sync_for_user.sync_date, synclog.date)
def test_sync_log(self):
from casexml.apps.phone.models import SyncLog, SimplifiedSyncLog
from corehq.apps.users.models import WebUser, CommCareUser
from casexml.apps.phone.models import get_sync_log_class_by_format
web_user = WebUser.create(
domain=self.domain_name,
username='******',
password='******',
email='*****@*****.**',
)
mobile_user = CommCareUser.create(
self.domain_name, 'mobile_user1', 'secret'
)
other_user = CommCareUser.create(
'other_domain', 'mobile_user2', 'secret'
)
self.addCleanup(other_user.delete)
l1 = SyncLog(user_id=web_user._id)
l1.save()
l2 = SimplifiedSyncLog(user_id=mobile_user._id)
l2.save()
other_log = SyncLog(user_id=other_user._id)
other_log.save()
def _synclog_to_class(doc):
if doc['doc_type'] == 'SyncLog':
return get_sync_log_class_by_format(doc.get('log_format'))
expected_docs = [web_user, mobile_user, l1, l2]
not_expected_docs = [other_user, other_log]
self._dump_and_load(expected_docs, not_expected_docs, doc_to_doc_class=_synclog_to_class)
def testCachingResponse(self):
log = SyncLog()
log.save()
self.assertFalse(log.has_cached_payload(V1))
self.assertFalse(log.has_cached_payload(V2))
self.assertEqual(None, log.get_cached_payload(V1))
self.assertEqual(None, log.get_cached_payload(V2))
log.invalidate_cached_payloads()
payload = "<node>melting hippo</node>"
log.set_cached_payload(payload, V1)
self.assertTrue(log.has_cached_payload(V1))
self.assertFalse(log.has_cached_payload(V2))
self.assertEqual(payload, log.get_cached_payload(V1))
self.assertEqual(None, log.get_cached_payload(V2))
v2_payload = "<node>melting hippo 2.0</node>"
log.set_cached_payload(v2_payload, V2)
self.assertTrue(log.has_cached_payload(V1))
self.assertTrue(log.has_cached_payload(V2))
self.assertEqual(payload, log.get_cached_payload(V1))
self.assertEqual(v2_payload, log.get_cached_payload(V2))
log.invalidate_cached_payloads()
self.assertFalse(log.has_cached_payload(V1))
self.assertFalse(log.has_cached_payload(V2))
self.assertEqual(None, log.get_cached_payload(V1))
self.assertEqual(None, log.get_cached_payload(V2))
def test_pillow(self):
from corehq.apps.change_feed.topics import get_topic_offset
from corehq.pillows.synclog import get_user_sync_history_pillow
consumer = get_test_kafka_consumer(topics.SYNCLOG_SQL)
# get the seq id before the change is published
kafka_seq = get_topic_offset(topics.SYNCLOG_SQL)
# make sure user has empty reporting-metadata before a sync
self.assertEqual(self.ccuser.reporting_metadata.last_syncs, [])
# do a sync
synclog = SyncLog(domain=self.domain.name, user_id=self.ccuser._id,
date=datetime.datetime(2015, 7, 1, 0, 0))
synclog.save()
# make sure kafka change updates the user with latest sync info
message = next(consumer)
change_meta = change_meta_from_kafka_message(message.value)
synclog = self._get_latest_synclog()
self.assertEqual(change_meta.document_id, synclog._id)
self.assertEqual(change_meta.domain, self.domain.name)
# make sure processor updates the user correctly
pillow = get_user_sync_history_pillow()
pillow.process_changes(since=kafka_seq, forever=False)
ccuser = CommCareUser.get(self.ccuser._id)
self.assertEqual(len(ccuser.reporting_metadata.last_syncs), 1)
self.assertEqual(ccuser.reporting_metadata.last_syncs[0].sync_date, synclog.date)
self.assertEqual(ccuser.reporting_metadata.last_sync_for_user.sync_date, synclog.date)
def test_sync_log(self):
from casexml.apps.phone.models import SyncLog, SimplifiedSyncLog
from corehq.apps.users.models import WebUser, CommCareUser
from casexml.apps.phone.models import get_sync_log_class_by_format
web_user = WebUser.create(
domain=self.domain_name,
username='******',
password='******',
email='*****@*****.**',
)
mobile_user = CommCareUser.create(
self.domain_name, 'mobile_user1', 'secret'
)
other_user = CommCareUser.create(
'other_domain', 'mobile_user2', 'secret'
)
self.addCleanup(other_user.delete)
l1 = SyncLog(user_id=web_user._id)
l1.save()
l2 = SimplifiedSyncLog(user_id=mobile_user._id)
l2.save()
other_log = SyncLog(user_id=other_user._id)
other_log.save()
def _synclog_to_class(doc):
if doc['doc_type'] == 'SyncLog':
return get_sync_log_class_by_format(doc.get('log_format'))
expected_docs = [web_user, mobile_user, l1, l2]
not_expected_docs = [other_user, other_log]
self._dump_and_load(expected_docs, not_expected_docs, doc_to_doc_class=_synclog_to_class)
def test_default(self):
synclog = SyncLog(domain='test',
user_id='user1',
date=datetime.datetime(2015, 7, 1, 0, 0))
synclog.save()
self.assertEqual(self._count(), 1)
delete_synclog(synclog._id)
self.assertEqual(self._count(), 0)
def test_sync_log_invalidation_bug(self):
sync_log = SyncLog(user_id='6dac4940-913e-11e0-9d4b-005056aa7fb5')
sync_log.save()
self.testAttachInCreate()
# this used to fail before we fixed http://manage.dimagi.com/default.asp?158373
self._doSubmitUpdateWithMultimedia(new_attachments=['commcare_logo_file'], removes=[],
sync_token=sync_log._id)
sync_log.delete()
def get_payload(self):
user = self.user
last_sync = self.sync_log
self.validate()
cached_payload = self.get_cached_payload()
if cached_payload:
return cached_payload
sync_operation = user.get_case_updates(last_sync)
case_xml_elements = [
xml.get_case_element(op.case, op.required_updates, self.version)
for op in sync_operation.actual_cases_to_sync
]
commtrack_elements = self.get_stock_payload(sync_operation)
last_seq = str(get_db().info()["update_seq"])
# create a sync log for this
previous_log_id = last_sync.get_id if last_sync else None
synclog = SyncLog(
user_id=user.user_id,
last_seq=last_seq,
owner_ids_on_phone=user.get_owner_ids(),
date=datetime.utcnow(),
previous_log_id=previous_log_id,
cases_on_phone=[CaseState.from_case(c) for c in sync_operation.actual_owned_cases],
dependent_cases_on_phone=[CaseState.from_case(c) for c in sync_operation.actual_extended_cases],
)
synclog.save(**get_safe_write_kwargs())
# start with standard response
response = get_response_element(
"Successfully restored account %s!" % user.username, ResponseNature.OTA_RESTORE_SUCCESS
)
# add sync token info
response.append(xml.get_sync_element(synclog.get_id))
# registration block
response.append(xml.get_registration_element(user))
# fixture block
for fixture in generator.get_fixtures(user, self.version, last_sync):
response.append(fixture)
# case blocks
for case_elem in case_xml_elements:
response.append(case_elem)
for ct_elem in commtrack_elements:
response.append(ct_elem)
if self.items:
response.attrib["items"] = "%d" % len(response.getchildren())
resp = xml.tostring(response)
self.set_cached_payload_if_enabled(resp)
return resp
def create_sync_log(self):
previous_log_id = None if self.is_initial else self.last_sync_log._id
last_seq = str(get_db().info()["update_seq"])
new_synclog = SyncLog(user_id=self.user.user_id,
last_seq=last_seq,
owner_ids_on_phone=list(self.owner_ids),
date=datetime.utcnow(),
previous_log_id=previous_log_id)
new_synclog.save(**get_safe_write_kwargs())
return new_synclog
def test_sync_log_invalidation_bug(self):
sync_log = SyncLog(user_id='6dac4940-913e-11e0-9d4b-005056aa7fb5')
sync_log.save()
self.testAttachInCreate()
# this used to fail before we fixed http://manage.dimagi.com/default.asp?158373
self._doSubmitUpdateWithMultimedia(
new_attachments=['commcare_logo_file'],
removes=[],
sync_token=sync_log._id)
sync_log.delete()
def create_sync_log(self):
previous_log_id = None if self.is_initial else self.last_sync_log._id
last_seq = str(get_db().info()["update_seq"])
new_synclog = SyncLog(
user_id=self.user.user_id,
last_seq=last_seq,
owner_ids_on_phone=list(self.owner_ids),
date=datetime.utcnow(),
previous_log_id=previous_log_id
)
new_synclog.save(**get_safe_write_kwargs())
return new_synclog
def get_payload(self):
"""
This function currently returns either a full string payload or a string name of a file
that contains the contents of the payload. If FILE_RESTORE toggle is enabled, then this will return
the filename, otherwise it will return the full string payload
"""
user = self.user
last_synclog = self.sync_log
self.validate()
cached_response = self.get_cached_payload()
if cached_response.exists():
return cached_response
start_time = datetime.utcnow()
last_seq = str(get_db().info()["update_seq"])
# create a sync log for this
previous_log_id = last_synclog.get_id if last_synclog else None
new_synclog = SyncLog(
user_id=user.user_id,
last_seq=last_seq,
owner_ids_on_phone=user.get_owner_ids(),
date=datetime.utcnow(),
previous_log_id=previous_log_id
)
new_synclog.save(**get_safe_write_kwargs())
# start with standard response
with get_restore_class(user)(user.username, items=self.items) as response:
# add sync token info
response.append(xml.get_sync_element(new_synclog.get_id))
# registration block
response.append(xml.get_registration_element(user))
# fixture block
for fixture in generator.get_fixtures(user, self.version, last_synclog):
response.append(fixture)
case_response, self.num_batches = get_case_payload_batched(
self.domain, self.stock_settings, self.version, user, last_synclog, new_synclog
)
combined_response = response + case_response
case_response.close()
combined_response.finalize()
duration = datetime.utcnow() - start_time
new_synclog.duration = duration.seconds
new_synclog.save()
self.set_cached_payload_if_necessary(combined_response, duration)
return combined_response
def get_payload(self):
user = self.user
last_sync = self.sync_log
self.validate()
cached_payload = self.get_cached_payload()
if cached_payload:
return cached_payload
start_time = datetime.utcnow()
last_seq = str(get_db().info()["update_seq"])
# create a sync log for this
previous_log_id = last_sync.get_id if last_sync else None
synclog = SyncLog(
user_id=user.user_id,
last_seq=last_seq,
owner_ids_on_phone=user.get_owner_ids(),
date=datetime.utcnow(),
previous_log_id=previous_log_id
)
synclog.save(**get_safe_write_kwargs())
# start with standard response
batch_enabled = BATCHED_RESTORE.enabled(self.user.domain) or BATCHED_RESTORE.enabled(self.user.username)
logger.debug('Batch restore enabled: %s', batch_enabled)
if batch_enabled:
response = StringRestoreResponse(user.username, items=self.items)
else:
response = EtreeRestoreResponse(user.username, items=self.items)
# add sync token info
response.append(xml.get_sync_element(synclog.get_id))
# registration block
response.append(xml.get_registration_element(user))
# fixture block
for fixture in generator.get_fixtures(user, self.version, last_sync):
response.append(fixture)
payload_fn = self._get_case_payload_batched if batch_enabled else self._get_case_payload
response = payload_fn(response, user, last_sync, synclog)
resp = str(response)
duration = datetime.utcnow() - start_time
synclog.duration = duration.seconds
synclog.save()
add_custom_parameter('restore_response_size', response.num_items)
self.set_cached_payload_if_necessary(resp, duration)
return resp
for op in sync_operation.actual_cases_to_sync]
commtrack_elements = self.get_stock_payload(sync_operation)
last_seq = str(get_db().info()["update_seq"])
# create a sync log for this
previous_log_id = last_sync.get_id if last_sync else None
synclog = SyncLog(user_id=user.user_id, last_seq=last_seq,
owner_ids_on_phone=user.get_owner_ids(),
date=datetime.utcnow(), previous_log_id=previous_log_id,
cases_on_phone=[CaseState.from_case(c) for c in \
sync_operation.actual_owned_cases],
dependent_cases_on_phone=[CaseState.from_case(c) for c in \
sync_operation.actual_extended_cases])
synclog.save(**get_safe_write_kwargs())
# start with standard response
response = get_response_element(
"Successfully restored account %s!" % user.username,
ResponseNature.OTA_RESTORE_SUCCESS)
# add sync token info
response.append(xml.get_sync_element(synclog.get_id))
# registration block
response.append(xml.get_registration_element(user))
# fixture block
for fixture in generator.get_fixtures(user, self.version, last_sync):
response.append(fixture)
# case blocks
for case_elem in case_xml_elements:
def generate_restore_payload(user, restore_id="", version="1.0", state_hash=""):
"""
Gets an XML payload suitable for OTA restore. If you need to do something
other than find all cases matching user_id = user.user_id then you have
to pass in a user object that overrides the get_case_updates() method.
It should match the same signature as models.user.get_case_updates():
user: who the payload is for. must implement get_case_updates
restore_id: sync token
version: the CommCare version
returns: the xml payload of the sync operation
"""
check_version(version)
last_sync = None
if restore_id:
try:
last_sync = SyncLog.get(restore_id)
except Exception:
logging.error("Request for bad sync log %s by %s, ignoring..." % (restore_id, user))
if last_sync and state_hash:
parsed_hash = CaseStateHash.parse(state_hash)
if last_sync.get_state_hash() != parsed_hash:
raise BadStateException(expected=last_sync.get_state_hash(),
actual=parsed_hash,
case_ids=last_sync.get_footprint_of_cases_on_phone())
sync_operation = user.get_case_updates(last_sync)
case_xml_elements = [xml.get_case_element(op.case, op.required_updates, version) \
for op in sync_operation.actual_cases_to_sync]
last_seq = get_db().info()["update_seq"]
# create a sync log for this
previous_log_id = last_sync.get_id if last_sync else None
synclog = SyncLog(user_id=user.user_id, last_seq=last_seq,
owner_ids_on_phone=user.get_owner_ids(),
date=datetime.utcnow(), previous_log_id=previous_log_id,
cases_on_phone=[CaseState.from_case(c) for c in \
sync_operation.actual_owned_cases],
dependent_cases_on_phone=[CaseState.from_case(c) for c in \
sync_operation.actual_extended_cases])
synclog.save()
# start with standard response
response = get_response_element(
"Successfully restored account %s!" % user.username,
ResponseNature.OTA_RESTORE_SUCCESS)
# add sync token info
response.append(xml.get_sync_element(synclog.get_id))
# registration block
response.append(xml.get_registration_element(user))
# fixture block
for fixture in generator.get_fixtures(user, version, last_sync):
response.append(fixture)
# case blocks
for case_elem in case_xml_elements:
response.append(case_elem)
return xml.tostring(response)
