def testCachingResponse(self): log = SyncLog() log.save() self.assertFalse(log.has_cached_payload(V1)) self.assertFalse(log.has_cached_payload(V2)) self.assertEqual(None, log.get_cached_payload(V1)) self.assertEqual(None, log.get_cached_payload(V2)) log.invalidate_cached_payloads() payload = "<node>melting hippo</node>" log.set_cached_payload(payload, V1) self.assertTrue(log.has_cached_payload(V1)) self.assertFalse(log.has_cached_payload(V2)) self.assertEqual(payload, log.get_cached_payload(V1)) self.assertEqual(None, log.get_cached_payload(V2)) v2_payload = "<node>melting hippo 2.0</node>" log.set_cached_payload(v2_payload, V2) self.assertTrue(log.has_cached_payload(V1)) self.assertTrue(log.has_cached_payload(V2)) self.assertEqual(payload, log.get_cached_payload(V1)) self.assertEqual(v2_payload, log.get_cached_payload(V2)) log.invalidate_cached_payloads() self.assertFalse(log.has_cached_payload(V1)) self.assertFalse(log.has_cached_payload(V2)) self.assertEqual(None, log.get_cached_payload(V1)) self.assertEqual(None, log.get_cached_payload(V2))
def test_pillow(self): from corehq.apps.change_feed.topics import get_topic_offset from corehq.pillows.synclog import get_user_sync_history_pillow consumer = get_test_kafka_consumer(topics.SYNCLOG_SQL) # get the seq id before the change is published kafka_seq = get_topic_offset(topics.SYNCLOG_SQL) # make sure user has empty reporting-metadata before a sync self.assertEqual(self.ccuser.reporting_metadata.last_syncs, []) # do a sync synclog = SyncLog(domain=self.domain.name, user_id=self.ccuser._id, date=datetime.datetime(2015, 7, 1, 0, 0)) synclog.save() # make sure kafka change updates the user with latest sync info message = next(consumer) change_meta = change_meta_from_kafka_message(message.value) synclog = self._get_latest_synclog() self.assertEqual(change_meta.document_id, synclog._id) self.assertEqual(change_meta.domain, self.domain.name) # make sure processor updates the user correctly pillow = get_user_sync_history_pillow() pillow.process_changes(since=kafka_seq, forever=False) ccuser = CommCareUser.get(self.ccuser._id) self.assertEqual(len(ccuser.reporting_metadata.last_syncs), 1) self.assertEqual(ccuser.reporting_metadata.last_syncs[0].sync_date, synclog.date) self.assertEqual(ccuser.reporting_metadata.last_sync_for_user.sync_date, synclog.date)
def test_sync_log(self): from casexml.apps.phone.models import SyncLog, SimplifiedSyncLog from corehq.apps.users.models import WebUser, CommCareUser from casexml.apps.phone.models import get_sync_log_class_by_format web_user = WebUser.create( domain=self.domain_name, username='******', password='******', email='*****@*****.**', ) mobile_user = CommCareUser.create( self.domain_name, 'mobile_user1', 'secret' ) other_user = CommCareUser.create( 'other_domain', 'mobile_user2', 'secret' ) self.addCleanup(other_user.delete) l1 = SyncLog(user_id=web_user._id) l1.save() l2 = SimplifiedSyncLog(user_id=mobile_user._id) l2.save() other_log = SyncLog(user_id=other_user._id) other_log.save() def _synclog_to_class(doc): if doc['doc_type'] == 'SyncLog': return get_sync_log_class_by_format(doc.get('log_format')) expected_docs = [web_user, mobile_user, l1, l2] not_expected_docs = [other_user, other_log] self._dump_and_load(expected_docs, not_expected_docs, doc_to_doc_class=_synclog_to_class)
def test_default(self): synclog = SyncLog(domain='test', user_id='user1', date=datetime.datetime(2015, 7, 1, 0, 0)) synclog.save() self.assertEqual(self._count(), 1) delete_synclog(synclog._id) self.assertEqual(self._count(), 0)
def test_sync_log_invalidation_bug(self): sync_log = SyncLog(user_id='6dac4940-913e-11e0-9d4b-005056aa7fb5') sync_log.save() self.testAttachInCreate() # this used to fail before we fixed http://manage.dimagi.com/default.asp?158373 self._doSubmitUpdateWithMultimedia(new_attachments=['commcare_logo_file'], removes=[], sync_token=sync_log._id) sync_log.delete()
def get_payload(self): user = self.user last_sync = self.sync_log self.validate() cached_payload = self.get_cached_payload() if cached_payload: return cached_payload sync_operation = user.get_case_updates(last_sync) case_xml_elements = [ xml.get_case_element(op.case, op.required_updates, self.version) for op in sync_operation.actual_cases_to_sync ] commtrack_elements = self.get_stock_payload(sync_operation) last_seq = str(get_db().info()["update_seq"]) # create a sync log for this previous_log_id = last_sync.get_id if last_sync else None synclog = SyncLog( user_id=user.user_id, last_seq=last_seq, owner_ids_on_phone=user.get_owner_ids(), date=datetime.utcnow(), previous_log_id=previous_log_id, cases_on_phone=[CaseState.from_case(c) for c in sync_operation.actual_owned_cases], dependent_cases_on_phone=[CaseState.from_case(c) for c in sync_operation.actual_extended_cases], ) synclog.save(**get_safe_write_kwargs()) # start with standard response response = get_response_element( "Successfully restored account %s!" % user.username, ResponseNature.OTA_RESTORE_SUCCESS ) # add sync token info response.append(xml.get_sync_element(synclog.get_id)) # registration block response.append(xml.get_registration_element(user)) # fixture block for fixture in generator.get_fixtures(user, self.version, last_sync): response.append(fixture) # case blocks for case_elem in case_xml_elements: response.append(case_elem) for ct_elem in commtrack_elements: response.append(ct_elem) if self.items: response.attrib["items"] = "%d" % len(response.getchildren()) resp = xml.tostring(response) self.set_cached_payload_if_enabled(resp) return resp
def create_sync_log(self): previous_log_id = None if self.is_initial else self.last_sync_log._id last_seq = str(get_db().info()["update_seq"]) new_synclog = SyncLog(user_id=self.user.user_id, last_seq=last_seq, owner_ids_on_phone=list(self.owner_ids), date=datetime.utcnow(), previous_log_id=previous_log_id) new_synclog.save(**get_safe_write_kwargs()) return new_synclog
def test_sync_log_invalidation_bug(self): sync_log = SyncLog(user_id='6dac4940-913e-11e0-9d4b-005056aa7fb5') sync_log.save() self.testAttachInCreate() # this used to fail before we fixed http://manage.dimagi.com/default.asp?158373 self._doSubmitUpdateWithMultimedia( new_attachments=['commcare_logo_file'], removes=[], sync_token=sync_log._id) sync_log.delete()
def create_sync_log(self): previous_log_id = None if self.is_initial else self.last_sync_log._id last_seq = str(get_db().info()["update_seq"]) new_synclog = SyncLog( user_id=self.user.user_id, last_seq=last_seq, owner_ids_on_phone=list(self.owner_ids), date=datetime.utcnow(), previous_log_id=previous_log_id ) new_synclog.save(**get_safe_write_kwargs()) return new_synclog
def get_payload(self): """ This function currently returns either a full string payload or a string name of a file that contains the contents of the payload. If FILE_RESTORE toggle is enabled, then this will return the filename, otherwise it will return the full string payload """ user = self.user last_synclog = self.sync_log self.validate() cached_response = self.get_cached_payload() if cached_response.exists(): return cached_response start_time = datetime.utcnow() last_seq = str(get_db().info()["update_seq"]) # create a sync log for this previous_log_id = last_synclog.get_id if last_synclog else None new_synclog = SyncLog( user_id=user.user_id, last_seq=last_seq, owner_ids_on_phone=user.get_owner_ids(), date=datetime.utcnow(), previous_log_id=previous_log_id ) new_synclog.save(**get_safe_write_kwargs()) # start with standard response with get_restore_class(user)(user.username, items=self.items) as response: # add sync token info response.append(xml.get_sync_element(new_synclog.get_id)) # registration block response.append(xml.get_registration_element(user)) # fixture block for fixture in generator.get_fixtures(user, self.version, last_synclog): response.append(fixture) case_response, self.num_batches = get_case_payload_batched( self.domain, self.stock_settings, self.version, user, last_synclog, new_synclog ) combined_response = response + case_response case_response.close() combined_response.finalize() duration = datetime.utcnow() - start_time new_synclog.duration = duration.seconds new_synclog.save() self.set_cached_payload_if_necessary(combined_response, duration) return combined_response
def get_payload(self): user = self.user last_sync = self.sync_log self.validate() cached_payload = self.get_cached_payload() if cached_payload: return cached_payload start_time = datetime.utcnow() last_seq = str(get_db().info()["update_seq"]) # create a sync log for this previous_log_id = last_sync.get_id if last_sync else None synclog = SyncLog( user_id=user.user_id, last_seq=last_seq, owner_ids_on_phone=user.get_owner_ids(), date=datetime.utcnow(), previous_log_id=previous_log_id ) synclog.save(**get_safe_write_kwargs()) # start with standard response batch_enabled = BATCHED_RESTORE.enabled(self.user.domain) or BATCHED_RESTORE.enabled(self.user.username) logger.debug('Batch restore enabled: %s', batch_enabled) if batch_enabled: response = StringRestoreResponse(user.username, items=self.items) else: response = EtreeRestoreResponse(user.username, items=self.items) # add sync token info response.append(xml.get_sync_element(synclog.get_id)) # registration block response.append(xml.get_registration_element(user)) # fixture block for fixture in generator.get_fixtures(user, self.version, last_sync): response.append(fixture) payload_fn = self._get_case_payload_batched if batch_enabled else self._get_case_payload response = payload_fn(response, user, last_sync, synclog) resp = str(response) duration = datetime.utcnow() - start_time synclog.duration = duration.seconds synclog.save() add_custom_parameter('restore_response_size', response.num_items) self.set_cached_payload_if_necessary(resp, duration) return resp
for op in sync_operation.actual_cases_to_sync] commtrack_elements = self.get_stock_payload(sync_operation) last_seq = str(get_db().info()["update_seq"]) # create a sync log for this previous_log_id = last_sync.get_id if last_sync else None synclog = SyncLog(user_id=user.user_id, last_seq=last_seq, owner_ids_on_phone=user.get_owner_ids(), date=datetime.utcnow(), previous_log_id=previous_log_id, cases_on_phone=[CaseState.from_case(c) for c in \ sync_operation.actual_owned_cases], dependent_cases_on_phone=[CaseState.from_case(c) for c in \ sync_operation.actual_extended_cases]) synclog.save(**get_safe_write_kwargs()) # start with standard response response = get_response_element( "Successfully restored account %s!" % user.username, ResponseNature.OTA_RESTORE_SUCCESS) # add sync token info response.append(xml.get_sync_element(synclog.get_id)) # registration block response.append(xml.get_registration_element(user)) # fixture block for fixture in generator.get_fixtures(user, self.version, last_sync): response.append(fixture) # case blocks for case_elem in case_xml_elements:
def generate_restore_payload(user, restore_id="", version="1.0", state_hash=""): """ Gets an XML payload suitable for OTA restore. If you need to do something other than find all cases matching user_id = user.user_id then you have to pass in a user object that overrides the get_case_updates() method. It should match the same signature as models.user.get_case_updates(): user: who the payload is for. must implement get_case_updates restore_id: sync token version: the CommCare version returns: the xml payload of the sync operation """ check_version(version) last_sync = None if restore_id: try: last_sync = SyncLog.get(restore_id) except Exception: logging.error("Request for bad sync log %s by %s, ignoring..." % (restore_id, user)) if last_sync and state_hash: parsed_hash = CaseStateHash.parse(state_hash) if last_sync.get_state_hash() != parsed_hash: raise BadStateException(expected=last_sync.get_state_hash(), actual=parsed_hash, case_ids=last_sync.get_footprint_of_cases_on_phone()) sync_operation = user.get_case_updates(last_sync) case_xml_elements = [xml.get_case_element(op.case, op.required_updates, version) \ for op in sync_operation.actual_cases_to_sync] last_seq = get_db().info()["update_seq"] # create a sync log for this previous_log_id = last_sync.get_id if last_sync else None synclog = SyncLog(user_id=user.user_id, last_seq=last_seq, owner_ids_on_phone=user.get_owner_ids(), date=datetime.utcnow(), previous_log_id=previous_log_id, cases_on_phone=[CaseState.from_case(c) for c in \ sync_operation.actual_owned_cases], dependent_cases_on_phone=[CaseState.from_case(c) for c in \ sync_operation.actual_extended_cases]) synclog.save() # start with standard response response = get_response_element( "Successfully restored account %s!" % user.username, ResponseNature.OTA_RESTORE_SUCCESS) # add sync token info response.append(xml.get_sync_element(synclog.get_id)) # registration block response.append(xml.get_registration_element(user)) # fixture block for fixture in generator.get_fixtures(user, version, last_sync): response.append(fixture) # case blocks for case_elem in case_xml_elements: response.append(case_elem) return xml.tostring(response)