def test_dequeuing_update_rmcs_last_saved_by(self): self.assertFalse( RecordMaxCounter.objects.filter( instance_id=self.current_id.id).exists()) with connection.cursor() as cursor: current_id = InstanceIDModel.get_current_instance_and_increment_counter( ) _dequeuing_update_rmcs_last_saved_by( cursor, current_id, self.data['sc'].current_transfer_session.id) self.assertTrue( RecordMaxCounter.objects.filter( instance_id=current_id.id).exists())
def test_dequeuing_merge_conflict_hard_delete(self): store = Store.objects.get(id=self.data['model7']) self.assertEqual(store.serialized, "store") self.assertEqual(store.conflicting_serialized_data, "store") with connection.cursor() as cursor: current_id = InstanceIDModel.get_current_instance_and_increment_counter( ) DBBackend._dequeuing_merge_conflict_buffer( cursor, current_id, self.data['sc'].current_transfer_session.id) store.refresh_from_db() self.assertEqual(store.serialized, "") self.assertEqual(store.conflicting_serialized_data, "")
def test_dequeuing_merge_conflict_buffer_rmcb_less_rmc(self): store = Store.objects.get(id=self.data['model5']) self.assertNotEqual(store.last_saved_instance, self.current_id.id) self.assertEqual(store.conflicting_serialized_data, "store") with connection.cursor() as cursor: current_id = InstanceIDModel.get_current_instance_and_increment_counter( ) _dequeuing_merge_conflict_buffer( cursor, current_id, self.data['sc'].current_transfer_session.id) store = Store.objects.get(id=self.data['model5']) self.assertEqual(store.last_saved_instance, current_id.id) self.assertEqual(store.last_saved_counter, current_id.counter) self.assertEqual(store.conflicting_serialized_data, "buffer\nstore")
def _dequeue_into_store(transfersession): """ Takes data from the buffers and merges into the store and record max counters. """ with connection.cursor() as cursor: DBBackend._dequeuing_delete_rmcb_records(cursor, transfersession.id) DBBackend._dequeuing_delete_buffered_records(cursor, transfersession.id) current_id = InstanceIDModel.get_current_instance_and_increment_counter() DBBackend._dequeuing_merge_conflict_buffer(cursor, current_id, transfersession.id) DBBackend._dequeuing_merge_conflict_rmcb(cursor, transfersession.id) DBBackend._dequeuing_update_rmcs_last_saved_by(cursor, current_id, transfersession.id) DBBackend._dequeuing_delete_mc_rmcb(cursor, transfersession.id) DBBackend._dequeuing_delete_mc_buffer(cursor, transfersession.id) DBBackend._dequeuing_insert_remaining_buffer(cursor, transfersession.id) DBBackend._dequeuing_insert_remaining_rmcb(cursor, transfersession.id) DBBackend._dequeuing_delete_remaining_rmcb(cursor, transfersession.id) DBBackend._dequeuing_delete_remaining_buffer(cursor, transfersession.id) if getattr(settings, 'MORANGO_DESERIALIZE_AFTER_DEQUEUING', True): _deserialize_from_store(transfersession.sync_session.profile)
def _serialize_into_store(profile, filter=None): """ Takes data from app layer and serializes the models into the store. """ # ensure that we write and retrieve the counter in one go for consistency current_id = InstanceIDModel.get_current_instance_and_increment_counter() with transaction.atomic(): # create Q objects for filtering by prefixes prefix_condition = None if filter: prefix_condition = functools.reduce(lambda x, y: x | y, [ Q(_morango_partition__startswith=prefix) for prefix in filter ]) # filter through all models with the dirty bit turned on syncable_dict = _profile_models[profile] for (_, klass_model) in iteritems(syncable_dict): new_store_records = [] new_rmc_records = [] klass_queryset = klass_model.objects.filter( _morango_dirty_bit=True) if prefix_condition: klass_queryset = klass_queryset.filter(prefix_condition) for app_model in klass_queryset: try: store_model = Store.objects.get(id=app_model.id) # if store record dirty and app record dirty, append store serialized to conflicting data if store_model.dirty_bit: store_model.conflicting_serialized_data = store_model.serialized + "\n" + store_model.conflicting_serialized_data store_model.dirty_bit = False # set new serialized data on this store model ser_dict = json.loads(store_model.serialized) ser_dict.update(app_model.serialize()) store_model.serialized = DjangoJSONEncoder().encode( ser_dict) # create or update instance and counter on the record max counter for this store model RecordMaxCounter.objects.update_or_create( defaults={'counter': current_id.counter}, instance_id=current_id.id, store_model_id=store_model.id) # update last saved bys for this store model store_model.last_saved_instance = current_id.id store_model.last_saved_counter = current_id.counter store_model.deleted = False # update fields for this store model store_model.save(update_fields=[ 'serialized', 'last_saved_instance', 'last_saved_counter', 'conflicting_serialized_data', 'deleted' ]) except Store.DoesNotExist: kwargs = { 'id': app_model.id, 'serialized': DjangoJSONEncoder().encode(app_model.serialize()), 'last_saved_instance': current_id.id, 'last_saved_counter': current_id.counter, 'model_name': app_model.morango_model_name, 'profile': app_model.morango_profile, 'partition': app_model._morango_partition, 'source_id': app_model._morango_source_id, } # check if model has FK pointing to itand add the value to a field on the store self_ref_fk = _self_referential_fk(klass_model) if self_ref_fk: self_ref_fk_value = getattr(app_model, self_ref_fk) kwargs.update( {'_self_ref_fk': self_ref_fk_value or ''}) # create store model and record max counter for the app model new_store_records.append(Store(**kwargs)) new_rmc_records.append( RecordMaxCounter(store_model_id=app_model.id, instance_id=current_id.id, counter=current_id.counter)) # bulk create store and rmc records for this class Store.objects.bulk_create(new_store_records) RecordMaxCounter.objects.bulk_create(new_rmc_records) # set dirty bit to false for all instances of this model klass_queryset.update(update_dirty_bit_to=False) # get list of ids of deleted models deleted_ids = DeletedModels.objects.filter( profile=profile).values_list('id', flat=True) # update last_saved_bys and deleted flag of all deleted store model instances deleted_store_records = Store.objects.filter(id__in=deleted_ids) deleted_store_records.update(dirty_bit=False, deleted=True, last_saved_instance=current_id.id, last_saved_counter=current_id.counter) # update rmcs counters for deleted models that have our instance id RecordMaxCounter.objects.filter( instance_id=current_id.id, store_model_id__in=deleted_ids).update(counter=current_id.counter) # get a list of deleted model ids that don't have an rmc for our instance id new_rmc_ids = deleted_store_records.exclude( recordmaxcounter__instance_id=current_id.id).values_list("id", flat=True) # bulk create these new rmcs RecordMaxCounter.objects.bulk_create([ RecordMaxCounter(store_model_id=r_id, instance_id=current_id.id, counter=current_id.counter) for r_id in new_rmc_ids ]) # clear deleted models table for this profile DeletedModels.objects.filter(profile=profile).delete() # update our own database max counters after serialization if not filter: DatabaseMaxCounter.objects.update_or_create( instance_id=current_id.id, partition="", defaults={'counter': current_id.counter}) else: for f in filter: DatabaseMaxCounter.objects.update_or_create( instance_id=current_id.id, partition=f, defaults={'counter': current_id.counter})