def test_dequeuing_update_rmcs_last_saved_by(self):
self.assertFalse(
RecordMaxCounter.objects.filter(
instance_id=self.current_id.id).exists())
with connection.cursor() as cursor:
current_id = InstanceIDModel.get_current_instance_and_increment_counter(
)
_dequeuing_update_rmcs_last_saved_by(
cursor, current_id,
self.data['sc'].current_transfer_session.id)
self.assertTrue(
RecordMaxCounter.objects.filter(
instance_id=current_id.id).exists())
def test_dequeuing_merge_conflict_hard_delete(self):
store = Store.objects.get(id=self.data['model7'])
self.assertEqual(store.serialized, "store")
self.assertEqual(store.conflicting_serialized_data, "store")
with connection.cursor() as cursor:
current_id = InstanceIDModel.get_current_instance_and_increment_counter(
)
DBBackend._dequeuing_merge_conflict_buffer(
cursor, current_id,
self.data['sc'].current_transfer_session.id)
store.refresh_from_db()
self.assertEqual(store.serialized, "")
self.assertEqual(store.conflicting_serialized_data, "")
def test_dequeuing_merge_conflict_buffer_rmcb_less_rmc(self):
store = Store.objects.get(id=self.data['model5'])
self.assertNotEqual(store.last_saved_instance, self.current_id.id)
self.assertEqual(store.conflicting_serialized_data, "store")
with connection.cursor() as cursor:
current_id = InstanceIDModel.get_current_instance_and_increment_counter(
)
_dequeuing_merge_conflict_buffer(
cursor, current_id,
self.data['sc'].current_transfer_session.id)
store = Store.objects.get(id=self.data['model5'])
self.assertEqual(store.last_saved_instance, current_id.id)
self.assertEqual(store.last_saved_counter, current_id.counter)
self.assertEqual(store.conflicting_serialized_data, "buffer\nstore")
def _dequeue_into_store(transfersession):
"""
Takes data from the buffers and merges into the store and record max counters.
"""
with connection.cursor() as cursor:
DBBackend._dequeuing_delete_rmcb_records(cursor, transfersession.id)
DBBackend._dequeuing_delete_buffered_records(cursor, transfersession.id)
current_id = InstanceIDModel.get_current_instance_and_increment_counter()
DBBackend._dequeuing_merge_conflict_buffer(cursor, current_id, transfersession.id)
DBBackend._dequeuing_merge_conflict_rmcb(cursor, transfersession.id)
DBBackend._dequeuing_update_rmcs_last_saved_by(cursor, current_id, transfersession.id)
DBBackend._dequeuing_delete_mc_rmcb(cursor, transfersession.id)
DBBackend._dequeuing_delete_mc_buffer(cursor, transfersession.id)
DBBackend._dequeuing_insert_remaining_buffer(cursor, transfersession.id)
DBBackend._dequeuing_insert_remaining_rmcb(cursor, transfersession.id)
DBBackend._dequeuing_delete_remaining_rmcb(cursor, transfersession.id)
DBBackend._dequeuing_delete_remaining_buffer(cursor, transfersession.id)
if getattr(settings, 'MORANGO_DESERIALIZE_AFTER_DEQUEUING', True):
_deserialize_from_store(transfersession.sync_session.profile)
def _serialize_into_store(profile, filter=None):
"""
Takes data from app layer and serializes the models into the store.
"""
# ensure that we write and retrieve the counter in one go for consistency
current_id = InstanceIDModel.get_current_instance_and_increment_counter()
with transaction.atomic():
# create Q objects for filtering by prefixes
prefix_condition = None
if filter:
prefix_condition = functools.reduce(lambda x, y: x | y, [
Q(_morango_partition__startswith=prefix) for prefix in filter
])
# filter through all models with the dirty bit turned on
syncable_dict = _profile_models[profile]
for (_, klass_model) in iteritems(syncable_dict):
new_store_records = []
new_rmc_records = []
klass_queryset = klass_model.objects.filter(
_morango_dirty_bit=True)
if prefix_condition:
klass_queryset = klass_queryset.filter(prefix_condition)
for app_model in klass_queryset:
try:
store_model = Store.objects.get(id=app_model.id)
# if store record dirty and app record dirty, append store serialized to conflicting data
if store_model.dirty_bit:
store_model.conflicting_serialized_data = store_model.serialized + "\n" + store_model.conflicting_serialized_data
store_model.dirty_bit = False
# set new serialized data on this store model
ser_dict = json.loads(store_model.serialized)
ser_dict.update(app_model.serialize())
store_model.serialized = DjangoJSONEncoder().encode(
ser_dict)
# create or update instance and counter on the record max counter for this store model
RecordMaxCounter.objects.update_or_create(
defaults={'counter': current_id.counter},
instance_id=current_id.id,
store_model_id=store_model.id)
# update last saved bys for this store model
store_model.last_saved_instance = current_id.id
store_model.last_saved_counter = current_id.counter
store_model.deleted = False
# update fields for this store model
store_model.save(update_fields=[
'serialized', 'last_saved_instance',
'last_saved_counter', 'conflicting_serialized_data',
'deleted'
])
except Store.DoesNotExist:
kwargs = {
'id':
app_model.id,
'serialized':
DjangoJSONEncoder().encode(app_model.serialize()),
'last_saved_instance':
current_id.id,
'last_saved_counter':
current_id.counter,
'model_name':
app_model.morango_model_name,
'profile':
app_model.morango_profile,
'partition':
app_model._morango_partition,
'source_id':
app_model._morango_source_id,
}
# check if model has FK pointing to itand add the value to a field on the store
self_ref_fk = _self_referential_fk(klass_model)
if self_ref_fk:
self_ref_fk_value = getattr(app_model, self_ref_fk)
kwargs.update(
{'_self_ref_fk': self_ref_fk_value or ''})
# create store model and record max counter for the app model
new_store_records.append(Store(**kwargs))
new_rmc_records.append(
RecordMaxCounter(store_model_id=app_model.id,
instance_id=current_id.id,
counter=current_id.counter))
# bulk create store and rmc records for this class
Store.objects.bulk_create(new_store_records)
RecordMaxCounter.objects.bulk_create(new_rmc_records)
# set dirty bit to false for all instances of this model
klass_queryset.update(update_dirty_bit_to=False)
# get list of ids of deleted models
deleted_ids = DeletedModels.objects.filter(
profile=profile).values_list('id', flat=True)
# update last_saved_bys and deleted flag of all deleted store model instances
deleted_store_records = Store.objects.filter(id__in=deleted_ids)
deleted_store_records.update(dirty_bit=False,
deleted=True,
last_saved_instance=current_id.id,
last_saved_counter=current_id.counter)
# update rmcs counters for deleted models that have our instance id
RecordMaxCounter.objects.filter(
instance_id=current_id.id,
store_model_id__in=deleted_ids).update(counter=current_id.counter)
# get a list of deleted model ids that don't have an rmc for our instance id
new_rmc_ids = deleted_store_records.exclude(
recordmaxcounter__instance_id=current_id.id).values_list("id",
flat=True)
# bulk create these new rmcs
RecordMaxCounter.objects.bulk_create([
RecordMaxCounter(store_model_id=r_id,
instance_id=current_id.id,
counter=current_id.counter)
for r_id in new_rmc_ids
])
# clear deleted models table for this profile
DeletedModels.objects.filter(profile=profile).delete()
# update our own database max counters after serialization
if not filter:
DatabaseMaxCounter.objects.update_or_create(
instance_id=current_id.id,
partition="",
defaults={'counter': current_id.counter})
else:
for f in filter:
DatabaseMaxCounter.objects.update_or_create(
instance_id=current_id.id,
partition=f,
defaults={'counter': current_id.counter})