def test_compression_correctness(): addstuff() changestuff() session = Session() ops = compressed_operations(session.query(models.Operation).all()) groups = group_by(lambda op: (op.content_type_id, op.row_id), ops) for g in groups.itervalues(): logging.info(g) assert len(g) == 1 # assert correctness when compressing operations from a pull # message pull_ops = [ models.Operation(command='i', content_type_id=1, row_id=1, order=1), models.Operation(command='d', content_type_id=1, row_id=1, order=2), models.Operation(command='i', content_type_id=1, row_id=1, order=3), models.Operation(command='u', content_type_id=1, row_id=1, order=4), # result of above should be a single 'i' models.Operation(command='u', content_type_id=2, row_id=1, order=5), models.Operation(command='d', content_type_id=2, row_id=1, order=6), models.Operation(command='i', content_type_id=2, row_id=1, order=7), models.Operation(command='d', content_type_id=2, row_id=1, order=8), # result of above should be a single 'd' models.Operation(command='d', content_type_id=3, row_id=1, order=9), models.Operation(command='i', content_type_id=3, row_id=1, order=10), # result of above should be an 'u' models.Operation(command='i', content_type_id=4, row_id=1, order=11), models.Operation(command='u', content_type_id=4, row_id=1, order=12), models.Operation(command='d', content_type_id=4, row_id=1, order=13), # result of above should be no operations models.Operation(command='d', content_type_id=5, row_id=1, order=14), models.Operation(command='i', content_type_id=5, row_id=1, order=15), models.Operation(command='d', content_type_id=5, row_id=1, order=16), # result of above should be a single 'd' models.Operation(command='u', content_type_id=6, row_id=1, order=17), models.Operation(command='d', content_type_id=6, row_id=1, order=18), models.Operation(command='i', content_type_id=6, row_id=1, order=19), # result of above should be an 'u' models.Operation(command='d', content_type_id=7, row_id=1, order=20), models.Operation(command='i', content_type_id=7, row_id=1, order=21), models.Operation(command='u', content_type_id=7, row_id=1, order=22) # result of above should be an 'u' ] compressed = compressed_operations(pull_ops) logging.info("len(compressed) == {0}".format(len(compressed))) logging.info("\n".join(repr(op) for op in compressed)) assert len(compressed) == 6 assert compressed[0].command == 'i' assert compressed[1].command == 'd' assert compressed[2].command == 'u' assert compressed[3].command == 'd' assert compressed[4].command == 'u' assert compressed[5].command == 'u'
def test_compression_correctness(): addstuff() changestuff() session = Session() ops = compressed_operations(session.query(models.Operation).all()) groups = group_by(lambda op: (op.content_type_id, op.row_id), ops) for g in groups.values(): logging.info(g) assert len(g) == 1 # assert correctness when compressing operations from a pull # message pull_ops = [ models.Operation(command='i', content_type_id=1, row_id=1, order=1), models.Operation(command='d', content_type_id=1, row_id=1, order=2), models.Operation(command='i', content_type_id=1, row_id=1, order=3), models.Operation(command='u', content_type_id=1, row_id=1, order=4), # result of above should be a single 'i' models.Operation(command='u', content_type_id=2, row_id=1, order=5), models.Operation(command='d', content_type_id=2, row_id=1, order=6), models.Operation(command='i', content_type_id=2, row_id=1, order=7), models.Operation(command='d', content_type_id=2, row_id=1, order=8), # result of above should be a single 'd' models.Operation(command='d', content_type_id=3, row_id=1, order=9), models.Operation(command='i', content_type_id=3, row_id=1, order=10), # result of above should be an 'u' models.Operation(command='i', content_type_id=4, row_id=1, order=11), models.Operation(command='u', content_type_id=4, row_id=1, order=12), models.Operation(command='d', content_type_id=4, row_id=1, order=13), # result of above should be no operations models.Operation(command='d', content_type_id=5, row_id=1, order=14), models.Operation(command='i', content_type_id=5, row_id=1, order=15), models.Operation(command='d', content_type_id=5, row_id=1, order=16), # result of above should be a single 'd' models.Operation(command='u', content_type_id=6, row_id=1, order=17), models.Operation(command='d', content_type_id=6, row_id=1, order=18), models.Operation(command='i', content_type_id=6, row_id=1, order=19), # result of above should be an 'u' models.Operation(command='d', content_type_id=7, row_id=1, order=20), models.Operation(command='i', content_type_id=7, row_id=1, order=21), models.Operation(command='u', content_type_id=7, row_id=1, order=22) # result of above should be an 'u' ] compressed = compressed_operations(pull_ops) logging.info("len(compressed) == {0}".format(len(compressed))) logging.info("\n".join(repr(op) for op in compressed)) assert len(compressed) == 6 assert compressed[0].command == 'i' assert compressed[1].command == 'd' assert compressed[2].command == 'u' assert compressed[3].command == 'd' assert compressed[4].command == 'u' assert compressed[5].command == 'u'
def test_compression_consistency(): addstuff() changestuff() session = Session() ops = session.query(models.Operation).all() compress() news = session.query(models.Operation).order_by(models.Operation.order).all() assert news == compressed_operations(ops)
def test_compression_consistency(): addstuff() changestuff() session = Session() ops = session.query(models.Operation).all() compress() news = session.query(models.Operation).order_by( models.Operation.order).all() assert news == compressed_operations(ops)
def merge(pull_message, session=None): """ Merges a message from the server with the local database. *pull_message* is an instance of dbsync.messages.pull.PullMessage. """ if not isinstance(pull_message, PullMessage): raise TypeError("need an instance of dbsync.messages.pull.PullMessage " "to perform the local merge operation") valid_cts = set(ct for ct in core.synched_models.ids) unversioned_ops = compress(session=session) pull_ops = filter( attr('content_type_id').in_(valid_cts), pull_message.operations) pull_ops = compressed_operations(pull_ops) # I) first phase: resolve unique constraint conflicts if # possible. Abort early if a human error is detected unique_conflicts, unique_errors = find_unique_conflicts( pull_ops, unversioned_ops, pull_message, session) if unique_errors: raise UniqueConstraintError(unique_errors) conflicting_objects = set() for uc in unique_conflicts: obj = uc['object'] conflicting_objects.add(obj) for key, value in izip(uc['columns'], uc['new_values']): setattr(obj, key, value) # Resolve potential cyclical conflicts by deleting and reinserting for obj in conflicting_objects: make_transient(obj) # remove from session for model in set(type(obj) for obj in conflicting_objects): pk_name = get_pk(model) pks = [ getattr(obj, pk_name) for obj in conflicting_objects if type(obj) is model ] session.query(model).filter(getattr(model, pk_name).in_(pks)).\ delete(synchronize_session=False) # remove from the database session.add_all(conflicting_objects) # reinsert them session.flush() # II) second phase: detect conflicts between pulled operations and # unversioned ones direct_conflicts = find_direct_conflicts(pull_ops, unversioned_ops) # in which the delete operation is registered on the pull message dependency_conflicts = find_dependency_conflicts(pull_ops, unversioned_ops, session) # in which the delete operation was performed locally reversed_dependency_conflicts = find_reversed_dependency_conflicts( pull_ops, unversioned_ops, pull_message) insert_conflicts = find_insert_conflicts(pull_ops, unversioned_ops) # III) third phase: perform pull operations, when allowed and # while resolving conflicts def extract(op, conflicts): return [local for remote, local in conflicts if remote is op] def purgelocal(local): session.delete(local) exclude = lambda tup: tup[1] is not local mfilter(exclude, direct_conflicts) mfilter(exclude, dependency_conflicts) mfilter(exclude, reversed_dependency_conflicts) mfilter(exclude, insert_conflicts) unversioned_ops.remove(local) for pull_op in pull_ops: # flag to control whether the remote operation is free of obstacles can_perform = True # flag to detect the early exclusion of a remote operation reverted = False # the class of the operation class_ = pull_op.tracked_model direct = extract(pull_op, direct_conflicts) if direct: if pull_op.command == 'd': can_perform = False for local in direct: pair = (pull_op.command, local.command) if pair == ('u', 'u'): can_perform = False # favor local changes over remote ones elif pair == ('u', 'd'): pull_op.command = 'i' # negate the local delete purgelocal(local) elif pair == ('d', 'u'): local.command = 'i' # negate the remote delete session.flush() reverted = True else: # ('d', 'd') purgelocal(local) dependency = extract(pull_op, dependency_conflicts) if dependency and not reverted: can_perform = False order = min(op.order for op in unversioned_ops) # first move all operations further in order, to make way # for the new one for op in unversioned_ops: op.order = op.order + 1 session.flush() # then create operation to reflect the reinsertion and # maintain a correct operation history session.add( Operation(row_id=pull_op.row_id, content_type_id=pull_op.content_type_id, command='i', order=order)) reversed_dependency = extract(pull_op, reversed_dependency_conflicts) for local in reversed_dependency: # reinsert record local.command = 'i' local.perform(pull_message, session) # delete trace of deletion purgelocal(local) insert = extract(pull_op, insert_conflicts) for local in insert: session.flush() next_id = max(max_remote(class_, pull_message), max_local(class_, session)) + 1 update_local_id(local.row_id, next_id, class_, session) local.row_id = next_id if can_perform: pull_op.perform(pull_message, session) session.flush() # IV) fourth phase: insert versions from the pull_message for pull_version in pull_message.versions: session.add(pull_version)
def merge(pull_message, session=None): """ Merges a message from the server with the local database. *pull_message* is an instance of dbsync.messages.pull.PullMessage. """ if not isinstance(pull_message, PullMessage): raise TypeError("need an instance of dbsync.messages.pull.PullMessage " "to perform the local merge operation") valid_cts = set(ct for ct in core.synched_models.ids) unversioned_ops = compress(session=session) pull_ops = filter(attr('content_type_id').in_(valid_cts), pull_message.operations) pull_ops = compressed_operations(pull_ops) # I) first phase: resolve unique constraint conflicts if # possible. Abort early if a human error is detected unique_conflicts, unique_errors = find_unique_conflicts( pull_ops, unversioned_ops, pull_message, session) if unique_errors: raise UniqueConstraintError(unique_errors) conflicting_objects = set() for uc in unique_conflicts: obj = uc['object'] conflicting_objects.add(obj) for key, value in izip(uc['columns'], uc['new_values']): setattr(obj, key, value) # Resolve potential cyclical conflicts by deleting and reinserting for obj in conflicting_objects: make_transient(obj) # remove from session for model in set(type(obj) for obj in conflicting_objects): pk_name = get_pk(model) pks = [getattr(obj, pk_name) for obj in conflicting_objects if type(obj) is model] session.query(model).filter(getattr(model, pk_name).in_(pks)).\ delete(synchronize_session=False) # remove from the database session.add_all(conflicting_objects) # reinsert them session.flush() # II) second phase: detect conflicts between pulled operations and # unversioned ones direct_conflicts = find_direct_conflicts(pull_ops, unversioned_ops) # in which the delete operation is registered on the pull message dependency_conflicts = find_dependency_conflicts( pull_ops, unversioned_ops, session) # in which the delete operation was performed locally reversed_dependency_conflicts = find_reversed_dependency_conflicts( pull_ops, unversioned_ops, pull_message) insert_conflicts = find_insert_conflicts(pull_ops, unversioned_ops) # III) third phase: perform pull operations, when allowed and # while resolving conflicts def extract(op, conflicts): return [local for remote, local in conflicts if remote is op] def purgelocal(local): session.delete(local) exclude = lambda tup: tup[1] is not local mfilter(exclude, direct_conflicts) mfilter(exclude, dependency_conflicts) mfilter(exclude, reversed_dependency_conflicts) mfilter(exclude, insert_conflicts) unversioned_ops.remove(local) for pull_op in pull_ops: # flag to control whether the remote operation is free of obstacles can_perform = True # flag to detect the early exclusion of a remote operation reverted = False # the class of the operation class_ = pull_op.tracked_model direct = extract(pull_op, direct_conflicts) if direct: if pull_op.command == 'd': can_perform = False for local in direct: pair = (pull_op.command, local.command) if pair == ('u', 'u'): can_perform = False # favor local changes over remote ones elif pair == ('u', 'd'): pull_op.command = 'i' # negate the local delete purgelocal(local) elif pair == ('d', 'u'): local.command = 'i' # negate the remote delete session.flush() reverted = True else: # ('d', 'd') purgelocal(local) dependency = extract(pull_op, dependency_conflicts) if dependency and not reverted: can_perform = False order = min(op.order for op in unversioned_ops) # first move all operations further in order, to make way # for the new one for op in unversioned_ops: op.order = op.order + 1 session.flush() # then create operation to reflect the reinsertion and # maintain a correct operation history session.add(Operation(row_id=pull_op.row_id, content_type_id=pull_op.content_type_id, command='i', order=order)) reversed_dependency = extract(pull_op, reversed_dependency_conflicts) for local in reversed_dependency: # reinsert record local.command = 'i' local.perform(pull_message, session) # delete trace of deletion purgelocal(local) insert = extract(pull_op, insert_conflicts) for local in insert: session.flush() next_id = max(max_remote(class_, pull_message), max_local(class_, session)) + 1 update_local_id(local.row_id, next_id, class_, session) local.row_id = next_id if can_perform: pull_op.perform(pull_message, session) session.flush() # IV) fourth phase: insert versions from the pull_message for pull_version in pull_message.versions: session.add(pull_version)