def test_compression_correctness():
addstuff()
changestuff()
session = Session()
ops = compressed_operations(session.query(models.Operation).all())
groups = group_by(lambda op: (op.content_type_id, op.row_id), ops)
for g in groups.itervalues():
logging.info(g)
assert len(g) == 1
# assert correctness when compressing operations from a pull
# message
pull_ops = [
models.Operation(command='i', content_type_id=1, row_id=1, order=1),
models.Operation(command='d', content_type_id=1, row_id=1, order=2),
models.Operation(command='i', content_type_id=1, row_id=1, order=3),
models.Operation(command='u', content_type_id=1, row_id=1, order=4),
# result of above should be a single 'i'
models.Operation(command='u', content_type_id=2, row_id=1, order=5),
models.Operation(command='d', content_type_id=2, row_id=1, order=6),
models.Operation(command='i', content_type_id=2, row_id=1, order=7),
models.Operation(command='d', content_type_id=2, row_id=1, order=8),
# result of above should be a single 'd'
models.Operation(command='d', content_type_id=3, row_id=1, order=9),
models.Operation(command='i', content_type_id=3, row_id=1, order=10),
# result of above should be an 'u'
models.Operation(command='i', content_type_id=4, row_id=1, order=11),
models.Operation(command='u', content_type_id=4, row_id=1, order=12),
models.Operation(command='d', content_type_id=4, row_id=1, order=13),
# result of above should be no operations
models.Operation(command='d', content_type_id=5, row_id=1, order=14),
models.Operation(command='i', content_type_id=5, row_id=1, order=15),
models.Operation(command='d', content_type_id=5, row_id=1, order=16),
# result of above should be a single 'd'
models.Operation(command='u', content_type_id=6, row_id=1, order=17),
models.Operation(command='d', content_type_id=6, row_id=1, order=18),
models.Operation(command='i', content_type_id=6, row_id=1, order=19),
# result of above should be an 'u'
models.Operation(command='d', content_type_id=7, row_id=1, order=20),
models.Operation(command='i', content_type_id=7, row_id=1, order=21),
models.Operation(command='u', content_type_id=7, row_id=1, order=22)
# result of above should be an 'u'
]
compressed = compressed_operations(pull_ops)
logging.info("len(compressed) == {0}".format(len(compressed)))
logging.info("\n".join(repr(op) for op in compressed))
assert len(compressed) == 6
assert compressed[0].command == 'i'
assert compressed[1].command == 'd'
assert compressed[2].command == 'u'
assert compressed[3].command == 'd'
assert compressed[4].command == 'u'
assert compressed[5].command == 'u'
def test_compression_correctness():
addstuff()
changestuff()
session = Session()
ops = compressed_operations(session.query(models.Operation).all())
groups = group_by(lambda op: (op.content_type_id, op.row_id), ops)
for g in groups.values():
logging.info(g)
assert len(g) == 1
# assert correctness when compressing operations from a pull
# message
pull_ops = [
models.Operation(command='i', content_type_id=1, row_id=1, order=1),
models.Operation(command='d', content_type_id=1, row_id=1, order=2),
models.Operation(command='i', content_type_id=1, row_id=1, order=3),
models.Operation(command='u', content_type_id=1, row_id=1, order=4),
# result of above should be a single 'i'
models.Operation(command='u', content_type_id=2, row_id=1, order=5),
models.Operation(command='d', content_type_id=2, row_id=1, order=6),
models.Operation(command='i', content_type_id=2, row_id=1, order=7),
models.Operation(command='d', content_type_id=2, row_id=1, order=8),
# result of above should be a single 'd'
models.Operation(command='d', content_type_id=3, row_id=1, order=9),
models.Operation(command='i', content_type_id=3, row_id=1, order=10),
# result of above should be an 'u'
models.Operation(command='i', content_type_id=4, row_id=1, order=11),
models.Operation(command='u', content_type_id=4, row_id=1, order=12),
models.Operation(command='d', content_type_id=4, row_id=1, order=13),
# result of above should be no operations
models.Operation(command='d', content_type_id=5, row_id=1, order=14),
models.Operation(command='i', content_type_id=5, row_id=1, order=15),
models.Operation(command='d', content_type_id=5, row_id=1, order=16),
# result of above should be a single 'd'
models.Operation(command='u', content_type_id=6, row_id=1, order=17),
models.Operation(command='d', content_type_id=6, row_id=1, order=18),
models.Operation(command='i', content_type_id=6, row_id=1, order=19),
# result of above should be an 'u'
models.Operation(command='d', content_type_id=7, row_id=1, order=20),
models.Operation(command='i', content_type_id=7, row_id=1, order=21),
models.Operation(command='u', content_type_id=7, row_id=1, order=22)
# result of above should be an 'u'
]
compressed = compressed_operations(pull_ops)
logging.info("len(compressed) == {0}".format(len(compressed)))
logging.info("\n".join(repr(op) for op in compressed))
assert len(compressed) == 6
assert compressed[0].command == 'i'
assert compressed[1].command == 'd'
assert compressed[2].command == 'u'
assert compressed[3].command == 'd'
assert compressed[4].command == 'u'
assert compressed[5].command == 'u'
def test_compression_consistency():
addstuff()
changestuff()
session = Session()
ops = session.query(models.Operation).all()
compress()
news = session.query(models.Operation).order_by(models.Operation.order).all()
assert news == compressed_operations(ops)
def test_compression_consistency():
addstuff()
changestuff()
session = Session()
ops = session.query(models.Operation).all()
compress()
news = session.query(models.Operation).order_by(
models.Operation.order).all()
assert news == compressed_operations(ops)
def merge(pull_message, session=None):
"""
Merges a message from the server with the local database.
*pull_message* is an instance of dbsync.messages.pull.PullMessage.
"""
if not isinstance(pull_message, PullMessage):
raise TypeError("need an instance of dbsync.messages.pull.PullMessage "
"to perform the local merge operation")
valid_cts = set(ct for ct in core.synched_models.ids)
unversioned_ops = compress(session=session)
pull_ops = filter(
attr('content_type_id').in_(valid_cts), pull_message.operations)
pull_ops = compressed_operations(pull_ops)
# I) first phase: resolve unique constraint conflicts if
# possible. Abort early if a human error is detected
unique_conflicts, unique_errors = find_unique_conflicts(
pull_ops, unversioned_ops, pull_message, session)
if unique_errors:
raise UniqueConstraintError(unique_errors)
conflicting_objects = set()
for uc in unique_conflicts:
obj = uc['object']
conflicting_objects.add(obj)
for key, value in izip(uc['columns'], uc['new_values']):
setattr(obj, key, value)
# Resolve potential cyclical conflicts by deleting and reinserting
for obj in conflicting_objects:
make_transient(obj) # remove from session
for model in set(type(obj) for obj in conflicting_objects):
pk_name = get_pk(model)
pks = [
getattr(obj, pk_name) for obj in conflicting_objects
if type(obj) is model
]
session.query(model).filter(getattr(model, pk_name).in_(pks)).\
delete(synchronize_session=False) # remove from the database
session.add_all(conflicting_objects) # reinsert them
session.flush()
# II) second phase: detect conflicts between pulled operations and
# unversioned ones
direct_conflicts = find_direct_conflicts(pull_ops, unversioned_ops)
# in which the delete operation is registered on the pull message
dependency_conflicts = find_dependency_conflicts(pull_ops, unversioned_ops,
session)
# in which the delete operation was performed locally
reversed_dependency_conflicts = find_reversed_dependency_conflicts(
pull_ops, unversioned_ops, pull_message)
insert_conflicts = find_insert_conflicts(pull_ops, unversioned_ops)
# III) third phase: perform pull operations, when allowed and
# while resolving conflicts
def extract(op, conflicts):
return [local for remote, local in conflicts if remote is op]
def purgelocal(local):
session.delete(local)
exclude = lambda tup: tup[1] is not local
mfilter(exclude, direct_conflicts)
mfilter(exclude, dependency_conflicts)
mfilter(exclude, reversed_dependency_conflicts)
mfilter(exclude, insert_conflicts)
unversioned_ops.remove(local)
for pull_op in pull_ops:
# flag to control whether the remote operation is free of obstacles
can_perform = True
# flag to detect the early exclusion of a remote operation
reverted = False
# the class of the operation
class_ = pull_op.tracked_model
direct = extract(pull_op, direct_conflicts)
if direct:
if pull_op.command == 'd':
can_perform = False
for local in direct:
pair = (pull_op.command, local.command)
if pair == ('u', 'u'):
can_perform = False # favor local changes over remote ones
elif pair == ('u', 'd'):
pull_op.command = 'i' # negate the local delete
purgelocal(local)
elif pair == ('d', 'u'):
local.command = 'i' # negate the remote delete
session.flush()
reverted = True
else: # ('d', 'd')
purgelocal(local)
dependency = extract(pull_op, dependency_conflicts)
if dependency and not reverted:
can_perform = False
order = min(op.order for op in unversioned_ops)
# first move all operations further in order, to make way
# for the new one
for op in unversioned_ops:
op.order = op.order + 1
session.flush()
# then create operation to reflect the reinsertion and
# maintain a correct operation history
session.add(
Operation(row_id=pull_op.row_id,
content_type_id=pull_op.content_type_id,
command='i',
order=order))
reversed_dependency = extract(pull_op, reversed_dependency_conflicts)
for local in reversed_dependency:
# reinsert record
local.command = 'i'
local.perform(pull_message, session)
# delete trace of deletion
purgelocal(local)
insert = extract(pull_op, insert_conflicts)
for local in insert:
session.flush()
next_id = max(max_remote(class_, pull_message),
max_local(class_, session)) + 1
update_local_id(local.row_id, next_id, class_, session)
local.row_id = next_id
if can_perform:
pull_op.perform(pull_message, session)
session.flush()
# IV) fourth phase: insert versions from the pull_message
for pull_version in pull_message.versions:
session.add(pull_version)
def merge(pull_message, session=None):
"""
Merges a message from the server with the local database.
*pull_message* is an instance of dbsync.messages.pull.PullMessage.
"""
if not isinstance(pull_message, PullMessage):
raise TypeError("need an instance of dbsync.messages.pull.PullMessage "
"to perform the local merge operation")
valid_cts = set(ct for ct in core.synched_models.ids)
unversioned_ops = compress(session=session)
pull_ops = filter(attr('content_type_id').in_(valid_cts),
pull_message.operations)
pull_ops = compressed_operations(pull_ops)
# I) first phase: resolve unique constraint conflicts if
# possible. Abort early if a human error is detected
unique_conflicts, unique_errors = find_unique_conflicts(
pull_ops, unversioned_ops, pull_message, session)
if unique_errors:
raise UniqueConstraintError(unique_errors)
conflicting_objects = set()
for uc in unique_conflicts:
obj = uc['object']
conflicting_objects.add(obj)
for key, value in izip(uc['columns'], uc['new_values']):
setattr(obj, key, value)
# Resolve potential cyclical conflicts by deleting and reinserting
for obj in conflicting_objects:
make_transient(obj) # remove from session
for model in set(type(obj) for obj in conflicting_objects):
pk_name = get_pk(model)
pks = [getattr(obj, pk_name)
for obj in conflicting_objects
if type(obj) is model]
session.query(model).filter(getattr(model, pk_name).in_(pks)).\
delete(synchronize_session=False) # remove from the database
session.add_all(conflicting_objects) # reinsert them
session.flush()
# II) second phase: detect conflicts between pulled operations and
# unversioned ones
direct_conflicts = find_direct_conflicts(pull_ops, unversioned_ops)
# in which the delete operation is registered on the pull message
dependency_conflicts = find_dependency_conflicts(
pull_ops, unversioned_ops, session)
# in which the delete operation was performed locally
reversed_dependency_conflicts = find_reversed_dependency_conflicts(
pull_ops, unversioned_ops, pull_message)
insert_conflicts = find_insert_conflicts(pull_ops, unversioned_ops)
# III) third phase: perform pull operations, when allowed and
# while resolving conflicts
def extract(op, conflicts):
return [local for remote, local in conflicts if remote is op]
def purgelocal(local):
session.delete(local)
exclude = lambda tup: tup[1] is not local
mfilter(exclude, direct_conflicts)
mfilter(exclude, dependency_conflicts)
mfilter(exclude, reversed_dependency_conflicts)
mfilter(exclude, insert_conflicts)
unversioned_ops.remove(local)
for pull_op in pull_ops:
# flag to control whether the remote operation is free of obstacles
can_perform = True
# flag to detect the early exclusion of a remote operation
reverted = False
# the class of the operation
class_ = pull_op.tracked_model
direct = extract(pull_op, direct_conflicts)
if direct:
if pull_op.command == 'd':
can_perform = False
for local in direct:
pair = (pull_op.command, local.command)
if pair == ('u', 'u'):
can_perform = False # favor local changes over remote ones
elif pair == ('u', 'd'):
pull_op.command = 'i' # negate the local delete
purgelocal(local)
elif pair == ('d', 'u'):
local.command = 'i' # negate the remote delete
session.flush()
reverted = True
else: # ('d', 'd')
purgelocal(local)
dependency = extract(pull_op, dependency_conflicts)
if dependency and not reverted:
can_perform = False
order = min(op.order for op in unversioned_ops)
# first move all operations further in order, to make way
# for the new one
for op in unversioned_ops:
op.order = op.order + 1
session.flush()
# then create operation to reflect the reinsertion and
# maintain a correct operation history
session.add(Operation(row_id=pull_op.row_id,
content_type_id=pull_op.content_type_id,
command='i',
order=order))
reversed_dependency = extract(pull_op, reversed_dependency_conflicts)
for local in reversed_dependency:
# reinsert record
local.command = 'i'
local.perform(pull_message, session)
# delete trace of deletion
purgelocal(local)
insert = extract(pull_op, insert_conflicts)
for local in insert:
session.flush()
next_id = max(max_remote(class_, pull_message),
max_local(class_, session)) + 1
update_local_id(local.row_id, next_id, class_, session)
local.row_id = next_id
if can_perform:
pull_op.perform(pull_message, session)
session.flush()
# IV) fourth phase: insert versions from the pull_message
for pull_version in pull_message.versions:
session.add(pull_version)
