def listener(mapper, connection, target, session=None):
if getattr(core.SessionClass.object_session(target),
core.INTERNAL_SESSION_ATTR,
False):
return
if not core.listening:
logger.warning("dbsync is disabled; "
"aborting listener to '{0}' command".format(command))
return
if command == 'u' and not core.SessionClass.object_session(target).\
is_modified(target, include_collections=False):
return
tname = mapper.mapped_table.name
if tname not in core.synched_models.tables:
logging.error("you must track a mapped class to table {0} "\
"to log operations".format(tname))
return
# one version for each operation
version = Version(created=datetime.datetime.now())
pk = getattr(target, mapper.primary_key[0].name)
op = Operation(
row_id=pk,
content_type_id=core.synched_models.tables[tname].id,
command=command)
session.add(version)
session.add(op)
op.version = version
def _add_operation(command: str,
mapper: Mapper,
target: SQLClass,
session: Optional[Session] = None,
force=False) -> Optional[Operation]:
if session is None:
session = core.SessionClass.object_session(target)
if getattr(session, core.INTERNAL_SESSION_ATTR, False):
return None
if not core.listening:
logger.warning("dbsync is disabled; "
"aborting listener to '{0}' command".format(command))
return None
if command == 'u' and not force and not session.\
is_modified(target, include_collections=False):
return None
mt = mapper.mapped_table
if isinstance(mt, Join):
tname = mapper.mapped_table.right.name
else:
tname = mapper.mapped_table.name
if tname not in core.synched_models.tables:
logging.error("you must track a mapped class to table {0} "\
"to log operations".format(tname))
return None
try:
call_before_tracking_fn(session, command, target)
pk = getattr(target, mapper.primary_key[0].name)
op = Operation(
row_id=pk,
version_id=None, # operation not yet versioned
content_type_id=core.synched_models.tables[tname].id,
command=command)
op._target = target
_operations_queue.append(op)
return op
except SkipOperation:
logger.info(f"operation {command} skipped for {target}")
return None
def compressed_operations(operations):
"""
Compresses a set of operations so as to avoid redundant
ones. Returns the compressed set sorted by operation order. This
procedure doesn't perform database operations.
"""
seqs = group_by(lambda op: (op.row_id, op.content_type_id),
sorted(operations, key=attr('order')))
compressed = []
for seq in list(seqs.values()):
if len(seq) == 1:
compressed.append(seq[0])
elif seq[0].command == 'i':
if seq[-1].command == 'd':
pass
else:
compressed.append(seq[0])
elif seq[0].command == 'u':
if seq[-1].command == 'd':
compressed.append(seq[-1])
else:
compressed.append(seq[0])
else: # seq[0].command == 'd':
if seq[-1].command == 'd':
compressed.append(seq[0])
elif seq[-1].command == 'u':
compressed.append(seq[-1])
else: # seq[-1].command == 'i':
op = seq[-1]
compressed.append(
Operation(order=op.order,
content_type_id=op.content_type_id,
row_id=op.row_id,
version_id=op.version_id,
command='u'))
compressed.sort(key=attr('order'))
return compressed
def listener(mapper, connection, target):
if getattr(core.SessionClass.object_session(target),
core.INTERNAL_SESSION_ATTR, False):
return
if not core.listening:
logger.warning(
"dbsync is disabled; "
"aborting listener to '{0}' command".format(command))
return
if command == 'u' and not core.SessionClass.object_session(target).\
is_modified(target, include_collections=False):
return
tname = mapper.mapped_table.name
if tname not in core.synched_models.tables:
logging.error("you must track a mapped class to table {0} "\
"to log operations".format(tname))
return
pk = getattr(target, mapper.primary_key[0].name)
op = Operation(
row_id=pk,
version_id=None, # operation not yet versioned
content_type_id=core.synched_models.tables[tname].id,
command=command)
_operations_queue.append(op)
def listener(mapper, connection, target, session=None) -> None:
logger.info(f"tracking {target}")
if getattr(core.SessionClass.object_session(target),
core.INTERNAL_SESSION_ATTR, False):
logger.debug(f"internal session object not tracked: {target}")
return
if not core.listening:
logger.warning(
"dbsync is disabled; "
"aborting listener to '{0}' command".format(command))
return
if command == 'u' and not core.SessionClass.object_session(target).\
is_modified(target, include_collections=False):
logger.debug(f"updated and not modified -> no tracking: {target}")
return
mt = mapper.mapped_table
if isinstance(mt, Join):
tname = mt.right.name
else:
tname = mt.name
if tname not in core.synched_models.tables:
logging.error("you must track a mapped class to table {0} "\
"to log operations".format(tname))
return
# one version for each operation
# TODO: can be minimized by collecting ops in one flush queue
try:
call_before_tracking_fn(session, command, target)
except SkipOperation:
logger.info(f"skip operation for {target}")
return
# TODO:
# we should try to make only one version per transaction
# so adding a new version should happen in the flush
# is not so easy but we should do that
# perhaps by creating a version for each new session.begin,
# holding that version until the flush/commit
# other idea:
# for the first time an operation is added to this session,
# we create a version object and pin it to the session(session.current_version=version)
# point the operations to this version
# and finally during flush() the operations have only one session
# problem is that the @core.session_committing decorator creates a new
# session for each call to this function => have to dig deeper
target_session = object_session(target)
def clear_on_flush(session, flush_context):
session.__current_version__ = None
version = Version(created=datetime.datetime.now())
if not getattr(target_session, '__current_version__', None):
target_session.__current_version__ = Version(
created=datetime.datetime.now())
session.add(target_session.__current_version__)
event.listen(target_session, "after_flush", clear_on_flush)
version = target_session.__current_version__
logger.info(f"new version: {version.version_id}")
pk = getattr(target, mapper.primary_key[0].name)
op = Operation(row_id=pk,
content_type_id=core.synched_models.tables[tname].id,
command=command)
session.add(version)
call_after_tracking_fn(session, op, target)
session.add(op)
op.version = version
async def handle_push(connection: Connection,
session: sqlalchemy.orm.Session) -> Optional[int]:
msgs_got = 0
version: Optional[Version] = None
async for msg in connection.socket:
msgs_got += 1
msg_json = json.loads(msg)
pushmsg = PushMessage(msg_json)
# print(f"pushmsg: {msg}")
if not pushmsg.operations:
logger.warn("empty operations list in client PushMessage")
for op in pushmsg.operations:
logger.info(f"operation: {op}")
# await connection.socket.send(f"answer is:{msg}")
logger.info(f"message key={pushmsg.key}")
latest_version_id = core.get_latest_version_id(session=session)
logger.info(
f"** version on server:{latest_version_id}, version in pushmsg:{pushmsg.latest_version_id}"
)
if latest_version_id != pushmsg.latest_version_id:
exc = f"version identifier isn't the latest one; " \
f"incoming: {pushmsg.latest_version_id}, on server:{latest_version_id}"
if latest_version_id is None:
logger.warn(exc)
raise PushRejected(exc)
if pushmsg.latest_version_id is None:
logger.warn(exc)
raise PullSuggested(exc)
if pushmsg.latest_version_id < latest_version_id:
logger.warn(exc)
raise PullSuggested(exc)
raise PushRejected(exc)
if not pushmsg.islegit(session):
raise PushRejected("message isn't properly signed")
for listener in before_push:
listener(session, pushmsg)
# I) detect unique constraint conflicts and resolve them if possible
unique_conflicts = find_unique_conflicts(pushmsg, session)
conflicting_objects = set()
for uc in unique_conflicts:
obj = uc['object']
conflicting_objects.add(obj)
for key, value in zip(uc['columns'], uc['new_values']):
setattr(obj, key, value)
for obj in conflicting_objects:
make_transient(obj) # remove from session
for model in set(type(obj) for obj in conflicting_objects):
pk_name = get_pk(model)
pks = [
getattr(obj, pk_name) for obj in conflicting_objects
if type(obj) is model
]
session.query(model).filter(getattr(model, pk_name).in_(pks)). \
delete(synchronize_session=False) # remove from the database
session.add_all(conflicting_objects) # reinsert
session.flush()
# II) perform the operations
operations = [
o for o in pushmsg.operations if o.tracked_model is not None
]
post_operations: List[Tuple[Operation, SQLClass,
Optional[SQLClass]]] = []
try:
op: Operation
for op in operations:
(obj,
old_obj) = await op.perform_async(pushmsg, session,
pushmsg.node_id,
connection.socket)
if obj is not None:
# if the op has been skipped, it wont be appended for post_operation handling
post_operations.append((op, obj, old_obj))
resp = dict(type="info",
op=dict(
row_id=op.row_id,
version=op.version,
command=op.command,
content_type_id=op.content_type_id,
))
call_after_tracking_fn(session, op, obj)
await connection.socket.send(json.dumps(resp))
except OperationError as e:
logger.exception(
"Couldn't perform operation in push from node %s.",
pushmsg.node_id)
raise PushRejected("at least one operation couldn't be performed",
*e.args)
# III) insert a new version
if post_operations: # only if operations have been done -> create the new version
version = Version(created=datetime.datetime.now(),
node_id=pushmsg.node_id)
session.add(version)
# IV) insert the operations, discarding the 'order' column
accomplished_operations = [
op for (op, obj, old_obj) in post_operations
]
for op in sorted(accomplished_operations, key=attr('order')):
new_op = Operation()
for k in [k for k in properties_dict(op) if k != 'order']:
setattr(new_op, k, getattr(op, k))
session.add(new_op)
new_op.version = version
session.flush()
for op, obj, old_obj in post_operations:
op.call_after_operation_fn(session, obj, old_obj)
# from woodmaster.model.sql.model import WoodPile, Measurement
# orphans = session.query(Measurement).filter(Measurement.woodpile_id == None).all()
# print(f"orphans:{orphans}")
for listener in after_push:
listener(session, pushmsg)
# return the new version id back to the client
logger.info(f"version is: {version}")
if version:
await connection.socket.send(
json.dumps(
dict(type="result", new_version_id=version.version_id)))
return {'new_version_id': version.version_id}
else:
await connection.socket.send(
json.dumps(dict(type="result", new_version_id=None)))
logger.info("sent nothing message")
await connection.socket.close()
logger.info("push ready")
def merge(pull_message, session=None):
"""
Merges a message from the server with the local database.
*pull_message* is an instance of dbsync.messages.pull.PullMessage.
"""
if not isinstance(pull_message, PullMessage):
raise TypeError("need an instance of dbsync.messages.pull.PullMessage "
"to perform the local merge operation")
valid_cts = set(ct for ct in core.synched_models.ids)
unversioned_ops = compress(session=session)
pull_ops = filter(
attr('content_type_id').in_(valid_cts), pull_message.operations)
pull_ops = compressed_operations(pull_ops)
# I) first phase: resolve unique constraint conflicts if
# possible. Abort early if a human error is detected
unique_conflicts, unique_errors = find_unique_conflicts(
pull_ops, unversioned_ops, pull_message, session)
if unique_errors:
raise UniqueConstraintError(unique_errors)
conflicting_objects = set()
for uc in unique_conflicts:
obj = uc['object']
conflicting_objects.add(obj)
for key, value in izip(uc['columns'], uc['new_values']):
setattr(obj, key, value)
# Resolve potential cyclical conflicts by deleting and reinserting
for obj in conflicting_objects:
make_transient(obj) # remove from session
for model in set(type(obj) for obj in conflicting_objects):
pk_name = get_pk(model)
pks = [
getattr(obj, pk_name) for obj in conflicting_objects
if type(obj) is model
]
session.query(model).filter(getattr(model, pk_name).in_(pks)).\
delete(synchronize_session=False) # remove from the database
session.add_all(conflicting_objects) # reinsert them
session.flush()
# II) second phase: detect conflicts between pulled operations and
# unversioned ones
direct_conflicts = find_direct_conflicts(pull_ops, unversioned_ops)
# in which the delete operation is registered on the pull message
dependency_conflicts = find_dependency_conflicts(pull_ops, unversioned_ops,
session)
# in which the delete operation was performed locally
reversed_dependency_conflicts = find_reversed_dependency_conflicts(
pull_ops, unversioned_ops, pull_message)
insert_conflicts = find_insert_conflicts(pull_ops, unversioned_ops)
# III) third phase: perform pull operations, when allowed and
# while resolving conflicts
def extract(op, conflicts):
return [local for remote, local in conflicts if remote is op]
def purgelocal(local):
session.delete(local)
exclude = lambda tup: tup[1] is not local
mfilter(exclude, direct_conflicts)
mfilter(exclude, dependency_conflicts)
mfilter(exclude, reversed_dependency_conflicts)
mfilter(exclude, insert_conflicts)
unversioned_ops.remove(local)
for pull_op in pull_ops:
# flag to control whether the remote operation is free of obstacles
can_perform = True
# flag to detect the early exclusion of a remote operation
reverted = False
# the class of the operation
class_ = pull_op.tracked_model
direct = extract(pull_op, direct_conflicts)
if direct:
if pull_op.command == 'd':
can_perform = False
for local in direct:
pair = (pull_op.command, local.command)
if pair == ('u', 'u'):
can_perform = False # favor local changes over remote ones
elif pair == ('u', 'd'):
pull_op.command = 'i' # negate the local delete
purgelocal(local)
elif pair == ('d', 'u'):
local.command = 'i' # negate the remote delete
session.flush()
reverted = True
else: # ('d', 'd')
purgelocal(local)
dependency = extract(pull_op, dependency_conflicts)
if dependency and not reverted:
can_perform = False
order = min(op.order for op in unversioned_ops)
# first move all operations further in order, to make way
# for the new one
for op in unversioned_ops:
op.order = op.order + 1
session.flush()
# then create operation to reflect the reinsertion and
# maintain a correct operation history
session.add(
Operation(row_id=pull_op.row_id,
content_type_id=pull_op.content_type_id,
command='i',
order=order))
reversed_dependency = extract(pull_op, reversed_dependency_conflicts)
for local in reversed_dependency:
# reinsert record
local.command = 'i'
local.perform(pull_message, session)
# delete trace of deletion
purgelocal(local)
insert = extract(pull_op, insert_conflicts)
for local in insert:
session.flush()
next_id = max(max_remote(class_, pull_message),
max_local(class_, session)) + 1
update_local_id(local.row_id, next_id, class_, session)
local.row_id = next_id
if can_perform:
pull_op.perform(pull_message, session)
session.flush()
# IV) fourth phase: insert versions from the pull_message
for pull_version in pull_message.versions:
session.add(pull_version)
def handle_push(data: Dict[str, Any],
session: Optional[Session] = None) -> Dict[str, int]:
"""
Handle the push request and return a dictionary object to be sent
back to the node.
If the push is rejected, this procedure will raise a
dbsync.server.handlers.PushRejected exception.
*data* must be a dictionary-like object, usually the product of
parsing a JSON string.
"""
message: PushMessage
try:
message = PushMessage(data)
except KeyError:
raise PushRejected("request object isn't a valid PushMessage", data)
latest_version_id = core.get_latest_version_id(session=session)
if latest_version_id != message.latest_version_id:
exc = "version identifier isn't the latest one; "\
"given: %s" % message.latest_version_id
if latest_version_id is None:
raise PushRejected(exc)
if message.latest_version_id is None:
raise PullSuggested(exc)
if message.latest_version_id < latest_version_id:
raise PullSuggested(exc)
raise PushRejected(exc)
if not message.operations:
return {}
# raise PushRejected("message doesn't contain operations")
if not message.islegit(session):
raise PushRejected("message isn't properly signed")
for listener in before_push:
listener(session, message)
# I) detect unique constraint conflicts and resolve them if possible
unique_conflicts = find_unique_conflicts(message, session)
conflicting_objects = set()
for uc in unique_conflicts:
obj = uc['object']
conflicting_objects.add(obj)
for key, value in zip(uc['columns'], uc['new_values']):
setattr(obj, key, value)
for obj in conflicting_objects:
make_transient(obj) # remove from session
for model in set(type(obj) for obj in conflicting_objects):
pk_name = get_pk(model)
pks = [
getattr(obj, pk_name) for obj in conflicting_objects
if type(obj) is model
]
session.query(model).filter(getattr(model, pk_name).in_(pks)).\
delete(synchronize_session=False) # remove from the database
session.add_all(conflicting_objects) # reinsert
session.flush()
# II) perform the operations
operations = [o for o in message.operations if o.tracked_model is not None]
try:
for op in operations:
op.perform(message, session, message.node_id)
except OperationError as e:
logger.exception("Couldn't perform operation in push from node %s.",
message.node_id)
raise PushRejected("at least one operation couldn't be performed",
*e.args)
# III) insert a new version
version = Version(created=datetime.datetime.now(), node_id=message.node_id)
session.add(version)
# IV) insert the operations, discarding the 'order' column
for op in sorted(operations, key=attr('order')):
new_op = Operation()
for k in [k for k in properties_dict(op) if k != 'order']:
setattr(new_op, k, getattr(op, k))
session.add(new_op)
new_op.version = version
session.flush()
for listener in after_push:
listener(session, message)
# return the new version id back to the node
return {'new_version_id': version.version_id}
def handle_push(data, session=None):
"""
Handle the push request and return a dictionary object to be sent
back to the node.
If the push is rejected, this procedure will raise a
dbsync.server.handlers.PushRejected exception.
*data* must be a dictionary-like object, usually the product of
parsing a JSON string.
"""
message = None
try:
message = PushMessage(data)
except KeyError:
raise PushRejected("request object isn't a valid PushMessage", data)
latest_version_id = core.get_latest_version_id(session=session)
if latest_version_id != message.latest_version_id:
exc = "version identifier isn't the latest one; "\
"given: %s" % message.latest_version_id
if latest_version_id is None:
raise PushRejected(exc)
if message.latest_version_id is None:
raise PullSuggested(exc)
if message.latest_version_id < latest_version_id:
raise PullSuggested(exc)
raise PushRejected(exc)
if not message.operations:
raise PushRejected("message doesn't contain operations")
if not message.islegit(session):
raise PushRejected("message isn't properly signed")
for listener in before_push:
listener(session, message)
# I) detect unique constraint conflicts and resolve them if possible
unique_conflicts = find_unique_conflicts(message, session)
conflicting_objects = set()
for uc in unique_conflicts:
obj = uc['object']
conflicting_objects.add(obj)
for key, value in izip(uc['columns'], uc['new_values']):
setattr(obj, key, value)
for obj in conflicting_objects:
make_transient(obj) # remove from session
for model in set(type(obj) for obj in conflicting_objects):
pk_name = get_pk(model)
pks = [getattr(obj, pk_name)
for obj in conflicting_objects
if type(obj) is model]
session.query(model).filter(getattr(model, pk_name).in_(pks)).\
delete(synchronize_session=False) # remove from the database
session.add_all(conflicting_objects) # reinsert
session.flush()
# II) perform the operations
operations = filter(lambda o: o.tracked_model is not None, message.operations)
try:
for op in operations:
op.perform(message, session, message.node_id)
except OperationError as e:
logger.exception(u"Couldn't perform operation in push from node %s.",
message.node_id)
raise PushRejected("at least one operation couldn't be performed",
*e.args)
# III) insert a new version
version = Version(created=datetime.datetime.now(), node_id=message.node_id)
session.add(version)
# IV) insert the operations, discarding the 'order' column
for op in sorted(operations, key=attr('order')):
new_op = Operation()
for k in ifilter(lambda k: k != 'order', properties_dict(op)):
setattr(new_op, k, getattr(op, k))
session.add(new_op)
new_op.version = version
session.flush()
for listener in after_push:
listener(session, message)
# return the new version id back to the node
return {'new_version_id': version.version_id}
