def listener(mapper, connection, target, session=None): if getattr(core.SessionClass.object_session(target), core.INTERNAL_SESSION_ATTR, False): return if not core.listening: logger.warning("dbsync is disabled; " "aborting listener to '{0}' command".format(command)) return if command == 'u' and not core.SessionClass.object_session(target).\ is_modified(target, include_collections=False): return tname = mapper.mapped_table.name if tname not in core.synched_models.tables: logging.error("you must track a mapped class to table {0} "\ "to log operations".format(tname)) return # one version for each operation version = Version(created=datetime.datetime.now()) pk = getattr(target, mapper.primary_key[0].name) op = Operation( row_id=pk, content_type_id=core.synched_models.tables[tname].id, command=command) session.add(version) session.add(op) op.version = version
def _add_operation(command: str, mapper: Mapper, target: SQLClass, session: Optional[Session] = None, force=False) -> Optional[Operation]: if session is None: session = core.SessionClass.object_session(target) if getattr(session, core.INTERNAL_SESSION_ATTR, False): return None if not core.listening: logger.warning("dbsync is disabled; " "aborting listener to '{0}' command".format(command)) return None if command == 'u' and not force and not session.\ is_modified(target, include_collections=False): return None mt = mapper.mapped_table if isinstance(mt, Join): tname = mapper.mapped_table.right.name else: tname = mapper.mapped_table.name if tname not in core.synched_models.tables: logging.error("you must track a mapped class to table {0} "\ "to log operations".format(tname)) return None try: call_before_tracking_fn(session, command, target) pk = getattr(target, mapper.primary_key[0].name) op = Operation( row_id=pk, version_id=None, # operation not yet versioned content_type_id=core.synched_models.tables[tname].id, command=command) op._target = target _operations_queue.append(op) return op except SkipOperation: logger.info(f"operation {command} skipped for {target}") return None
def compressed_operations(operations): """ Compresses a set of operations so as to avoid redundant ones. Returns the compressed set sorted by operation order. This procedure doesn't perform database operations. """ seqs = group_by(lambda op: (op.row_id, op.content_type_id), sorted(operations, key=attr('order'))) compressed = [] for seq in list(seqs.values()): if len(seq) == 1: compressed.append(seq[0]) elif seq[0].command == 'i': if seq[-1].command == 'd': pass else: compressed.append(seq[0]) elif seq[0].command == 'u': if seq[-1].command == 'd': compressed.append(seq[-1]) else: compressed.append(seq[0]) else: # seq[0].command == 'd': if seq[-1].command == 'd': compressed.append(seq[0]) elif seq[-1].command == 'u': compressed.append(seq[-1]) else: # seq[-1].command == 'i': op = seq[-1] compressed.append( Operation(order=op.order, content_type_id=op.content_type_id, row_id=op.row_id, version_id=op.version_id, command='u')) compressed.sort(key=attr('order')) return compressed
def listener(mapper, connection, target): if getattr(core.SessionClass.object_session(target), core.INTERNAL_SESSION_ATTR, False): return if not core.listening: logger.warning( "dbsync is disabled; " "aborting listener to '{0}' command".format(command)) return if command == 'u' and not core.SessionClass.object_session(target).\ is_modified(target, include_collections=False): return tname = mapper.mapped_table.name if tname not in core.synched_models.tables: logging.error("you must track a mapped class to table {0} "\ "to log operations".format(tname)) return pk = getattr(target, mapper.primary_key[0].name) op = Operation( row_id=pk, version_id=None, # operation not yet versioned content_type_id=core.synched_models.tables[tname].id, command=command) _operations_queue.append(op)
def listener(mapper, connection, target, session=None) -> None: logger.info(f"tracking {target}") if getattr(core.SessionClass.object_session(target), core.INTERNAL_SESSION_ATTR, False): logger.debug(f"internal session object not tracked: {target}") return if not core.listening: logger.warning( "dbsync is disabled; " "aborting listener to '{0}' command".format(command)) return if command == 'u' and not core.SessionClass.object_session(target).\ is_modified(target, include_collections=False): logger.debug(f"updated and not modified -> no tracking: {target}") return mt = mapper.mapped_table if isinstance(mt, Join): tname = mt.right.name else: tname = mt.name if tname not in core.synched_models.tables: logging.error("you must track a mapped class to table {0} "\ "to log operations".format(tname)) return # one version for each operation # TODO: can be minimized by collecting ops in one flush queue try: call_before_tracking_fn(session, command, target) except SkipOperation: logger.info(f"skip operation for {target}") return # TODO: # we should try to make only one version per transaction # so adding a new version should happen in the flush # is not so easy but we should do that # perhaps by creating a version for each new session.begin, # holding that version until the flush/commit # other idea: # for the first time an operation is added to this session, # we create a version object and pin it to the session(session.current_version=version) # point the operations to this version # and finally during flush() the operations have only one session # problem is that the @core.session_committing decorator creates a new # session for each call to this function => have to dig deeper target_session = object_session(target) def clear_on_flush(session, flush_context): session.__current_version__ = None version = Version(created=datetime.datetime.now()) if not getattr(target_session, '__current_version__', None): target_session.__current_version__ = Version( created=datetime.datetime.now()) session.add(target_session.__current_version__) event.listen(target_session, "after_flush", clear_on_flush) version = target_session.__current_version__ logger.info(f"new version: {version.version_id}") pk = getattr(target, mapper.primary_key[0].name) op = Operation(row_id=pk, content_type_id=core.synched_models.tables[tname].id, command=command) session.add(version) call_after_tracking_fn(session, op, target) session.add(op) op.version = version
async def handle_push(connection: Connection, session: sqlalchemy.orm.Session) -> Optional[int]: msgs_got = 0 version: Optional[Version] = None async for msg in connection.socket: msgs_got += 1 msg_json = json.loads(msg) pushmsg = PushMessage(msg_json) # print(f"pushmsg: {msg}") if not pushmsg.operations: logger.warn("empty operations list in client PushMessage") for op in pushmsg.operations: logger.info(f"operation: {op}") # await connection.socket.send(f"answer is:{msg}") logger.info(f"message key={pushmsg.key}") latest_version_id = core.get_latest_version_id(session=session) logger.info( f"** version on server:{latest_version_id}, version in pushmsg:{pushmsg.latest_version_id}" ) if latest_version_id != pushmsg.latest_version_id: exc = f"version identifier isn't the latest one; " \ f"incoming: {pushmsg.latest_version_id}, on server:{latest_version_id}" if latest_version_id is None: logger.warn(exc) raise PushRejected(exc) if pushmsg.latest_version_id is None: logger.warn(exc) raise PullSuggested(exc) if pushmsg.latest_version_id < latest_version_id: logger.warn(exc) raise PullSuggested(exc) raise PushRejected(exc) if not pushmsg.islegit(session): raise PushRejected("message isn't properly signed") for listener in before_push: listener(session, pushmsg) # I) detect unique constraint conflicts and resolve them if possible unique_conflicts = find_unique_conflicts(pushmsg, session) conflicting_objects = set() for uc in unique_conflicts: obj = uc['object'] conflicting_objects.add(obj) for key, value in zip(uc['columns'], uc['new_values']): setattr(obj, key, value) for obj in conflicting_objects: make_transient(obj) # remove from session for model in set(type(obj) for obj in conflicting_objects): pk_name = get_pk(model) pks = [ getattr(obj, pk_name) for obj in conflicting_objects if type(obj) is model ] session.query(model).filter(getattr(model, pk_name).in_(pks)). \ delete(synchronize_session=False) # remove from the database session.add_all(conflicting_objects) # reinsert session.flush() # II) perform the operations operations = [ o for o in pushmsg.operations if o.tracked_model is not None ] post_operations: List[Tuple[Operation, SQLClass, Optional[SQLClass]]] = [] try: op: Operation for op in operations: (obj, old_obj) = await op.perform_async(pushmsg, session, pushmsg.node_id, connection.socket) if obj is not None: # if the op has been skipped, it wont be appended for post_operation handling post_operations.append((op, obj, old_obj)) resp = dict(type="info", op=dict( row_id=op.row_id, version=op.version, command=op.command, content_type_id=op.content_type_id, )) call_after_tracking_fn(session, op, obj) await connection.socket.send(json.dumps(resp)) except OperationError as e: logger.exception( "Couldn't perform operation in push from node %s.", pushmsg.node_id) raise PushRejected("at least one operation couldn't be performed", *e.args) # III) insert a new version if post_operations: # only if operations have been done -> create the new version version = Version(created=datetime.datetime.now(), node_id=pushmsg.node_id) session.add(version) # IV) insert the operations, discarding the 'order' column accomplished_operations = [ op for (op, obj, old_obj) in post_operations ] for op in sorted(accomplished_operations, key=attr('order')): new_op = Operation() for k in [k for k in properties_dict(op) if k != 'order']: setattr(new_op, k, getattr(op, k)) session.add(new_op) new_op.version = version session.flush() for op, obj, old_obj in post_operations: op.call_after_operation_fn(session, obj, old_obj) # from woodmaster.model.sql.model import WoodPile, Measurement # orphans = session.query(Measurement).filter(Measurement.woodpile_id == None).all() # print(f"orphans:{orphans}") for listener in after_push: listener(session, pushmsg) # return the new version id back to the client logger.info(f"version is: {version}") if version: await connection.socket.send( json.dumps( dict(type="result", new_version_id=version.version_id))) return {'new_version_id': version.version_id} else: await connection.socket.send( json.dumps(dict(type="result", new_version_id=None))) logger.info("sent nothing message") await connection.socket.close() logger.info("push ready")
def merge(pull_message, session=None): """ Merges a message from the server with the local database. *pull_message* is an instance of dbsync.messages.pull.PullMessage. """ if not isinstance(pull_message, PullMessage): raise TypeError("need an instance of dbsync.messages.pull.PullMessage " "to perform the local merge operation") valid_cts = set(ct for ct in core.synched_models.ids) unversioned_ops = compress(session=session) pull_ops = filter( attr('content_type_id').in_(valid_cts), pull_message.operations) pull_ops = compressed_operations(pull_ops) # I) first phase: resolve unique constraint conflicts if # possible. Abort early if a human error is detected unique_conflicts, unique_errors = find_unique_conflicts( pull_ops, unversioned_ops, pull_message, session) if unique_errors: raise UniqueConstraintError(unique_errors) conflicting_objects = set() for uc in unique_conflicts: obj = uc['object'] conflicting_objects.add(obj) for key, value in izip(uc['columns'], uc['new_values']): setattr(obj, key, value) # Resolve potential cyclical conflicts by deleting and reinserting for obj in conflicting_objects: make_transient(obj) # remove from session for model in set(type(obj) for obj in conflicting_objects): pk_name = get_pk(model) pks = [ getattr(obj, pk_name) for obj in conflicting_objects if type(obj) is model ] session.query(model).filter(getattr(model, pk_name).in_(pks)).\ delete(synchronize_session=False) # remove from the database session.add_all(conflicting_objects) # reinsert them session.flush() # II) second phase: detect conflicts between pulled operations and # unversioned ones direct_conflicts = find_direct_conflicts(pull_ops, unversioned_ops) # in which the delete operation is registered on the pull message dependency_conflicts = find_dependency_conflicts(pull_ops, unversioned_ops, session) # in which the delete operation was performed locally reversed_dependency_conflicts = find_reversed_dependency_conflicts( pull_ops, unversioned_ops, pull_message) insert_conflicts = find_insert_conflicts(pull_ops, unversioned_ops) # III) third phase: perform pull operations, when allowed and # while resolving conflicts def extract(op, conflicts): return [local for remote, local in conflicts if remote is op] def purgelocal(local): session.delete(local) exclude = lambda tup: tup[1] is not local mfilter(exclude, direct_conflicts) mfilter(exclude, dependency_conflicts) mfilter(exclude, reversed_dependency_conflicts) mfilter(exclude, insert_conflicts) unversioned_ops.remove(local) for pull_op in pull_ops: # flag to control whether the remote operation is free of obstacles can_perform = True # flag to detect the early exclusion of a remote operation reverted = False # the class of the operation class_ = pull_op.tracked_model direct = extract(pull_op, direct_conflicts) if direct: if pull_op.command == 'd': can_perform = False for local in direct: pair = (pull_op.command, local.command) if pair == ('u', 'u'): can_perform = False # favor local changes over remote ones elif pair == ('u', 'd'): pull_op.command = 'i' # negate the local delete purgelocal(local) elif pair == ('d', 'u'): local.command = 'i' # negate the remote delete session.flush() reverted = True else: # ('d', 'd') purgelocal(local) dependency = extract(pull_op, dependency_conflicts) if dependency and not reverted: can_perform = False order = min(op.order for op in unversioned_ops) # first move all operations further in order, to make way # for the new one for op in unversioned_ops: op.order = op.order + 1 session.flush() # then create operation to reflect the reinsertion and # maintain a correct operation history session.add( Operation(row_id=pull_op.row_id, content_type_id=pull_op.content_type_id, command='i', order=order)) reversed_dependency = extract(pull_op, reversed_dependency_conflicts) for local in reversed_dependency: # reinsert record local.command = 'i' local.perform(pull_message, session) # delete trace of deletion purgelocal(local) insert = extract(pull_op, insert_conflicts) for local in insert: session.flush() next_id = max(max_remote(class_, pull_message), max_local(class_, session)) + 1 update_local_id(local.row_id, next_id, class_, session) local.row_id = next_id if can_perform: pull_op.perform(pull_message, session) session.flush() # IV) fourth phase: insert versions from the pull_message for pull_version in pull_message.versions: session.add(pull_version)
def handle_push(data: Dict[str, Any], session: Optional[Session] = None) -> Dict[str, int]: """ Handle the push request and return a dictionary object to be sent back to the node. If the push is rejected, this procedure will raise a dbsync.server.handlers.PushRejected exception. *data* must be a dictionary-like object, usually the product of parsing a JSON string. """ message: PushMessage try: message = PushMessage(data) except KeyError: raise PushRejected("request object isn't a valid PushMessage", data) latest_version_id = core.get_latest_version_id(session=session) if latest_version_id != message.latest_version_id: exc = "version identifier isn't the latest one; "\ "given: %s" % message.latest_version_id if latest_version_id is None: raise PushRejected(exc) if message.latest_version_id is None: raise PullSuggested(exc) if message.latest_version_id < latest_version_id: raise PullSuggested(exc) raise PushRejected(exc) if not message.operations: return {} # raise PushRejected("message doesn't contain operations") if not message.islegit(session): raise PushRejected("message isn't properly signed") for listener in before_push: listener(session, message) # I) detect unique constraint conflicts and resolve them if possible unique_conflicts = find_unique_conflicts(message, session) conflicting_objects = set() for uc in unique_conflicts: obj = uc['object'] conflicting_objects.add(obj) for key, value in zip(uc['columns'], uc['new_values']): setattr(obj, key, value) for obj in conflicting_objects: make_transient(obj) # remove from session for model in set(type(obj) for obj in conflicting_objects): pk_name = get_pk(model) pks = [ getattr(obj, pk_name) for obj in conflicting_objects if type(obj) is model ] session.query(model).filter(getattr(model, pk_name).in_(pks)).\ delete(synchronize_session=False) # remove from the database session.add_all(conflicting_objects) # reinsert session.flush() # II) perform the operations operations = [o for o in message.operations if o.tracked_model is not None] try: for op in operations: op.perform(message, session, message.node_id) except OperationError as e: logger.exception("Couldn't perform operation in push from node %s.", message.node_id) raise PushRejected("at least one operation couldn't be performed", *e.args) # III) insert a new version version = Version(created=datetime.datetime.now(), node_id=message.node_id) session.add(version) # IV) insert the operations, discarding the 'order' column for op in sorted(operations, key=attr('order')): new_op = Operation() for k in [k for k in properties_dict(op) if k != 'order']: setattr(new_op, k, getattr(op, k)) session.add(new_op) new_op.version = version session.flush() for listener in after_push: listener(session, message) # return the new version id back to the node return {'new_version_id': version.version_id}
def handle_push(data, session=None): """ Handle the push request and return a dictionary object to be sent back to the node. If the push is rejected, this procedure will raise a dbsync.server.handlers.PushRejected exception. *data* must be a dictionary-like object, usually the product of parsing a JSON string. """ message = None try: message = PushMessage(data) except KeyError: raise PushRejected("request object isn't a valid PushMessage", data) latest_version_id = core.get_latest_version_id(session=session) if latest_version_id != message.latest_version_id: exc = "version identifier isn't the latest one; "\ "given: %s" % message.latest_version_id if latest_version_id is None: raise PushRejected(exc) if message.latest_version_id is None: raise PullSuggested(exc) if message.latest_version_id < latest_version_id: raise PullSuggested(exc) raise PushRejected(exc) if not message.operations: raise PushRejected("message doesn't contain operations") if not message.islegit(session): raise PushRejected("message isn't properly signed") for listener in before_push: listener(session, message) # I) detect unique constraint conflicts and resolve them if possible unique_conflicts = find_unique_conflicts(message, session) conflicting_objects = set() for uc in unique_conflicts: obj = uc['object'] conflicting_objects.add(obj) for key, value in izip(uc['columns'], uc['new_values']): setattr(obj, key, value) for obj in conflicting_objects: make_transient(obj) # remove from session for model in set(type(obj) for obj in conflicting_objects): pk_name = get_pk(model) pks = [getattr(obj, pk_name) for obj in conflicting_objects if type(obj) is model] session.query(model).filter(getattr(model, pk_name).in_(pks)).\ delete(synchronize_session=False) # remove from the database session.add_all(conflicting_objects) # reinsert session.flush() # II) perform the operations operations = filter(lambda o: o.tracked_model is not None, message.operations) try: for op in operations: op.perform(message, session, message.node_id) except OperationError as e: logger.exception(u"Couldn't perform operation in push from node %s.", message.node_id) raise PushRejected("at least one operation couldn't be performed", *e.args) # III) insert a new version version = Version(created=datetime.datetime.now(), node_id=message.node_id) session.add(version) # IV) insert the operations, discarding the 'order' column for op in sorted(operations, key=attr('order')): new_op = Operation() for k in ifilter(lambda k: k != 'order', properties_dict(op)): setattr(new_op, k, getattr(op, k)) session.add(new_op) new_op.version = version session.flush() for listener in after_push: listener(session, message) # return the new version id back to the node return {'new_version_id': version.version_id}