def update_local_id(old_id, new_id, model, session): """ Updates the tuple matching *old_id* with *new_id*, and updates all dependent tuples in other tables as well. """ # Updating either the tuple or the dependent tuples first would # cause integrity violations if the transaction is flushed in # between. The order doesn't matter. if model is None: raise ValueError("null model given to update_local_id subtransaction") # must load fully, don't know yet why obj = query_model(session, model).\ filter_by(**{get_pk(model): old_id}).first() setattr(obj, get_pk(model), new_id) # Then the dependent ones related_tables = get_related_tables(model) mapped_fks = ifilter( lambda (m, fks): m is not None and fks, [(core.synched_models.tables.get(t.name, core.null_model).model, get_fks(t, class_mapper(model).mapped_table)) for t in related_tables]) for model, fks in mapped_fks: for fk in fks: for obj in query_model(session, model).filter_by(**{fk: old_id}): setattr(obj, fk, new_id) session.flush() # raise integrity errors now
def fill_for(self, request, swell=False, include_extensions=True, session=None): """ Fills this pull message (response) with versions, operations and objects, for the given request (PullRequestMessage). The *swell* parameter is deprecated and considered ``True`` regardless of the value given. This means that parent objects will always be added to the message. *include_extensions* dictates whether the pull message will include model extensions or not. """ assert isinstance(request, PullRequestMessage), "invalid request" versions = session.query(Version) if request.latest_version_id is not None: versions = versions.\ filter(Version.version_id > request.latest_version_id) required_objects = {} required_parents = {} for v in versions: self.versions.append(v) for op in v.operations: model = op.tracked_model if model is None: raise ValueError("operation linked to model %s "\ "which isn't being tracked" % model) if model not in pulled_models: continue self.operations.append(op) if op.command != 'd': pks = required_objects.get(model, set()) pks.add(op.row_id) required_objects[model] = pks for model, pks in ((m, batch) for m, pks in required_objects.iteritems() for batch in grouper(pks, MAX_SQL_VARIABLES)): for obj in query_model(session, model).filter( getattr(model, get_pk(model)).in_(list(pks))).all(): self.add_object(obj, include_extensions=include_extensions) # add parent objects to resolve conflicts in merge for pmodel, ppk in parent_references( obj, synched_models.models.keys()): parent_pks = required_parents.get(pmodel, set()) parent_pks.add(ppk) required_parents[pmodel] = parent_pks for pmodel, ppks in ((m, batch) for m, pks in required_parents.iteritems() for batch in grouper(pks, MAX_SQL_VARIABLES)): for parent in query_model(session, pmodel).filter( getattr(pmodel, get_pk(pmodel)).in_(list(ppks))).all(): self.add_object(parent, include_extensions=include_extensions) return self
def related_local_ids(operation, session): """ For the given operation, return a set of row id values mapped to content type ids that correspond to objects that are dependent by foreign key on the object being operated upon. The lookups are performed in the local database. """ parent_model = operation.tracked_model if parent_model is None: return set() related_tables = get_related_tables(parent_model) mapped_fks = [ m_fks for m_fks in [ ( synched_models.tables.get(entity_name(t), null_model).model, get_fks(t, class_mapper(parent_model).mapped_table) ) for t in related_tables ] if m_fks[0] is not None and m_fks[1] ] try: return set( (pk, ct.id) for pk, ct in ( (getattr(obj, get_pk(obj)), synched_models.models.get(model, None)) for model, fks in mapped_fks for obj in query_model(session, model) \ # removed the pk_only param, because that fails with joins .filter( or_( *( getattr(model, fk) == operation.row_id for fk in fks ) ) ).all() ) if ct is not None ) except Exception as ex: logger.exception(f"collecting conflicts failed: {ex}") r0 = [query_model(session, model) for model, fks in mapped_fks] r1 = [query_model(session, model).all() for model, fks in mapped_fks] raise
def fill_for(self, request, swell=False, include_extensions=True, session=None): """ Fills this pull message (response) with versions, operations and objects, for the given request (PullRequestMessage). The *swell* parameter is deprecated and considered ``True`` regardless of the value given. This means that parent objects will always be added to the message. *include_extensions* dictates whether the pull message will include model extensions or not. """ assert isinstance(request, PullRequestMessage), "invalid request" versions = session.query(Version) if request.latest_version_id is not None: versions = versions.filter(Version.version_id > request.latest_version_id) required_objects = {} required_parents = {} for v in versions: self.versions.append(v) for op in v.operations: model = op.tracked_model if model is None: raise ValueError("operation linked to model %s " "which isn't being tracked" % model) if model not in pulled_models: continue self.operations.append(op) if op.command != "d": pks = required_objects.get(model, set()) pks.add(op.row_id) required_objects[model] = pks for model, pks in ( (m, batch) for m, pks in required_objects.iteritems() for batch in grouper(pks, MAX_SQL_VARIABLES) ): for obj in query_model(session, model).filter(getattr(model, get_pk(model)).in_(list(pks))).all(): self.add_object(obj, include_extensions=include_extensions) # add parent objects to resolve conflicts in merge for pmodel, ppk in parent_references(obj, synched_models.models.keys()): parent_pks = required_parents.get(pmodel, set()) parent_pks.add(ppk) required_parents[pmodel] = parent_pks for pmodel, ppks in ( (m, batch) for m, pks in required_parents.iteritems() for batch in grouper(pks, MAX_SQL_VARIABLES) ): for parent in query_model(session, pmodel).filter(getattr(pmodel, get_pk(pmodel)).in_(list(ppks))).all(): self.add_object(parent, include_extensions=include_extensions) return self
def related_local_ids(operation, session): """ For the given operation, return a set of row id values mapped to content type ids that correspond to objects that are dependent by foreign key on the object being operated upon. The lookups are performed in the local database. """ parent_model = operation.tracked_model if parent_model is None: return set() related_tables = get_related_tables(parent_model) mapped_fks = ifilter( lambda (m, fks): m is not None and fks, [(synched_models.tables.get(t.name, null_model).model, get_fks(t, class_mapper(parent_model).mapped_table)) for t in related_tables]) return set( (pk, ct.id) for pk, ct in \ ((getattr(obj, get_pk(obj)), synched_models.models.get(model, None)) for model, fks in mapped_fks for obj in query_model(session, model, only_pk=True).\ filter(or_(*(getattr(model, fk) == operation.row_id for fk in fks))).all()) if ct is not None)
def add_unversioned_operations(self, session=None, include_extensions=True): """ Adds all unversioned operations to this message, including the required objects for them to be performed. """ operations = session.query(Operation).\ filter(Operation.version_id == None).all() if any(op.content_type_id not in synched_models.ids for op in operations): raise ValueError("version includes operation linked "\ "to model not currently being tracked") required_objects = {} for op in operations: model = op.tracked_model if model not in pushed_models: continue self.operations.append(op) if op.command != 'd': pks = required_objects.get(model, set()) pks.add(op.row_id) required_objects[model] = pks for model, pks in ((m, batch) for m, pks in required_objects.iteritems() for batch in grouper(pks, MAX_SQL_VARIABLES)): for obj in query_model(session, model).filter( getattr(model, get_pk(model)).in_(list(pks))).all(): self.add_object(obj, include_extensions=include_extensions) if self.key is not None: # overwrite since it's probably an incorrect key self._sign() return self
def related_local_ids(operation, session): """ For the given operation, return a set of row id values mapped to content type ids that correspond to objects that are dependent by foreign key on the object being operated upon. The lookups are performed in the local database. """ parent_model = operation.tracked_model if parent_model is None: return set() related_tables = get_related_tables(parent_model) mapped_fks = ifilter(lambda (m, fks): m is not None and fks, [(synched_models.tables.get(t.name, null_model).model, get_fks(t, class_mapper(parent_model).mapped_table)) for t in related_tables]) return set( (pk, ct.id) for pk, ct in \ ((getattr(obj, get_pk(obj)), synched_models.models.get(model, None)) for model, fks in mapped_fks for obj in query_model(session, model, only_pk=True).\ filter(or_(*(getattr(model, fk) == operation.row_id for fk in fks))).all()) if ct is not None)
def tracked(o, **kws): if _has_delete_functions(o): if always: deleted.append((copy(o), None)) else: prev = query_model(session, type(o)).filter_by( **{get_pk(o): getattr(o, get_pk(o), None)}).\ first() if prev is not None: deleted.append((copy(prev), o)) return fn(o, **kws)
def find_unique_conflicts(push_message, session): """ Returns a list of conflicts caused by unique constraints in the given push message contrasted against the database. Each conflict is a dictionary with the following fields:: object: the conflicting object in database, bound to the session columns: tuple of column names in the unique constraint new_values: tuple of values that can be used to update the conflicting object. """ conflicts = [] for pk, model in ((op.row_id, op.tracked_model) for op in push_message.operations if op.command != 'd'): if model is None: continue mt = class_mapper(model).mapped_table if isinstance(mt, Join): constraints = mt.left.constraints.union(mt.right.constraints) else: constraints = mt.constraints for constraint in [ c for c in constraints if isinstance(c, UniqueConstraint) ]: unique_columns = tuple(col.name for col in constraint.columns) remote_obj = push_message.query(model).\ filter(attr('__pk__') == pk).first() remote_values = tuple( getattr(remote_obj, col, None) for col in unique_columns) if all(value is None for value in remote_values): continue local_obj = query_model(session, model).\ filter_by(**dict(list(zip(unique_columns, remote_values)))).first() if local_obj is None: continue local_pk = getattr(local_obj, get_pk(model)) if local_pk == pk: continue push_obj = push_message.query(model).\ filter(attr('__pk__') == local_pk).first() if push_obj is None: continue # push will fail conflicts.append({ 'object': local_obj, 'columns': unique_columns, 'new_values': tuple(getattr(push_obj, col) for col in unique_columns) }) return conflicts
def verify_constraint(model, columns, values): """ Checks to see whether some local object exists with conflicting values. """ match = query_model(session, model, only_pk=True).\ options(*(undefer(column) for column in columns)).\ filter_by(**dict((column, value) for column, value in izip(columns, values))).first() pk = get_pk(model) return match, getattr(match, pk, None)
def handle_repair(data=None, session=None): "Handle repair request. Return whole server database." include_extensions = 'exclude_extensions' not in (data or {}) latest_version_id = core.get_latest_version_id(session=session) message = BaseMessage() for model in list(core.synched_models.models.keys()): for obj in query_model(session, model): message.add_object(obj, include_extensions=include_extensions) response = message.to_json() response['latest_version_id'] = latest_version_id return response
def handle_repair(data=None, session=None): "Handle repair request. Return whole server database." include_extensions = 'exclude_extensions' not in (data or {}) latest_version_id = core.get_latest_version_id(session=session) message = BaseMessage() for model in core.synched_models.models.iterkeys(): for obj in query_model(session, model): message.add_object(obj, include_extensions=include_extensions) response = message.to_json() response['latest_version_id'] = latest_version_id return response
def handle_query(data, session=None): "Responds to a query request." model = core.synched_models.model_names.\ get(data.get('model', None), core.null_model).model if model is None: return None mname = model.__name__ filters = dict((k, v) for k, v in ((k[len(mname) + 1:], v) for k, v in data.iteritems() if k.startswith(mname + '_')) if k and k in column_properties(model)) message = BaseMessage() q = query_model(session, model) if filters: q = q.filter_by(**filters) for obj in q: message.add_object(obj) return message.to_json()
def find_unique_conflicts(push_message, session): """ Returns a list of conflicts caused by unique constraints in the given push message contrasted against the database. Each conflict is a dictionary with the following fields:: object: the conflicting object in database, bound to the session columns: tuple of column names in the unique constraint new_values: tuple of values that can be used to update the conflicting object. """ conflicts = [] for pk, model in ((op.row_id, op.tracked_model) for op in push_message.operations if op.command != 'd'): if model is None: continue for constraint in ifilter(lambda c: isinstance(c, UniqueConstraint), class_mapper(model).mapped_table.constraints): unique_columns = tuple(col.name for col in constraint.columns) remote_obj = push_message.query(model).\ filter(attr('__pk__') == pk).first() remote_values = tuple(getattr(remote_obj, col, None) for col in unique_columns) if all(value is None for value in remote_values): continue local_obj = query_model(session, model).\ filter_by(**dict(izip(unique_columns, remote_values))).first() if local_obj is None: continue local_pk = getattr(local_obj, get_pk(model)) if local_pk == pk: continue push_obj = push_message.query(model).\ filter(attr('__pk__') == local_pk).first() if push_obj is None: continue # push will fail conflicts.append( {'object': local_obj, 'columns': unique_columns, 'new_values': tuple(getattr(push_obj, col) for col in unique_columns)}) return conflicts
def handle_query(data, session=None): "Responds to a query request." model = core.synched_models.model_names.\ get(data.get('model', None), core.null_model).model if model is None: return None mname = model.__name__ filters = dict((k, v) for k, v in ((k[len(mname) + 1:], v) for k, v in list(data.items()) if k.startswith(mname + '_')) if k and k in column_properties(model)) message = BaseMessage() q = query_model(session, model) if filters: q = q.filter_by(**filters) for obj in q: message.add_object(obj) return message.to_json()
def add_operation(self, op, swell=True, session=None): """ Adds an operation to the message, including the required object if it's possible to include it. If *swell* is given and set to ``False``, the operation and object will be added bare, without parent objects. Otherwise, the parent objects will be added to aid in conflict resolution. A delete operation doesn't include the associated object. If *session* is given, the procedure won't instantiate a new session. This operation might fail, (due to database inconsitency) in which case the internal state of the message won't be affected (i.e. it won't end in an inconsistent state). DEPRECATED in favor of `fill_for` """ model = op.tracked_model if model is None: raise ValueError("operation linked to model %s "\ "which isn't being tracked" % model) if model not in pulled_models: return self obj = query_model(session, model).\ filter_by(**{get_pk(model): op.row_id}).first() \ if op.command != 'd' else None self.operations.append(op) # if the object isn't there it's because the operation is old, # and should be able to be compressed out when performing the # conflict resolution phase if obj is not None: self.add_object(obj) if swell: # add parent objects to resolve possible conflicts in merge for parent in parent_objects(obj, synched_models.models.keys(), session): self.add_object(parent) return self
def add_operation(self, op, swell=True, session=None): """ Adds an operation to the message, including the required object if it's possible to include it. If *swell* is given and set to ``False``, the operation and object will be added bare, without parent objects. Otherwise, the parent objects will be added to aid in conflict resolution. A delete operation doesn't include the associated object. If *session* is given, the procedure won't instantiate a new session. This operation might fail, (due to database inconsitency) in which case the internal state of the message won't be affected (i.e. it won't end in an inconsistent state). DEPRECATED in favor of `fill_for` """ model = op.tracked_model if model is None: raise ValueError("operation linked to model %s " "which isn't being tracked" % model) if model not in pulled_models: return self obj = query_model(session, model).filter_by(**{get_pk(model): op.row_id}).first() if op.command != "d" else None self.operations.append(op) # if the object isn't there it's because the operation is old, # and should be able to be compressed out when performing the # conflict resolution phase if obj is not None: self.add_object(obj) if swell: # add parent objects to resolve possible conflicts in merge for parent in parent_objects(obj, synched_models.models.keys(), session): self.add_object(parent) return self
def fill_for(self, request, swell=False, include_extensions=True, session=None, connection=None, **kw): """ Fills this pull message (response) with versions, operations and objects, for the given request (PullRequestMessage). The *swell* parameter is deprecated and considered ``True`` regardless of the value given. This means that parent objects will always be added to the message. *include_extensions* dictates whether the pull message will include model extensions or not. """ assert isinstance(request, PullRequestMessage), "invalid request" versions = session.query(Version) if request.latest_version_id is not None: versions = versions. \ filter(Version.version_id > request.latest_version_id) required_objects = {} required_parents = {} # TODO: since there can be really many versions here # we should rebuild this part so that we make an aggregate query # gettting all operations ordered by their version and by their permissions # something like # select * from # operation, version # where # operation.version_id=version.id # and # version.version_id > request.latest_version_id # and # " one of the user's roles is in operation.allowed_users_and_roles " # order by # operation.order # # the basic query can be done here, # per dep injection we must add the query for allowed_users self.versions = versions.all() ops: Query = session.query(Operation) if request.latest_version_id is not None: ops = ops.filter(Operation.version_id > request.latest_version_id) ops = call_filter_operations(connection, session, ops) ops = ops.order_by(Operation.order) self.operations = [] logger.info(f"request.latest_version_id = {request.latest_version_id}") logger.info(f"querying for {ops}") # logger.info(f"query result: {ops.all()}") logger.info(f"query result #ops: {len(ops.all())}") for op in ops: model = op.tracked_model if model is None: logger.warn( f"op {op} has no model (perhaps removed from tracking)") # raise ValueError("operation linked to model %s " \ # "which isn't being tracked" % model) if model not in pulled_models: continue obj = query_model(session, model).get(op.row_id) if obj is None: if op.command != 'd': logger.error( f"this should not happen, obj is None for op: {op} - ignoring" ) continue try: call_before_server_add_operation_fn(connection, session, op, obj) self.operations.append(op) except SkipOperation: continue if op.command != 'd': pks = required_objects.get(model, set()) pks.add(op.row_id) required_objects[model] = pks for model, pks in ((m, batch) for m, pks in list(required_objects.items()) for batch in grouper(pks, MAX_SQL_VARIABLES)): for obj in query_model(session, model).filter( getattr(model, get_pk(model)).in_(list(pks))).all(): self.add_object(obj, include_extensions=include_extensions) # add parent objects to resolve conflicts in merge for pmodel, ppk in parent_references( obj, list(synched_models.models.keys())): parent_pks = required_parents.get(pmodel, set()) parent_pks.add(ppk) required_parents[pmodel] = parent_pks for pmodel, ppks in ((m, batch) for m, pks in list(required_parents.items()) for batch in grouper(pks, MAX_SQL_VARIABLES)): for parent in query_model(session, pmodel).filter( getattr(pmodel, get_pk(pmodel)).in_(list(ppks))).all(): self.add_object(parent, include_extensions=include_extensions) logger.info(f"operations result: {self.operations}") return self
async def perform_async( operation: "Operation", container: "BaseMessage", session: Session, node_id=None, websocket: Optional[WebSocketCommonProtocol] = None ) -> (Optional[SQLClass], Optional[SQLClass]): """ Performs *operation*, looking for required data in *container*, and using *session* to perform it. *container* is an instance of dbsync.messages.base.BaseMessage. *node_id* is the node responsible for the operation, if known (else ``None``). If at any moment this operation fails for predictable causes, it will raise an *OperationError*. """ from dbsync.core import mode model: DeclarativeMeta = operation.tracked_model res: Tuple[Optional[SQLClass], Optional[SQLClass]] = (None, None) if model is None: raise OperationError("no content type for this operation", operation) if operation.command == 'i': # check if the given object is already in the database obj = query_model(session, model). \ filter(getattr(model, get_pk(model)) == operation.row_id).first() # retrieve the object from the PullMessage qu = container.query(model). \ filter(attr('__pk__') == operation.row_id) # breakpoint() pull_obj = qu.first() # pull_obj._session = session if pull_obj is None: raise OperationError( f"no object backing the operation in container on {mode}", operation) if obj is None: logger.info( f"insert: calling request_payloads_for_extension for: {pull_obj.id}" ) try: operation.call_before_operation_fn(session, pull_obj) await request_payloads_for_extension( operation, pull_obj, websocket, session) session.add(pull_obj) res = pull_obj, None except SkipOperation as e: logger.info(f"operation {operation} skipped") # operation.call_after_operation_fn(pull_obj, session) else: # Don't raise an exception if the incoming object is # exactly the same as the local one. if properties_dict(obj) == properties_dict(pull_obj): logger.warning("insert attempted when an identical object " "already existed in local database: " "model {0} pk {1}".format( model.__name__, operation.row_id)) else: raise OperationError( "insert attempted when the object already existed: " "model {0} pk {1}".format(model.__name__, operation.row_id)) elif operation.command == 'u': obj = query_model(session, model). \ filter(getattr(model, get_pk(model)) == operation.row_id).one_or_none() if obj is not None: logger.info( f"update: calling request_payloads_for_extension for: {obj.id}" ) # breakpoint() else: # For now, the record will be created again, but is an # error because nothing should be deleted without # using dbsync # raise OperationError( # "the referenced object doesn't exist in database", operation) # addendum: # this can happen when tracking of an object has been suppressed and # later been activated during a 'u' operation, # so we keep this logic logger.warning( "The referenced object doesn't exist in database. " "Node %s. Operation %s", node_id, operation) # get new object from the PushMessage pull_obj = container.query(model). \ filter(attr('__pk__') == operation.row_id).first() if pull_obj is None: raise OperationError( "no object backing the operation in container", operation) try: operation.call_before_operation_fn(session, pull_obj, obj) await request_payloads_for_extension(operation, pull_obj, websocket, session) if obj is None: logger.warn(f"obj is None") old_obj = copy(obj) if obj is not None else None session.merge(pull_obj) res = pull_obj, old_obj except SkipOperation as e: logger.info(f"operation {operation} skipped") # operation.call_after_operation_fn(pull_obj, session) elif operation.command == 'd': try: obj = query_model(session, model, only_pk=True). \ filter(getattr(model, get_pk(model)) == operation.row_id).first() except NoSuchColumnError as ex: # for joins only_pk doesnt seem to work obj = query_model(session, model, only_pk=False). \ filter(getattr(model, get_pk(model)) == operation.row_id).first() if obj is None: # The object is already deleted in the server # The final state in node and server are the same. But # it's an error because nothing should be deleted # without using dbsync logger.warning( "The referenced object doesn't exist in database. " "Node %s. Operation %s", node_id, operation) else: try: # breakpoint() operation.call_before_operation_fn(session, obj) session.delete(obj) res = obj, None except SkipOperation as e: logger.info(f"operation {operation} skipped") else: raise OperationError( "the operation doesn't specify a valid command ('i', 'u', 'd')", operation) return res
def compress(session=None) -> List[Operation]: """ Compresses unversioned operations in the database. For each row in the operations table, this deletes unnecesary operations that would otherwise bloat the message. This procedure is called internally before the 'push' request happens, and before the local 'merge' happens. """ unversioned: Query = session.query(Operation).\ filter(Operation.version_id == None).order_by(Operation.order.desc()) seqs = group_by(lambda op: (op.row_id, op.content_type_id), unversioned) # Check errors on sequences for seq in list(seqs.values()): _assert_operation_sequence(seq, session) for seq in [seq for seq in iter(list(seqs.values())) if len(seq) > 1]: if seq[-1].command == 'i': if all(op.command == 'u' for op in seq[:-1]): # updates are superfluous list(map(session.delete, seq[:-1])) elif seq[0].command == 'd': # it's as if the object never existed list(map(session.delete, seq)) elif seq[-1].command == 'u': if all(op.command == 'u' for op in seq[:-1]): # leave a single update list(map(session.delete, seq[1:])) elif seq[0].command == 'd': # leave the delete statement list(map(session.delete, seq[1:])) session.flush() # repair inconsistencies for operation in session.query(Operation).\ filter(Operation.version_id == None).\ order_by(Operation.order.desc()).all(): session.flush() model = operation.tracked_model if not model: logger.error("operation linked to content type " "not tracked: %s" % operation.content_type_id) continue if operation.command in ('i', 'u'): if query_model(session, model, only_pk=True).\ filter_by(**{get_pk(model): operation.row_id}).count() == 0: logger.warning("deleting operation %s for model %s " "for absence of backing object" % (operation, model.__name__)) session.delete(operation) continue if operation.command == 'u': subsequent = session.query(Operation).\ filter(Operation.content_type_id == operation.content_type_id, Operation.version_id == None, Operation.row_id == operation.row_id, Operation.order > operation.order).all() if any(op.command == 'i' for op in subsequent) and \ all(op.command != 'd' for op in subsequent): logger.warning( "deleting update operation %s for model %s " "for preceding an insert operation" %\ (operation, model.__name__)) session.delete(operation) continue if session.query(Operation).\ filter(Operation.content_type_id == operation.content_type_id, Operation.command == operation.command, Operation.version_id == None, Operation.row_id == operation.row_id, Operation.order != operation.order).count() > 0: logger.warning( "deleting operation %s for model %s " "for being redundant after compression" %\ (operation, model.__name__)) session.delete(operation) continue session.commit() return session.query(Operation).\ filter(Operation.version_id == None).\ order_by(Operation.order.asc()).all()
def compress(session=None): """ Compresses unversioned operations in the database. For each row in the operations table, this deletes unnecesary operations that would otherwise bloat the message. This procedure is called internally before the 'push' request happens, and before the local 'merge' happens. """ unversioned = session.query(Operation).\ filter(Operation.version_id == None).order_by(Operation.order.desc()) seqs = group_by(lambda op: (op.row_id, op.content_type_id), unversioned) # Check errors on sequences for seq in seqs.itervalues(): _assert_operation_sequence(seq, session) for seq in ifilter(lambda seq: len(seq) > 1, seqs.itervalues()): if seq[-1].command == 'i': if all(op.command == 'u' for op in seq[:-1]): # updates are superfluous map(session.delete, seq[:-1]) elif seq[0].command == 'd': # it's as if the object never existed map(session.delete, seq) elif seq[-1].command == 'u': if all(op.command == 'u' for op in seq[:-1]): # leave a single update map(session.delete, seq[1:]) elif seq[0].command == 'd': # leave the delete statement map(session.delete, seq[1:]) session.flush() # repair inconsistencies for operation in session.query(Operation).\ filter(Operation.version_id == None).\ order_by(Operation.order.desc()).all(): session.flush() model = operation.tracked_model if not model: logger.error( "operation linked to content type " "not tracked: %s" % operation.content_type_id) continue if operation.command in ('i', 'u'): if query_model(session, model, only_pk=True).\ filter_by(**{get_pk(model): operation.row_id}).count() == 0: logger.warning( "deleting operation %s for model %s " "for absence of backing object" % (operation, model.__name__)) session.delete(operation) continue if operation.command == 'u': subsequent = session.query(Operation).\ filter(Operation.content_type_id == operation.content_type_id, Operation.version_id == None, Operation.row_id == operation.row_id, Operation.order > operation.order).all() if any(op.command == 'i' for op in subsequent) and \ all(op.command != 'd' for op in subsequent): logger.warning( "deleting update operation %s for model %s " "for preceding an insert operation" %\ (operation, model.__name__)) session.delete(operation) continue if session.query(Operation).\ filter(Operation.content_type_id == operation.content_type_id, Operation.command == operation.command, Operation.version_id == None, Operation.row_id == operation.row_id, Operation.order != operation.order).count() > 0: logger.warning( "deleting operation %s for model %s " "for being redundant after compression" %\ (operation, model.__name__)) session.delete(operation) continue return session.query(Operation).\ filter(Operation.version_id == None).\ order_by(Operation.order.asc()).all()
def perform(operation, container, session, node_id=None): """ Performs *operation*, looking for required data in *container*, and using *session* to perform it. *container* is an instance of dbsync.messages.base.BaseMessage. *node_id* is the node responsible for the operation, if known (else ``None``). If at any moment this operation fails for predictable causes, it will raise an *OperationError*. """ model = operation.tracked_model if model is None: raise OperationError("no content type for this operation", operation) if operation.command == 'i': obj = query_model(session, model).\ filter(getattr(model, get_pk(model)) == operation.row_id).first() pull_obj = container.query(model).\ filter(attr('__pk__') == operation.row_id).first() if pull_obj is None: raise OperationError( "no object backing the operation in container", operation) if obj is None: session.add(pull_obj) else: # Don't raise an exception if the incoming object is # exactly the same as the local one. if properties_dict(obj) == properties_dict(pull_obj): logger.warning(u"insert attempted when an identical object " u"already existed in local database: " u"model {0} pk {1}".format(model.__name__, operation.row_id)) else: raise OperationError( u"insert attempted when the object already existed: " u"model {0} pk {1}".format(model.__name__, operation.row_id)) elif operation.command == 'u': obj = query_model(session, model).\ filter(getattr(model, get_pk(model)) == operation.row_id).first() if obj is None: # For now, the record will be created again, but is an # error because nothing should be deleted without # using dbsync # raise OperationError( # "the referenced object doesn't exist in database", operation) logger.warning( u"The referenced object doesn't exist in database. " u"Node %s. Operation %s", node_id, operation) pull_obj = container.query(model).\ filter(attr('__pk__') == operation.row_id).first() if pull_obj is None: raise OperationError( "no object backing the operation in container", operation) session.merge(pull_obj) elif operation.command == 'd': obj = query_model(session, model, only_pk=True).\ filter(getattr(model, get_pk(model)) == operation.row_id).first() if obj is None: # The object is already deleted in the server # The final state in node and server are the same. But # it's an error because nothing should be deleted # without using dbsync logger.warning( "The referenced object doesn't exist in database. " u"Node %s. Operation %s", node_id, operation) else: session.delete(obj) else: raise OperationError( "the operation doesn't specify a valid command ('i', 'u', 'd')", operation)