Beispiel #1
0
def get_related_tables(sa_class):
    """
    Returns a list of related SA tables dependent on the given SA
    model by foreign key.
    """
    mapper = class_mapper(sa_class)
    models = synched_models.models.iterkeys()
    return [table for table in (class_mapper(model).mapped_table
                                for model in models)
            if mapper.mapped_table in [key.column.table
                                       for key in table.foreign_keys]]
Beispiel #2
0
def get_related_tables(sa_class):
    """
    Returns a list of related SA tables dependent on the given SA
    model by foreign key.
    """
    mapper = class_mapper(sa_class)
    models = synched_models.models.iterkeys()
    return [
        table
        for table in (class_mapper(model).mapped_table for model in models)
        if mapper.mapped_table in
        [key.column.table for key in table.foreign_keys]
    ]
Beispiel #3
0
def related_remote_ids(operation, container):
    """
    Like *related_local_ids*, but the lookups are performed in
    *container*, that's an instance of
    *dbsync.messages.base.BaseMessage*.
    """
    parent_model = operation.tracked_model
    if parent_model is None:
        return set()
    related_tables = get_related_tables(parent_model)

    mapped_fks = ifilter(lambda (m, fks): m is not None and fks,
                         [(synched_models.tables.get(t.name, null_model).model,
                           get_fks(t,
                                   class_mapper(parent_model).mapped_table))
                          for t in related_tables])
    return set(
        (pk, ct.id)
        for pk, ct in \
            ((getattr(obj, get_pk(obj)), synched_models.models.get(model, None))
             for model, fks in mapped_fks
             for obj in container.query(model).\
                 filter(lambda obj: any(getattr(obj, fk) == operation.row_id
                                        for fk in fks)))
        if ct is not None)
Beispiel #4
0
def related_local_ids(operation, session):
    """
    For the given operation, return a set of row id values mapped to
    content type ids that correspond to objects that are dependent by
    foreign key on the object being operated upon. The lookups are
    performed in the local database.
    """
    parent_model = operation.tracked_model
    if parent_model is None:
        return set()
    related_tables = get_related_tables(parent_model)

    mapped_fks = ifilter(lambda (m, fks): m is not None and fks,
                         [(synched_models.tables.get(t.name, null_model).model,
                           get_fks(t,
                                   class_mapper(parent_model).mapped_table))
                          for t in related_tables])
    return set(
        (pk, ct.id)
        for pk, ct in \
            ((getattr(obj, get_pk(obj)), synched_models.models.get(model, None))
             for model, fks in mapped_fks
             for obj in query_model(session, model, only_pk=True).\
                 filter(or_(*(getattr(model, fk) == operation.row_id
                              for fk in fks))).all())
        if ct is not None)
Beispiel #5
0
def related_remote_ids(operation, container):
    """
    Like *related_local_ids*, but the lookups are performed in
    *container*, that's an instance of
    *dbsync.messages.base.BaseMessage*.
    """
    parent_model = operation.tracked_model
    if parent_model is None:
        return set()
    related_tables = get_related_tables(parent_model)

    mapped_fks = ifilter(
        lambda (m, fks): m is not None and fks,
        [(synched_models.tables.get(t.name, null_model).model,
          get_fks(t, class_mapper(parent_model).mapped_table))
         for t in related_tables])
    return set(
        (pk, ct.id)
        for pk, ct in \
            ((getattr(obj, get_pk(obj)), synched_models.models.get(model, None))
             for model, fks in mapped_fks
             for obj in container.query(model).\
                 filter(lambda obj: any(getattr(obj, fk) == operation.row_id
                                        for fk in fks)))
        if ct is not None)
Beispiel #6
0
def related_local_ids(operation, session):
    """
    For the given operation, return a set of row id values mapped to
    content type ids that correspond to objects that are dependent by
    foreign key on the object being operated upon. The lookups are
    performed in the local database.
    """
    parent_model = operation.tracked_model
    if parent_model is None:
        return set()
    related_tables = get_related_tables(parent_model)

    mapped_fks = ifilter(
        lambda (m, fks): m is not None and fks,
        [(synched_models.tables.get(t.name, null_model).model,
          get_fks(t, class_mapper(parent_model).mapped_table))
         for t in related_tables])
    return set(
        (pk, ct.id)
        for pk, ct in \
            ((getattr(obj, get_pk(obj)), synched_models.models.get(model, None))
             for model, fks in mapped_fks
             for obj in query_model(session, model, only_pk=True).\
                 filter(or_(*(getattr(model, fk) == operation.row_id
                              for fk in fks))).all())
        if ct is not None)
Beispiel #7
0
def update_local_id(old_id, new_id, model, session):
    """
    Updates the tuple matching *old_id* with *new_id*, and updates all
    dependent tuples in other tables as well.
    """
    # Updating either the tuple or the dependent tuples first would
    # cause integrity violations if the transaction is flushed in
    # between. The order doesn't matter.
    if model is None:
        raise ValueError("null model given to update_local_id subtransaction")
    # must load fully, don't know yet why
    obj = query_model(session, model).\
        filter_by(**{get_pk(model): old_id}).first()
    setattr(obj, get_pk(model), new_id)

    # Then the dependent ones
    related_tables = get_related_tables(model)
    mapped_fks = ifilter(
        lambda (m, fks): m is not None and fks,
        [(core.synched_models.tables.get(t.name, core.null_model).model,
          get_fks(t,
                  class_mapper(model).mapped_table)) for t in related_tables])
    for model, fks in mapped_fks:
        for fk in fks:
            for obj in query_model(session, model).filter_by(**{fk: old_id}):
                setattr(obj, fk, new_id)
    session.flush()  # raise integrity errors now
Beispiel #8
0
def update_local_id(old_id, new_id, model, session):
    """
    Updates the tuple matching *old_id* with *new_id*, and updates all
    dependent tuples in other tables as well.
    """
    # Updating either the tuple or the dependent tuples first would
    # cause integrity violations if the transaction is flushed in
    # between. The order doesn't matter.
    if model is None:
        raise ValueError("null model given to update_local_id subtransaction")
    # must load fully, don't know yet why
    obj = query_model(session, model).\
        filter_by(**{get_pk(model): old_id}).first()
    setattr(obj, get_pk(model), new_id)

    # Then the dependent ones
    related_tables = get_related_tables(model)
    mapped_fks = ifilter(
        lambda (m, fks): m is not None and fks,
        [(core.synched_models.tables.get(t.name, core.null_model).model,
          get_fks(t, class_mapper(model).mapped_table))
         for t in related_tables])
    for model, fks in mapped_fks:
        for fk in fks:
            for obj in query_model(session, model).filter_by(**{fk: old_id}):
                setattr(obj, fk, new_id)
    session.flush() # raise integrity errors now
Beispiel #9
0
def find_unique_conflicts(push_message, session):
    """
    Returns a list of conflicts caused by unique constraints in the
    given push message contrasted against the database. Each conflict
    is a dictionary with the following fields::

        object: the conflicting object in database, bound to the
                session
        columns: tuple of column names in the unique constraint
        new_values: tuple of values that can be used to update the
                    conflicting object.
    """
    conflicts = []

    for pk, model in ((op.row_id, op.tracked_model)
                      for op in push_message.operations if op.command != 'd'):
        if model is None: continue

        mt = class_mapper(model).mapped_table
        if isinstance(mt, Join):
            constraints = mt.left.constraints.union(mt.right.constraints)
        else:
            constraints = mt.constraints

        for constraint in [
                c for c in constraints if isinstance(c, UniqueConstraint)
        ]:

            unique_columns = tuple(col.name for col in constraint.columns)
            remote_obj = push_message.query(model).\
                filter(attr('__pk__') == pk).first()
            remote_values = tuple(
                getattr(remote_obj, col, None) for col in unique_columns)

            if all(value is None for value in remote_values): continue
            local_obj = query_model(session, model).\
                filter_by(**dict(list(zip(unique_columns, remote_values)))).first()
            if local_obj is None: continue
            local_pk = getattr(local_obj, get_pk(model))
            if local_pk == pk: continue

            push_obj = push_message.query(model).\
                filter(attr('__pk__') == local_pk).first()
            if push_obj is None: continue  # push will fail

            conflicts.append({
                'object':
                local_obj,
                'columns':
                unique_columns,
                'new_values':
                tuple(getattr(push_obj, col) for col in unique_columns)
            })

    return conflicts
Beispiel #10
0
def related_local_ids(operation, session):
    """
    For the given operation, return a set of row id values mapped to
    content type ids that correspond to objects that are dependent by
    foreign key on the object being operated upon. The lookups are
    performed in the local database.
    """
    parent_model = operation.tracked_model
    if parent_model is None:
        return set()
    related_tables = get_related_tables(parent_model)

    mapped_fks = [
        m_fks
        for m_fks
        in [
            (
                synched_models.tables.get(entity_name(t), null_model).model,
                get_fks(t, class_mapper(parent_model).mapped_table)
            )
            for t
            in related_tables
        ]
        if m_fks[0] is not None and m_fks[1]
    ]
    try:
        return set(
            (pk, ct.id)
            for pk, ct
            in (
                (getattr(obj, get_pk(obj)), synched_models.models.get(model, None))
                for model, fks in mapped_fks
                for obj in query_model(session, model) \
                    # removed the pk_only param, because that fails with joins
                    .filter(
                        or_(
                            *(
                                getattr(model, fk) == operation.row_id
                                for fk
                                in fks
                            )
                        )
                    ).all()
            )
            if ct is not None
        )
    except Exception as ex:
        logger.exception(f"collecting conflicts failed: {ex}")
        r0 = [query_model(session, model) for model, fks in mapped_fks]
        r1 = [query_model(session, model).all() for model, fks in mapped_fks]
        raise
Beispiel #11
0
def max_local(sa_class, session):
    """
    Returns the maximum primary key used for the given table.
    """
    engine = session.bind
    dialect = engine.name
    table_name = class_mapper(sa_class).mapped_table.name
    # default, strictly incorrect query
    found = session.query(func.max(getattr(sa_class, get_pk(sa_class)))).scalar()
    if dialect == 'sqlite':
        cursor = engine.execute("SELECT seq FROM sqlite_sequence WHERE name = ?",
                                table_name)
        result = cursor.fetchone()[0]
        cursor.close()
        return max(result, found)
    return found
Beispiel #12
0
def max_local(sa_class, session):
    """
    Returns the maximum primary key used for the given table.
    """
    engine = session.bind
    dialect = engine.name
    table_name = class_mapper(sa_class).mapped_table.name
    # default, strictly incorrect query
    found = session.query(func.max(getattr(sa_class, get_pk(sa_class)))).scalar()
    if dialect == 'sqlite':
        cursor = engine.execute("SELECT seq FROM sqlite_sequence WHERE name = ?",
                                table_name)
        result = cursor.fetchone()[0]
        cursor.close()
        return max(result, found)
    return found
Beispiel #13
0
def find_unique_conflicts(push_message, session):
    """
    Returns a list of conflicts caused by unique constraints in the
    given push message contrasted against the database. Each conflict
    is a dictionary with the following fields::

        object: the conflicting object in database, bound to the
                session
        columns: tuple of column names in the unique constraint
        new_values: tuple of values that can be used to update the
                    conflicting object.
    """
    conflicts = []

    for pk, model in ((op.row_id, op.tracked_model)
                      for op in push_message.operations
                      if op.command != 'd'):
        if model is None: continue

        for constraint in ifilter(lambda c: isinstance(c, UniqueConstraint),
                                  class_mapper(model).mapped_table.constraints):

            unique_columns = tuple(col.name for col in constraint.columns)
            remote_obj = push_message.query(model).\
                filter(attr('__pk__') == pk).first()
            remote_values = tuple(getattr(remote_obj, col, None)
                                  for col in unique_columns)

            if all(value is None for value in remote_values): continue
            local_obj = query_model(session, model).\
                filter_by(**dict(izip(unique_columns, remote_values))).first()
            if local_obj is None: continue
            local_pk = getattr(local_obj, get_pk(model))
            if local_pk == pk: continue

            push_obj = push_message.query(model).\
                filter(attr('__pk__') == local_pk).first()
            if push_obj is None: continue # push will fail

            conflicts.append(
                {'object': local_obj,
                 'columns': unique_columns,
                 'new_values': tuple(getattr(push_obj, col)
                                     for col in unique_columns)})

    return conflicts
Beispiel #14
0
def find_unique_conflicts(pull_ops, unversioned_ops, pull_message, session):
    """
    Unique constraints violated in a model. Returns two lists of
    dictionaries, the first one with the solvable conflicts, and the
    second one with the proper errors. Each conflict is a dictionary
    with the following fields::

        object: the local conflicting object, bound to the session
        columns: tuple of column names in the unique constraint
        new_values: tuple of values that can be used to update the
                    conflicting object

    Each error is a dictionary with the following fields::

        model: the model (class) of the conflicting object
        pk: the value of the primary key of the conflicting object
        columns: tuple of column names in the unique constraint
    """
    def verify_constraint(model, columns, values):
        """
        Checks to see whether some local object exists with
        conflicting values.
        """
        match = query_model(session, model, only_pk=True).\
            options(*(undefer(column) for column in columns)).\
            filter_by(**dict((column, value)
                             for column, value in izip(columns, values))).first()
        pk = get_pk(model)
        return match, getattr(match, pk, None)

    def get_remote_values(model, row_id, columns):
        """
        Gets the conflicting values out of the remote object set
        (*container*).
        """
        obj = pull_message.query(model).filter(
            attr('__pk__') == row_id).first()
        if obj is not None:
            return tuple(getattr(obj, column) for column in columns)
        return (None, )

    # keyed to content type
    unversioned_pks = dict(
        (ct_id,
         set(op.row_id for op in unversioned_ops if op.content_type_id == ct_id
             if op.command != 'd')) for ct_id in set(
                 operation.content_type_id for operation in unversioned_ops))
    # the lists to fill with conflicts and errors
    conflicts, errors = [], []

    for op in pull_ops:
        model = op.tracked_model

        for constraint in ifilter(
                lambda c: isinstance(c, UniqueConstraint),
                class_mapper(model).mapped_table.constraints):

            unique_columns = tuple(col.name for col in constraint.columns)
            # Unique values on the server, to check conflicts with local database
            remote_values = get_remote_values(model, op.row_id, unique_columns)

            obj_conflict, pk_conflict = verify_constraint(
                model, unique_columns, remote_values)

            is_unversioned = pk_conflict in unversioned_pks.get(
                op.content_type_id, set())

            if all(value is None for value in remote_values):
                continue  # Null value
            if pk_conflict is None: continue  # No problem
            if pk_conflict == op.row_id:
                if op.command == 'i':
                    # Two nodes created objects with the same unique
                    # value and same pk
                    errors.append({
                        'model': type(obj_conflict),
                        'pk': pk_conflict,
                        'columns': unique_columns
                    })
                continue

            # if pk_conflict != op.row_id:
            remote_obj = pull_message.query(model).\
                filter(attr('__pk__') == pk_conflict).first()

            if remote_obj is not None and not is_unversioned:
                old_values = tuple(
                    getattr(obj_conflict, column) for column in unique_columns)
                # The new unique value of the conflictive object
                # in server
                new_values = tuple(
                    getattr(remote_obj, column) for column in unique_columns)

                if old_values != new_values:
                    # Library error
                    # It's necesary to first update the unique value
                    session.refresh(obj_conflict,
                                    column_properties(obj_conflict))
                    conflicts.append({
                        'object': obj_conflict,
                        'columns': unique_columns,
                        'new_values': new_values
                    })
                else:
                    # The server allows two identical unique values
                    # This should be impossible
                    pass
            elif remote_obj is not None and is_unversioned:
                # Two nodes created objects with the same unique
                # values. Human error.
                errors.append({
                    'model': type(obj_conflict),
                    'pk': pk_conflict,
                    'columns': unique_columns
                })
            else:
                # The conflicting object hasn't been modified on the
                # server, which must mean the local user is attempting
                # an update that collides with one from another user.
                errors.append({
                    'model': type(obj_conflict),
                    'pk': pk_conflict,
                    'columns': unique_columns
                })
    return conflicts, errors
Beispiel #15
0
def find_unique_conflicts(pull_ops, unversioned_ops, pull_message, session):
    """
    Unique constraints violated in a model. Returns two lists of
    dictionaries, the first one with the solvable conflicts, and the
    second one with the proper errors. Each conflict is a dictionary
    with the following fields::

        object: the local conflicting object, bound to the session
        columns: tuple of column names in the unique constraint
        new_values: tuple of values that can be used to update the
                    conflicting object

    Each error is a dictionary with the following fields::

        model: the model (class) of the conflicting object
        pk: the value of the primary key of the conflicting object
        columns: tuple of column names in the unique constraint
    """

    def verify_constraint(model, columns, values):
        """
        Checks to see whether some local object exists with
        conflicting values.
        """
        match = query_model(session, model, only_pk=True).\
            options(*(undefer(column) for column in columns)).\
            filter_by(**dict((column, value)
                             for column, value in izip(columns, values))).first()
        pk = get_pk(model)
        return match, getattr(match, pk, None)

    def get_remote_values(model, row_id, columns):
        """
        Gets the conflicting values out of the remote object set
        (*container*).
        """
        obj = pull_message.query(model).filter(attr('__pk__') == row_id).first()
        if obj is not None:
            return tuple(getattr(obj, column) for column in columns)
        return (None,)

    # keyed to content type
    unversioned_pks = dict((ct_id, set(op.row_id for op in unversioned_ops
                                       if op.content_type_id == ct_id
                                       if op.command != 'd'))
                           for ct_id in set(operation.content_type_id
                                            for operation in unversioned_ops))
    # the lists to fill with conflicts and errors
    conflicts, errors = [], []

    for op in pull_ops:
        model = op.tracked_model

        for constraint in ifilter(lambda c: isinstance(c, UniqueConstraint),
                                  class_mapper(model).mapped_table.constraints):

            unique_columns = tuple(col.name for col in constraint.columns)
            # Unique values on the server, to check conflicts with local database
            remote_values = get_remote_values(model, op.row_id, unique_columns)

            obj_conflict, pk_conflict = verify_constraint(
                model, unique_columns, remote_values)

            is_unversioned = pk_conflict in unversioned_pks.get(
                op.content_type_id, set())

            if all(value is None for value in remote_values): continue # Null value
            if pk_conflict is None: continue # No problem
            if pk_conflict == op.row_id:
                if op.command == 'i':
                    # Two nodes created objects with the same unique
                    # value and same pk
                    errors.append(
                        {'model': type(obj_conflict),
                         'pk': pk_conflict,
                         'columns': unique_columns})
                continue

            # if pk_conflict != op.row_id:
            remote_obj = pull_message.query(model).\
                filter(attr('__pk__') == pk_conflict).first()

            if remote_obj is not None and not is_unversioned:
                old_values = tuple(getattr(obj_conflict, column)
                                   for column in unique_columns)
                # The new unique value of the conflictive object
                # in server
                new_values = tuple(getattr(remote_obj, column)
                                   for column in unique_columns)

                if old_values != new_values:
                    # Library error
                    # It's necesary to first update the unique value
                    session.refresh(obj_conflict, column_properties(obj_conflict))
                    conflicts.append(
                        {'object': obj_conflict,
                         'columns': unique_columns,
                         'new_values': new_values})
                else:
                    # The server allows two identical unique values
                    # This should be impossible
                    pass
            elif remote_obj is not None and is_unversioned:
                # Two nodes created objects with the same unique
                # values. Human error.
                errors.append(
                    {'model': type(obj_conflict),
                     'pk': pk_conflict,
                     'columns': unique_columns})
            else:
                # The conflicting object hasn't been modified on the
                # server, which must mean the local user is attempting
                # an update that collides with one from another user.
                errors.append(
                    {'model': type(obj_conflict),
                     'pk': pk_conflict,
                     'columns': unique_columns})
    return conflicts, errors