def get_related_tables(sa_class):
"""
Returns a list of related SA tables dependent on the given SA
model by foreign key.
"""
mapper = class_mapper(sa_class)
models = synched_models.models.iterkeys()
return [table for table in (class_mapper(model).mapped_table
for model in models)
if mapper.mapped_table in [key.column.table
for key in table.foreign_keys]]
def get_related_tables(sa_class):
"""
Returns a list of related SA tables dependent on the given SA
model by foreign key.
"""
mapper = class_mapper(sa_class)
models = synched_models.models.iterkeys()
return [
table
for table in (class_mapper(model).mapped_table for model in models)
if mapper.mapped_table in
[key.column.table for key in table.foreign_keys]
]
def related_remote_ids(operation, container):
"""
Like *related_local_ids*, but the lookups are performed in
*container*, that's an instance of
*dbsync.messages.base.BaseMessage*.
"""
parent_model = operation.tracked_model
if parent_model is None:
return set()
related_tables = get_related_tables(parent_model)
mapped_fks = ifilter(lambda (m, fks): m is not None and fks,
[(synched_models.tables.get(t.name, null_model).model,
get_fks(t,
class_mapper(parent_model).mapped_table))
for t in related_tables])
return set(
(pk, ct.id)
for pk, ct in \
((getattr(obj, get_pk(obj)), synched_models.models.get(model, None))
for model, fks in mapped_fks
for obj in container.query(model).\
filter(lambda obj: any(getattr(obj, fk) == operation.row_id
for fk in fks)))
if ct is not None)
def related_local_ids(operation, session):
"""
For the given operation, return a set of row id values mapped to
content type ids that correspond to objects that are dependent by
foreign key on the object being operated upon. The lookups are
performed in the local database.
"""
parent_model = operation.tracked_model
if parent_model is None:
return set()
related_tables = get_related_tables(parent_model)
mapped_fks = ifilter(lambda (m, fks): m is not None and fks,
[(synched_models.tables.get(t.name, null_model).model,
get_fks(t,
class_mapper(parent_model).mapped_table))
for t in related_tables])
return set(
(pk, ct.id)
for pk, ct in \
((getattr(obj, get_pk(obj)), synched_models.models.get(model, None))
for model, fks in mapped_fks
for obj in query_model(session, model, only_pk=True).\
filter(or_(*(getattr(model, fk) == operation.row_id
for fk in fks))).all())
if ct is not None)
def related_remote_ids(operation, container):
"""
Like *related_local_ids*, but the lookups are performed in
*container*, that's an instance of
*dbsync.messages.base.BaseMessage*.
"""
parent_model = operation.tracked_model
if parent_model is None:
return set()
related_tables = get_related_tables(parent_model)
mapped_fks = ifilter(
lambda (m, fks): m is not None and fks,
[(synched_models.tables.get(t.name, null_model).model,
get_fks(t, class_mapper(parent_model).mapped_table))
for t in related_tables])
return set(
(pk, ct.id)
for pk, ct in \
((getattr(obj, get_pk(obj)), synched_models.models.get(model, None))
for model, fks in mapped_fks
for obj in container.query(model).\
filter(lambda obj: any(getattr(obj, fk) == operation.row_id
for fk in fks)))
if ct is not None)
def related_local_ids(operation, session):
"""
For the given operation, return a set of row id values mapped to
content type ids that correspond to objects that are dependent by
foreign key on the object being operated upon. The lookups are
performed in the local database.
"""
parent_model = operation.tracked_model
if parent_model is None:
return set()
related_tables = get_related_tables(parent_model)
mapped_fks = ifilter(
lambda (m, fks): m is not None and fks,
[(synched_models.tables.get(t.name, null_model).model,
get_fks(t, class_mapper(parent_model).mapped_table))
for t in related_tables])
return set(
(pk, ct.id)
for pk, ct in \
((getattr(obj, get_pk(obj)), synched_models.models.get(model, None))
for model, fks in mapped_fks
for obj in query_model(session, model, only_pk=True).\
filter(or_(*(getattr(model, fk) == operation.row_id
for fk in fks))).all())
if ct is not None)
def update_local_id(old_id, new_id, model, session):
"""
Updates the tuple matching *old_id* with *new_id*, and updates all
dependent tuples in other tables as well.
"""
# Updating either the tuple or the dependent tuples first would
# cause integrity violations if the transaction is flushed in
# between. The order doesn't matter.
if model is None:
raise ValueError("null model given to update_local_id subtransaction")
# must load fully, don't know yet why
obj = query_model(session, model).\
filter_by(**{get_pk(model): old_id}).first()
setattr(obj, get_pk(model), new_id)
# Then the dependent ones
related_tables = get_related_tables(model)
mapped_fks = ifilter(
lambda (m, fks): m is not None and fks,
[(core.synched_models.tables.get(t.name, core.null_model).model,
get_fks(t,
class_mapper(model).mapped_table)) for t in related_tables])
for model, fks in mapped_fks:
for fk in fks:
for obj in query_model(session, model).filter_by(**{fk: old_id}):
setattr(obj, fk, new_id)
session.flush() # raise integrity errors now
def update_local_id(old_id, new_id, model, session):
"""
Updates the tuple matching *old_id* with *new_id*, and updates all
dependent tuples in other tables as well.
"""
# Updating either the tuple or the dependent tuples first would
# cause integrity violations if the transaction is flushed in
# between. The order doesn't matter.
if model is None:
raise ValueError("null model given to update_local_id subtransaction")
# must load fully, don't know yet why
obj = query_model(session, model).\
filter_by(**{get_pk(model): old_id}).first()
setattr(obj, get_pk(model), new_id)
# Then the dependent ones
related_tables = get_related_tables(model)
mapped_fks = ifilter(
lambda (m, fks): m is not None and fks,
[(core.synched_models.tables.get(t.name, core.null_model).model,
get_fks(t, class_mapper(model).mapped_table))
for t in related_tables])
for model, fks in mapped_fks:
for fk in fks:
for obj in query_model(session, model).filter_by(**{fk: old_id}):
setattr(obj, fk, new_id)
session.flush() # raise integrity errors now
def find_unique_conflicts(push_message, session):
"""
Returns a list of conflicts caused by unique constraints in the
given push message contrasted against the database. Each conflict
is a dictionary with the following fields::
object: the conflicting object in database, bound to the
session
columns: tuple of column names in the unique constraint
new_values: tuple of values that can be used to update the
conflicting object.
"""
conflicts = []
for pk, model in ((op.row_id, op.tracked_model)
for op in push_message.operations if op.command != 'd'):
if model is None: continue
mt = class_mapper(model).mapped_table
if isinstance(mt, Join):
constraints = mt.left.constraints.union(mt.right.constraints)
else:
constraints = mt.constraints
for constraint in [
c for c in constraints if isinstance(c, UniqueConstraint)
]:
unique_columns = tuple(col.name for col in constraint.columns)
remote_obj = push_message.query(model).\
filter(attr('__pk__') == pk).first()
remote_values = tuple(
getattr(remote_obj, col, None) for col in unique_columns)
if all(value is None for value in remote_values): continue
local_obj = query_model(session, model).\
filter_by(**dict(list(zip(unique_columns, remote_values)))).first()
if local_obj is None: continue
local_pk = getattr(local_obj, get_pk(model))
if local_pk == pk: continue
push_obj = push_message.query(model).\
filter(attr('__pk__') == local_pk).first()
if push_obj is None: continue # push will fail
conflicts.append({
'object':
local_obj,
'columns':
unique_columns,
'new_values':
tuple(getattr(push_obj, col) for col in unique_columns)
})
return conflicts
def related_local_ids(operation, session):
"""
For the given operation, return a set of row id values mapped to
content type ids that correspond to objects that are dependent by
foreign key on the object being operated upon. The lookups are
performed in the local database.
"""
parent_model = operation.tracked_model
if parent_model is None:
return set()
related_tables = get_related_tables(parent_model)
mapped_fks = [
m_fks
for m_fks
in [
(
synched_models.tables.get(entity_name(t), null_model).model,
get_fks(t, class_mapper(parent_model).mapped_table)
)
for t
in related_tables
]
if m_fks[0] is not None and m_fks[1]
]
try:
return set(
(pk, ct.id)
for pk, ct
in (
(getattr(obj, get_pk(obj)), synched_models.models.get(model, None))
for model, fks in mapped_fks
for obj in query_model(session, model) \
# removed the pk_only param, because that fails with joins
.filter(
or_(
*(
getattr(model, fk) == operation.row_id
for fk
in fks
)
)
).all()
)
if ct is not None
)
except Exception as ex:
logger.exception(f"collecting conflicts failed: {ex}")
r0 = [query_model(session, model) for model, fks in mapped_fks]
r1 = [query_model(session, model).all() for model, fks in mapped_fks]
raise
def max_local(sa_class, session):
"""
Returns the maximum primary key used for the given table.
"""
engine = session.bind
dialect = engine.name
table_name = class_mapper(sa_class).mapped_table.name
# default, strictly incorrect query
found = session.query(func.max(getattr(sa_class, get_pk(sa_class)))).scalar()
if dialect == 'sqlite':
cursor = engine.execute("SELECT seq FROM sqlite_sequence WHERE name = ?",
table_name)
result = cursor.fetchone()[0]
cursor.close()
return max(result, found)
return found
def max_local(sa_class, session):
"""
Returns the maximum primary key used for the given table.
"""
engine = session.bind
dialect = engine.name
table_name = class_mapper(sa_class).mapped_table.name
# default, strictly incorrect query
found = session.query(func.max(getattr(sa_class, get_pk(sa_class)))).scalar()
if dialect == 'sqlite':
cursor = engine.execute("SELECT seq FROM sqlite_sequence WHERE name = ?",
table_name)
result = cursor.fetchone()[0]
cursor.close()
return max(result, found)
return found
def find_unique_conflicts(push_message, session):
"""
Returns a list of conflicts caused by unique constraints in the
given push message contrasted against the database. Each conflict
is a dictionary with the following fields::
object: the conflicting object in database, bound to the
session
columns: tuple of column names in the unique constraint
new_values: tuple of values that can be used to update the
conflicting object.
"""
conflicts = []
for pk, model in ((op.row_id, op.tracked_model)
for op in push_message.operations
if op.command != 'd'):
if model is None: continue
for constraint in ifilter(lambda c: isinstance(c, UniqueConstraint),
class_mapper(model).mapped_table.constraints):
unique_columns = tuple(col.name for col in constraint.columns)
remote_obj = push_message.query(model).\
filter(attr('__pk__') == pk).first()
remote_values = tuple(getattr(remote_obj, col, None)
for col in unique_columns)
if all(value is None for value in remote_values): continue
local_obj = query_model(session, model).\
filter_by(**dict(izip(unique_columns, remote_values))).first()
if local_obj is None: continue
local_pk = getattr(local_obj, get_pk(model))
if local_pk == pk: continue
push_obj = push_message.query(model).\
filter(attr('__pk__') == local_pk).first()
if push_obj is None: continue # push will fail
conflicts.append(
{'object': local_obj,
'columns': unique_columns,
'new_values': tuple(getattr(push_obj, col)
for col in unique_columns)})
return conflicts
def find_unique_conflicts(pull_ops, unversioned_ops, pull_message, session):
"""
Unique constraints violated in a model. Returns two lists of
dictionaries, the first one with the solvable conflicts, and the
second one with the proper errors. Each conflict is a dictionary
with the following fields::
object: the local conflicting object, bound to the session
columns: tuple of column names in the unique constraint
new_values: tuple of values that can be used to update the
conflicting object
Each error is a dictionary with the following fields::
model: the model (class) of the conflicting object
pk: the value of the primary key of the conflicting object
columns: tuple of column names in the unique constraint
"""
def verify_constraint(model, columns, values):
"""
Checks to see whether some local object exists with
conflicting values.
"""
match = query_model(session, model, only_pk=True).\
options(*(undefer(column) for column in columns)).\
filter_by(**dict((column, value)
for column, value in izip(columns, values))).first()
pk = get_pk(model)
return match, getattr(match, pk, None)
def get_remote_values(model, row_id, columns):
"""
Gets the conflicting values out of the remote object set
(*container*).
"""
obj = pull_message.query(model).filter(
attr('__pk__') == row_id).first()
if obj is not None:
return tuple(getattr(obj, column) for column in columns)
return (None, )
# keyed to content type
unversioned_pks = dict(
(ct_id,
set(op.row_id for op in unversioned_ops if op.content_type_id == ct_id
if op.command != 'd')) for ct_id in set(
operation.content_type_id for operation in unversioned_ops))
# the lists to fill with conflicts and errors
conflicts, errors = [], []
for op in pull_ops:
model = op.tracked_model
for constraint in ifilter(
lambda c: isinstance(c, UniqueConstraint),
class_mapper(model).mapped_table.constraints):
unique_columns = tuple(col.name for col in constraint.columns)
# Unique values on the server, to check conflicts with local database
remote_values = get_remote_values(model, op.row_id, unique_columns)
obj_conflict, pk_conflict = verify_constraint(
model, unique_columns, remote_values)
is_unversioned = pk_conflict in unversioned_pks.get(
op.content_type_id, set())
if all(value is None for value in remote_values):
continue # Null value
if pk_conflict is None: continue # No problem
if pk_conflict == op.row_id:
if op.command == 'i':
# Two nodes created objects with the same unique
# value and same pk
errors.append({
'model': type(obj_conflict),
'pk': pk_conflict,
'columns': unique_columns
})
continue
# if pk_conflict != op.row_id:
remote_obj = pull_message.query(model).\
filter(attr('__pk__') == pk_conflict).first()
if remote_obj is not None and not is_unversioned:
old_values = tuple(
getattr(obj_conflict, column) for column in unique_columns)
# The new unique value of the conflictive object
# in server
new_values = tuple(
getattr(remote_obj, column) for column in unique_columns)
if old_values != new_values:
# Library error
# It's necesary to first update the unique value
session.refresh(obj_conflict,
column_properties(obj_conflict))
conflicts.append({
'object': obj_conflict,
'columns': unique_columns,
'new_values': new_values
})
else:
# The server allows two identical unique values
# This should be impossible
pass
elif remote_obj is not None and is_unversioned:
# Two nodes created objects with the same unique
# values. Human error.
errors.append({
'model': type(obj_conflict),
'pk': pk_conflict,
'columns': unique_columns
})
else:
# The conflicting object hasn't been modified on the
# server, which must mean the local user is attempting
# an update that collides with one from another user.
errors.append({
'model': type(obj_conflict),
'pk': pk_conflict,
'columns': unique_columns
})
return conflicts, errors
def find_unique_conflicts(pull_ops, unversioned_ops, pull_message, session):
"""
Unique constraints violated in a model. Returns two lists of
dictionaries, the first one with the solvable conflicts, and the
second one with the proper errors. Each conflict is a dictionary
with the following fields::
object: the local conflicting object, bound to the session
columns: tuple of column names in the unique constraint
new_values: tuple of values that can be used to update the
conflicting object
Each error is a dictionary with the following fields::
model: the model (class) of the conflicting object
pk: the value of the primary key of the conflicting object
columns: tuple of column names in the unique constraint
"""
def verify_constraint(model, columns, values):
"""
Checks to see whether some local object exists with
conflicting values.
"""
match = query_model(session, model, only_pk=True).\
options(*(undefer(column) for column in columns)).\
filter_by(**dict((column, value)
for column, value in izip(columns, values))).first()
pk = get_pk(model)
return match, getattr(match, pk, None)
def get_remote_values(model, row_id, columns):
"""
Gets the conflicting values out of the remote object set
(*container*).
"""
obj = pull_message.query(model).filter(attr('__pk__') == row_id).first()
if obj is not None:
return tuple(getattr(obj, column) for column in columns)
return (None,)
# keyed to content type
unversioned_pks = dict((ct_id, set(op.row_id for op in unversioned_ops
if op.content_type_id == ct_id
if op.command != 'd'))
for ct_id in set(operation.content_type_id
for operation in unversioned_ops))
# the lists to fill with conflicts and errors
conflicts, errors = [], []
for op in pull_ops:
model = op.tracked_model
for constraint in ifilter(lambda c: isinstance(c, UniqueConstraint),
class_mapper(model).mapped_table.constraints):
unique_columns = tuple(col.name for col in constraint.columns)
# Unique values on the server, to check conflicts with local database
remote_values = get_remote_values(model, op.row_id, unique_columns)
obj_conflict, pk_conflict = verify_constraint(
model, unique_columns, remote_values)
is_unversioned = pk_conflict in unversioned_pks.get(
op.content_type_id, set())
if all(value is None for value in remote_values): continue # Null value
if pk_conflict is None: continue # No problem
if pk_conflict == op.row_id:
if op.command == 'i':
# Two nodes created objects with the same unique
# value and same pk
errors.append(
{'model': type(obj_conflict),
'pk': pk_conflict,
'columns': unique_columns})
continue
# if pk_conflict != op.row_id:
remote_obj = pull_message.query(model).\
filter(attr('__pk__') == pk_conflict).first()
if remote_obj is not None and not is_unversioned:
old_values = tuple(getattr(obj_conflict, column)
for column in unique_columns)
# The new unique value of the conflictive object
# in server
new_values = tuple(getattr(remote_obj, column)
for column in unique_columns)
if old_values != new_values:
# Library error
# It's necesary to first update the unique value
session.refresh(obj_conflict, column_properties(obj_conflict))
conflicts.append(
{'object': obj_conflict,
'columns': unique_columns,
'new_values': new_values})
else:
# The server allows two identical unique values
# This should be impossible
pass
elif remote_obj is not None and is_unversioned:
# Two nodes created objects with the same unique
# values. Human error.
errors.append(
{'model': type(obj_conflict),
'pk': pk_conflict,
'columns': unique_columns})
else:
# The conflicting object hasn't been modified on the
# server, which must mean the local user is attempting
# an update that collides with one from another user.
errors.append(
{'model': type(obj_conflict),
'pk': pk_conflict,
'columns': unique_columns})
return conflicts, errors