def update_local_id(old_id, new_id, model, session):
"""
Updates the tuple matching *old_id* with *new_id*, and updates all
dependent tuples in other tables as well.
"""
# Updating either the tuple or the dependent tuples first would
# cause integrity violations if the transaction is flushed in
# between. The order doesn't matter.
if model is None:
raise ValueError("null model given to update_local_id subtransaction")
# must load fully, don't know yet why
obj = query_model(session, model).\
filter_by(**{get_pk(model): old_id}).first()
setattr(obj, get_pk(model), new_id)
# Then the dependent ones
related_tables = get_related_tables(model)
mapped_fks = ifilter(
lambda (m, fks): m is not None and fks,
[(core.synched_models.tables.get(t.name, core.null_model).model,
get_fks(t,
class_mapper(model).mapped_table)) for t in related_tables])
for model, fks in mapped_fks:
for fk in fks:
for obj in query_model(session, model).filter_by(**{fk: old_id}):
setattr(obj, fk, new_id)
session.flush() # raise integrity errors now
def update_local_id(old_id, new_id, model, session):
"""
Updates the tuple matching *old_id* with *new_id*, and updates all
dependent tuples in other tables as well.
"""
# Updating either the tuple or the dependent tuples first would
# cause integrity violations if the transaction is flushed in
# between. The order doesn't matter.
if model is None:
raise ValueError("null model given to update_local_id subtransaction")
# must load fully, don't know yet why
obj = query_model(session, model).\
filter_by(**{get_pk(model): old_id}).first()
setattr(obj, get_pk(model), new_id)
# Then the dependent ones
related_tables = get_related_tables(model)
mapped_fks = ifilter(
lambda (m, fks): m is not None and fks,
[(core.synched_models.tables.get(t.name, core.null_model).model,
get_fks(t, class_mapper(model).mapped_table))
for t in related_tables])
for model, fks in mapped_fks:
for fk in fks:
for obj in query_model(session, model).filter_by(**{fk: old_id}):
setattr(obj, fk, new_id)
session.flush() # raise integrity errors now
def fill_for(self,
request,
swell=False,
include_extensions=True,
session=None):
"""
Fills this pull message (response) with versions, operations
and objects, for the given request (PullRequestMessage).
The *swell* parameter is deprecated and considered ``True``
regardless of the value given. This means that parent objects
will always be added to the message.
*include_extensions* dictates whether the pull message will
include model extensions or not.
"""
assert isinstance(request, PullRequestMessage), "invalid request"
versions = session.query(Version)
if request.latest_version_id is not None:
versions = versions.\
filter(Version.version_id > request.latest_version_id)
required_objects = {}
required_parents = {}
for v in versions:
self.versions.append(v)
for op in v.operations:
model = op.tracked_model
if model is None:
raise ValueError("operation linked to model %s "\
"which isn't being tracked" % model)
if model not in pulled_models: continue
self.operations.append(op)
if op.command != 'd':
pks = required_objects.get(model, set())
pks.add(op.row_id)
required_objects[model] = pks
for model, pks in ((m, batch)
for m, pks in required_objects.iteritems()
for batch in grouper(pks, MAX_SQL_VARIABLES)):
for obj in query_model(session, model).filter(
getattr(model, get_pk(model)).in_(list(pks))).all():
self.add_object(obj, include_extensions=include_extensions)
# add parent objects to resolve conflicts in merge
for pmodel, ppk in parent_references(
obj, synched_models.models.keys()):
parent_pks = required_parents.get(pmodel, set())
parent_pks.add(ppk)
required_parents[pmodel] = parent_pks
for pmodel, ppks in ((m, batch)
for m, pks in required_parents.iteritems()
for batch in grouper(pks, MAX_SQL_VARIABLES)):
for parent in query_model(session, pmodel).filter(
getattr(pmodel, get_pk(pmodel)).in_(list(ppks))).all():
self.add_object(parent, include_extensions=include_extensions)
return self
def related_local_ids(operation, session):
"""
For the given operation, return a set of row id values mapped to
content type ids that correspond to objects that are dependent by
foreign key on the object being operated upon. The lookups are
performed in the local database.
"""
parent_model = operation.tracked_model
if parent_model is None:
return set()
related_tables = get_related_tables(parent_model)
mapped_fks = [
m_fks
for m_fks
in [
(
synched_models.tables.get(entity_name(t), null_model).model,
get_fks(t, class_mapper(parent_model).mapped_table)
)
for t
in related_tables
]
if m_fks[0] is not None and m_fks[1]
]
try:
return set(
(pk, ct.id)
for pk, ct
in (
(getattr(obj, get_pk(obj)), synched_models.models.get(model, None))
for model, fks in mapped_fks
for obj in query_model(session, model) \
# removed the pk_only param, because that fails with joins
.filter(
or_(
*(
getattr(model, fk) == operation.row_id
for fk
in fks
)
)
).all()
)
if ct is not None
)
except Exception as ex:
logger.exception(f"collecting conflicts failed: {ex}")
r0 = [query_model(session, model) for model, fks in mapped_fks]
r1 = [query_model(session, model).all() for model, fks in mapped_fks]
raise
def fill_for(self, request, swell=False, include_extensions=True, session=None):
"""
Fills this pull message (response) with versions, operations
and objects, for the given request (PullRequestMessage).
The *swell* parameter is deprecated and considered ``True``
regardless of the value given. This means that parent objects
will always be added to the message.
*include_extensions* dictates whether the pull message will
include model extensions or not.
"""
assert isinstance(request, PullRequestMessage), "invalid request"
versions = session.query(Version)
if request.latest_version_id is not None:
versions = versions.filter(Version.version_id > request.latest_version_id)
required_objects = {}
required_parents = {}
for v in versions:
self.versions.append(v)
for op in v.operations:
model = op.tracked_model
if model is None:
raise ValueError("operation linked to model %s " "which isn't being tracked" % model)
if model not in pulled_models:
continue
self.operations.append(op)
if op.command != "d":
pks = required_objects.get(model, set())
pks.add(op.row_id)
required_objects[model] = pks
for model, pks in (
(m, batch) for m, pks in required_objects.iteritems() for batch in grouper(pks, MAX_SQL_VARIABLES)
):
for obj in query_model(session, model).filter(getattr(model, get_pk(model)).in_(list(pks))).all():
self.add_object(obj, include_extensions=include_extensions)
# add parent objects to resolve conflicts in merge
for pmodel, ppk in parent_references(obj, synched_models.models.keys()):
parent_pks = required_parents.get(pmodel, set())
parent_pks.add(ppk)
required_parents[pmodel] = parent_pks
for pmodel, ppks in (
(m, batch) for m, pks in required_parents.iteritems() for batch in grouper(pks, MAX_SQL_VARIABLES)
):
for parent in query_model(session, pmodel).filter(getattr(pmodel, get_pk(pmodel)).in_(list(ppks))).all():
self.add_object(parent, include_extensions=include_extensions)
return self
def related_local_ids(operation, session):
"""
For the given operation, return a set of row id values mapped to
content type ids that correspond to objects that are dependent by
foreign key on the object being operated upon. The lookups are
performed in the local database.
"""
parent_model = operation.tracked_model
if parent_model is None:
return set()
related_tables = get_related_tables(parent_model)
mapped_fks = ifilter(
lambda (m, fks): m is not None and fks,
[(synched_models.tables.get(t.name, null_model).model,
get_fks(t, class_mapper(parent_model).mapped_table))
for t in related_tables])
return set(
(pk, ct.id)
for pk, ct in \
((getattr(obj, get_pk(obj)), synched_models.models.get(model, None))
for model, fks in mapped_fks
for obj in query_model(session, model, only_pk=True).\
filter(or_(*(getattr(model, fk) == operation.row_id
for fk in fks))).all())
if ct is not None)
def add_unversioned_operations(self, session=None, include_extensions=True):
"""
Adds all unversioned operations to this message, including the
required objects for them to be performed.
"""
operations = session.query(Operation).\
filter(Operation.version_id == None).all()
if any(op.content_type_id not in synched_models.ids
for op in operations):
raise ValueError("version includes operation linked "\
"to model not currently being tracked")
required_objects = {}
for op in operations:
model = op.tracked_model
if model not in pushed_models: continue
self.operations.append(op)
if op.command != 'd':
pks = required_objects.get(model, set())
pks.add(op.row_id)
required_objects[model] = pks
for model, pks in ((m, batch)
for m, pks in required_objects.iteritems()
for batch in grouper(pks, MAX_SQL_VARIABLES)):
for obj in query_model(session, model).filter(
getattr(model, get_pk(model)).in_(list(pks))).all():
self.add_object(obj, include_extensions=include_extensions)
if self.key is not None:
# overwrite since it's probably an incorrect key
self._sign()
return self
def related_local_ids(operation, session):
"""
For the given operation, return a set of row id values mapped to
content type ids that correspond to objects that are dependent by
foreign key on the object being operated upon. The lookups are
performed in the local database.
"""
parent_model = operation.tracked_model
if parent_model is None:
return set()
related_tables = get_related_tables(parent_model)
mapped_fks = ifilter(lambda (m, fks): m is not None and fks,
[(synched_models.tables.get(t.name, null_model).model,
get_fks(t,
class_mapper(parent_model).mapped_table))
for t in related_tables])
return set(
(pk, ct.id)
for pk, ct in \
((getattr(obj, get_pk(obj)), synched_models.models.get(model, None))
for model, fks in mapped_fks
for obj in query_model(session, model, only_pk=True).\
filter(or_(*(getattr(model, fk) == operation.row_id
for fk in fks))).all())
if ct is not None)
def tracked(o, **kws):
if _has_delete_functions(o):
if always: deleted.append((copy(o), None))
else:
prev = query_model(session, type(o)).filter_by(
**{get_pk(o): getattr(o, get_pk(o), None)}).\
first()
if prev is not None:
deleted.append((copy(prev), o))
return fn(o, **kws)
def find_unique_conflicts(push_message, session):
"""
Returns a list of conflicts caused by unique constraints in the
given push message contrasted against the database. Each conflict
is a dictionary with the following fields::
object: the conflicting object in database, bound to the
session
columns: tuple of column names in the unique constraint
new_values: tuple of values that can be used to update the
conflicting object.
"""
conflicts = []
for pk, model in ((op.row_id, op.tracked_model)
for op in push_message.operations if op.command != 'd'):
if model is None: continue
mt = class_mapper(model).mapped_table
if isinstance(mt, Join):
constraints = mt.left.constraints.union(mt.right.constraints)
else:
constraints = mt.constraints
for constraint in [
c for c in constraints if isinstance(c, UniqueConstraint)
]:
unique_columns = tuple(col.name for col in constraint.columns)
remote_obj = push_message.query(model).\
filter(attr('__pk__') == pk).first()
remote_values = tuple(
getattr(remote_obj, col, None) for col in unique_columns)
if all(value is None for value in remote_values): continue
local_obj = query_model(session, model).\
filter_by(**dict(list(zip(unique_columns, remote_values)))).first()
if local_obj is None: continue
local_pk = getattr(local_obj, get_pk(model))
if local_pk == pk: continue
push_obj = push_message.query(model).\
filter(attr('__pk__') == local_pk).first()
if push_obj is None: continue # push will fail
conflicts.append({
'object':
local_obj,
'columns':
unique_columns,
'new_values':
tuple(getattr(push_obj, col) for col in unique_columns)
})
return conflicts
def verify_constraint(model, columns, values):
"""
Checks to see whether some local object exists with
conflicting values.
"""
match = query_model(session, model, only_pk=True).\
options(*(undefer(column) for column in columns)).\
filter_by(**dict((column, value)
for column, value in izip(columns, values))).first()
pk = get_pk(model)
return match, getattr(match, pk, None)
def verify_constraint(model, columns, values):
"""
Checks to see whether some local object exists with
conflicting values.
"""
match = query_model(session, model, only_pk=True).\
options(*(undefer(column) for column in columns)).\
filter_by(**dict((column, value)
for column, value in izip(columns, values))).first()
pk = get_pk(model)
return match, getattr(match, pk, None)
def handle_repair(data=None, session=None):
"Handle repair request. Return whole server database."
include_extensions = 'exclude_extensions' not in (data or {})
latest_version_id = core.get_latest_version_id(session=session)
message = BaseMessage()
for model in list(core.synched_models.models.keys()):
for obj in query_model(session, model):
message.add_object(obj, include_extensions=include_extensions)
response = message.to_json()
response['latest_version_id'] = latest_version_id
return response
def handle_repair(data=None, session=None):
"Handle repair request. Return whole server database."
include_extensions = 'exclude_extensions' not in (data or {})
latest_version_id = core.get_latest_version_id(session=session)
message = BaseMessage()
for model in core.synched_models.models.iterkeys():
for obj in query_model(session, model):
message.add_object(obj, include_extensions=include_extensions)
response = message.to_json()
response['latest_version_id'] = latest_version_id
return response
def handle_query(data, session=None):
"Responds to a query request."
model = core.synched_models.model_names.\
get(data.get('model', None), core.null_model).model
if model is None: return None
mname = model.__name__
filters = dict((k, v) for k, v in ((k[len(mname) + 1:], v)
for k, v in data.iteritems()
if k.startswith(mname + '_'))
if k and k in column_properties(model))
message = BaseMessage()
q = query_model(session, model)
if filters:
q = q.filter_by(**filters)
for obj in q:
message.add_object(obj)
return message.to_json()
def find_unique_conflicts(push_message, session):
"""
Returns a list of conflicts caused by unique constraints in the
given push message contrasted against the database. Each conflict
is a dictionary with the following fields::
object: the conflicting object in database, bound to the
session
columns: tuple of column names in the unique constraint
new_values: tuple of values that can be used to update the
conflicting object.
"""
conflicts = []
for pk, model in ((op.row_id, op.tracked_model)
for op in push_message.operations
if op.command != 'd'):
if model is None: continue
for constraint in ifilter(lambda c: isinstance(c, UniqueConstraint),
class_mapper(model).mapped_table.constraints):
unique_columns = tuple(col.name for col in constraint.columns)
remote_obj = push_message.query(model).\
filter(attr('__pk__') == pk).first()
remote_values = tuple(getattr(remote_obj, col, None)
for col in unique_columns)
if all(value is None for value in remote_values): continue
local_obj = query_model(session, model).\
filter_by(**dict(izip(unique_columns, remote_values))).first()
if local_obj is None: continue
local_pk = getattr(local_obj, get_pk(model))
if local_pk == pk: continue
push_obj = push_message.query(model).\
filter(attr('__pk__') == local_pk).first()
if push_obj is None: continue # push will fail
conflicts.append(
{'object': local_obj,
'columns': unique_columns,
'new_values': tuple(getattr(push_obj, col)
for col in unique_columns)})
return conflicts
def handle_query(data, session=None):
"Responds to a query request."
model = core.synched_models.model_names.\
get(data.get('model', None), core.null_model).model
if model is None: return None
mname = model.__name__
filters = dict((k, v) for k, v in ((k[len(mname) + 1:], v)
for k, v in list(data.items())
if k.startswith(mname + '_'))
if k and k in column_properties(model))
message = BaseMessage()
q = query_model(session, model)
if filters:
q = q.filter_by(**filters)
for obj in q:
message.add_object(obj)
return message.to_json()
def add_operation(self, op, swell=True, session=None):
"""
Adds an operation to the message, including the required
object if it's possible to include it.
If *swell* is given and set to ``False``, the operation and
object will be added bare, without parent objects. Otherwise,
the parent objects will be added to aid in conflict
resolution.
A delete operation doesn't include the associated object. If
*session* is given, the procedure won't instantiate a new
session.
This operation might fail, (due to database inconsitency) in
which case the internal state of the message won't be affected
(i.e. it won't end in an inconsistent state).
DEPRECATED in favor of `fill_for`
"""
model = op.tracked_model
if model is None:
raise ValueError("operation linked to model %s "\
"which isn't being tracked" % model)
if model not in pulled_models:
return self
obj = query_model(session, model).\
filter_by(**{get_pk(model): op.row_id}).first() \
if op.command != 'd' else None
self.operations.append(op)
# if the object isn't there it's because the operation is old,
# and should be able to be compressed out when performing the
# conflict resolution phase
if obj is not None:
self.add_object(obj)
if swell:
# add parent objects to resolve possible conflicts in merge
for parent in parent_objects(obj, synched_models.models.keys(),
session):
self.add_object(parent)
return self
def add_operation(self, op, swell=True, session=None):
"""
Adds an operation to the message, including the required
object if it's possible to include it.
If *swell* is given and set to ``False``, the operation and
object will be added bare, without parent objects. Otherwise,
the parent objects will be added to aid in conflict
resolution.
A delete operation doesn't include the associated object. If
*session* is given, the procedure won't instantiate a new
session.
This operation might fail, (due to database inconsitency) in
which case the internal state of the message won't be affected
(i.e. it won't end in an inconsistent state).
DEPRECATED in favor of `fill_for`
"""
model = op.tracked_model
if model is None:
raise ValueError("operation linked to model %s " "which isn't being tracked" % model)
if model not in pulled_models:
return self
obj = query_model(session, model).filter_by(**{get_pk(model): op.row_id}).first() if op.command != "d" else None
self.operations.append(op)
# if the object isn't there it's because the operation is old,
# and should be able to be compressed out when performing the
# conflict resolution phase
if obj is not None:
self.add_object(obj)
if swell:
# add parent objects to resolve possible conflicts in merge
for parent in parent_objects(obj, synched_models.models.keys(), session):
self.add_object(parent)
return self
def fill_for(self,
request,
swell=False,
include_extensions=True,
session=None,
connection=None,
**kw):
"""
Fills this pull message (response) with versions, operations
and objects, for the given request (PullRequestMessage).
The *swell* parameter is deprecated and considered ``True``
regardless of the value given. This means that parent objects
will always be added to the message.
*include_extensions* dictates whether the pull message will
include model extensions or not.
"""
assert isinstance(request, PullRequestMessage), "invalid request"
versions = session.query(Version)
if request.latest_version_id is not None:
versions = versions. \
filter(Version.version_id > request.latest_version_id)
required_objects = {}
required_parents = {}
# TODO: since there can be really many versions here
# we should rebuild this part so that we make an aggregate query
# gettting all operations ordered by their version and by their permissions
# something like
# select * from
# operation, version
# where
# operation.version_id=version.id
# and
# version.version_id > request.latest_version_id
# and
# " one of the user's roles is in operation.allowed_users_and_roles "
# order by
# operation.order
#
# the basic query can be done here,
# per dep injection we must add the query for allowed_users
self.versions = versions.all()
ops: Query = session.query(Operation)
if request.latest_version_id is not None:
ops = ops.filter(Operation.version_id > request.latest_version_id)
ops = call_filter_operations(connection, session, ops)
ops = ops.order_by(Operation.order)
self.operations = []
logger.info(f"request.latest_version_id = {request.latest_version_id}")
logger.info(f"querying for {ops}")
# logger.info(f"query result: {ops.all()}")
logger.info(f"query result #ops: {len(ops.all())}")
for op in ops:
model = op.tracked_model
if model is None:
logger.warn(
f"op {op} has no model (perhaps removed from tracking)")
# raise ValueError("operation linked to model %s " \
# "which isn't being tracked" % model)
if model not in pulled_models: continue
obj = query_model(session, model).get(op.row_id)
if obj is None:
if op.command != 'd':
logger.error(
f"this should not happen, obj is None for op: {op} - ignoring"
)
continue
try:
call_before_server_add_operation_fn(connection, session, op,
obj)
self.operations.append(op)
except SkipOperation:
continue
if op.command != 'd':
pks = required_objects.get(model, set())
pks.add(op.row_id)
required_objects[model] = pks
for model, pks in ((m, batch)
for m, pks in list(required_objects.items())
for batch in grouper(pks, MAX_SQL_VARIABLES)):
for obj in query_model(session, model).filter(
getattr(model, get_pk(model)).in_(list(pks))).all():
self.add_object(obj, include_extensions=include_extensions)
# add parent objects to resolve conflicts in merge
for pmodel, ppk in parent_references(
obj, list(synched_models.models.keys())):
parent_pks = required_parents.get(pmodel, set())
parent_pks.add(ppk)
required_parents[pmodel] = parent_pks
for pmodel, ppks in ((m, batch)
for m, pks in list(required_parents.items())
for batch in grouper(pks, MAX_SQL_VARIABLES)):
for parent in query_model(session, pmodel).filter(
getattr(pmodel, get_pk(pmodel)).in_(list(ppks))).all():
self.add_object(parent, include_extensions=include_extensions)
logger.info(f"operations result: {self.operations}")
return self
async def perform_async(
operation: "Operation",
container: "BaseMessage",
session: Session,
node_id=None,
websocket: Optional[WebSocketCommonProtocol] = None
) -> (Optional[SQLClass], Optional[SQLClass]):
"""
Performs *operation*, looking for required data in
*container*, and using *session* to perform it.
*container* is an instance of
dbsync.messages.base.BaseMessage.
*node_id* is the node responsible for the operation, if known
(else ``None``).
If at any moment this operation fails for predictable causes,
it will raise an *OperationError*.
"""
from dbsync.core import mode
model: DeclarativeMeta = operation.tracked_model
res: Tuple[Optional[SQLClass], Optional[SQLClass]] = (None, None)
if model is None:
raise OperationError("no content type for this operation",
operation)
if operation.command == 'i':
# check if the given object is already in the database
obj = query_model(session, model). \
filter(getattr(model, get_pk(model)) == operation.row_id).first()
# retrieve the object from the PullMessage
qu = container.query(model). \
filter(attr('__pk__') == operation.row_id)
# breakpoint()
pull_obj = qu.first()
# pull_obj._session = session
if pull_obj is None:
raise OperationError(
f"no object backing the operation in container on {mode}",
operation)
if obj is None:
logger.info(
f"insert: calling request_payloads_for_extension for: {pull_obj.id}"
)
try:
operation.call_before_operation_fn(session, pull_obj)
await request_payloads_for_extension(
operation, pull_obj, websocket, session)
session.add(pull_obj)
res = pull_obj, None
except SkipOperation as e:
logger.info(f"operation {operation} skipped")
# operation.call_after_operation_fn(pull_obj, session)
else:
# Don't raise an exception if the incoming object is
# exactly the same as the local one.
if properties_dict(obj) == properties_dict(pull_obj):
logger.warning("insert attempted when an identical object "
"already existed in local database: "
"model {0} pk {1}".format(
model.__name__, operation.row_id))
else:
raise OperationError(
"insert attempted when the object already existed: "
"model {0} pk {1}".format(model.__name__,
operation.row_id))
elif operation.command == 'u':
obj = query_model(session, model). \
filter(getattr(model, get_pk(model)) == operation.row_id).one_or_none()
if obj is not None:
logger.info(
f"update: calling request_payloads_for_extension for: {obj.id}"
)
# breakpoint()
else:
# For now, the record will be created again, but is an
# error because nothing should be deleted without
# using dbsync
# raise OperationError(
# "the referenced object doesn't exist in database", operation)
# addendum:
# this can happen when tracking of an object has been suppressed and
# later been activated during a 'u' operation,
# so we keep this logic
logger.warning(
"The referenced object doesn't exist in database. "
"Node %s. Operation %s", node_id, operation)
# get new object from the PushMessage
pull_obj = container.query(model). \
filter(attr('__pk__') == operation.row_id).first()
if pull_obj is None:
raise OperationError(
"no object backing the operation in container", operation)
try:
operation.call_before_operation_fn(session, pull_obj, obj)
await request_payloads_for_extension(operation, pull_obj,
websocket, session)
if obj is None:
logger.warn(f"obj is None")
old_obj = copy(obj) if obj is not None else None
session.merge(pull_obj)
res = pull_obj, old_obj
except SkipOperation as e:
logger.info(f"operation {operation} skipped")
# operation.call_after_operation_fn(pull_obj, session)
elif operation.command == 'd':
try:
obj = query_model(session, model, only_pk=True). \
filter(getattr(model, get_pk(model)) == operation.row_id).first()
except NoSuchColumnError as ex:
# for joins only_pk doesnt seem to work
obj = query_model(session, model, only_pk=False). \
filter(getattr(model, get_pk(model)) == operation.row_id).first()
if obj is None:
# The object is already deleted in the server
# The final state in node and server are the same. But
# it's an error because nothing should be deleted
# without using dbsync
logger.warning(
"The referenced object doesn't exist in database. "
"Node %s. Operation %s", node_id, operation)
else:
try:
# breakpoint()
operation.call_before_operation_fn(session, obj)
session.delete(obj)
res = obj, None
except SkipOperation as e:
logger.info(f"operation {operation} skipped")
else:
raise OperationError(
"the operation doesn't specify a valid command ('i', 'u', 'd')",
operation)
return res
def compress(session=None) -> List[Operation]:
"""
Compresses unversioned operations in the database.
For each row in the operations table, this deletes unnecesary
operations that would otherwise bloat the message.
This procedure is called internally before the 'push' request
happens, and before the local 'merge' happens.
"""
unversioned: Query = session.query(Operation).\
filter(Operation.version_id == None).order_by(Operation.order.desc())
seqs = group_by(lambda op: (op.row_id, op.content_type_id), unversioned)
# Check errors on sequences
for seq in list(seqs.values()):
_assert_operation_sequence(seq, session)
for seq in [seq for seq in iter(list(seqs.values())) if len(seq) > 1]:
if seq[-1].command == 'i':
if all(op.command == 'u' for op in seq[:-1]):
# updates are superfluous
list(map(session.delete, seq[:-1]))
elif seq[0].command == 'd':
# it's as if the object never existed
list(map(session.delete, seq))
elif seq[-1].command == 'u':
if all(op.command == 'u' for op in seq[:-1]):
# leave a single update
list(map(session.delete, seq[1:]))
elif seq[0].command == 'd':
# leave the delete statement
list(map(session.delete, seq[1:]))
session.flush()
# repair inconsistencies
for operation in session.query(Operation).\
filter(Operation.version_id == None).\
order_by(Operation.order.desc()).all():
session.flush()
model = operation.tracked_model
if not model:
logger.error("operation linked to content type "
"not tracked: %s" % operation.content_type_id)
continue
if operation.command in ('i', 'u'):
if query_model(session, model, only_pk=True).\
filter_by(**{get_pk(model): operation.row_id}).count() == 0:
logger.warning("deleting operation %s for model %s "
"for absence of backing object" %
(operation, model.__name__))
session.delete(operation)
continue
if operation.command == 'u':
subsequent = session.query(Operation).\
filter(Operation.content_type_id == operation.content_type_id,
Operation.version_id == None,
Operation.row_id == operation.row_id,
Operation.order > operation.order).all()
if any(op.command == 'i' for op in subsequent) and \
all(op.command != 'd' for op in subsequent):
logger.warning(
"deleting update operation %s for model %s "
"for preceding an insert operation" %\
(operation, model.__name__))
session.delete(operation)
continue
if session.query(Operation).\
filter(Operation.content_type_id == operation.content_type_id,
Operation.command == operation.command,
Operation.version_id == None,
Operation.row_id == operation.row_id,
Operation.order != operation.order).count() > 0:
logger.warning(
"deleting operation %s for model %s "
"for being redundant after compression" %\
(operation, model.__name__))
session.delete(operation)
continue
session.commit()
return session.query(Operation).\
filter(Operation.version_id == None).\
order_by(Operation.order.asc()).all()
def compress(session=None):
"""
Compresses unversioned operations in the database.
For each row in the operations table, this deletes unnecesary
operations that would otherwise bloat the message.
This procedure is called internally before the 'push' request
happens, and before the local 'merge' happens.
"""
unversioned = session.query(Operation).\
filter(Operation.version_id == None).order_by(Operation.order.desc())
seqs = group_by(lambda op: (op.row_id, op.content_type_id), unversioned)
# Check errors on sequences
for seq in seqs.itervalues():
_assert_operation_sequence(seq, session)
for seq in ifilter(lambda seq: len(seq) > 1, seqs.itervalues()):
if seq[-1].command == 'i':
if all(op.command == 'u' for op in seq[:-1]):
# updates are superfluous
map(session.delete, seq[:-1])
elif seq[0].command == 'd':
# it's as if the object never existed
map(session.delete, seq)
elif seq[-1].command == 'u':
if all(op.command == 'u' for op in seq[:-1]):
# leave a single update
map(session.delete, seq[1:])
elif seq[0].command == 'd':
# leave the delete statement
map(session.delete, seq[1:])
session.flush()
# repair inconsistencies
for operation in session.query(Operation).\
filter(Operation.version_id == None).\
order_by(Operation.order.desc()).all():
session.flush()
model = operation.tracked_model
if not model:
logger.error(
"operation linked to content type "
"not tracked: %s" % operation.content_type_id)
continue
if operation.command in ('i', 'u'):
if query_model(session, model, only_pk=True).\
filter_by(**{get_pk(model): operation.row_id}).count() == 0:
logger.warning(
"deleting operation %s for model %s "
"for absence of backing object" % (operation, model.__name__))
session.delete(operation)
continue
if operation.command == 'u':
subsequent = session.query(Operation).\
filter(Operation.content_type_id == operation.content_type_id,
Operation.version_id == None,
Operation.row_id == operation.row_id,
Operation.order > operation.order).all()
if any(op.command == 'i' for op in subsequent) and \
all(op.command != 'd' for op in subsequent):
logger.warning(
"deleting update operation %s for model %s "
"for preceding an insert operation" %\
(operation, model.__name__))
session.delete(operation)
continue
if session.query(Operation).\
filter(Operation.content_type_id == operation.content_type_id,
Operation.command == operation.command,
Operation.version_id == None,
Operation.row_id == operation.row_id,
Operation.order != operation.order).count() > 0:
logger.warning(
"deleting operation %s for model %s "
"for being redundant after compression" %\
(operation, model.__name__))
session.delete(operation)
continue
return session.query(Operation).\
filter(Operation.version_id == None).\
order_by(Operation.order.asc()).all()
def perform(operation, container, session, node_id=None):
"""
Performs *operation*, looking for required data in
*container*, and using *session* to perform it.
*container* is an instance of
dbsync.messages.base.BaseMessage.
*node_id* is the node responsible for the operation, if known
(else ``None``).
If at any moment this operation fails for predictable causes,
it will raise an *OperationError*.
"""
model = operation.tracked_model
if model is None:
raise OperationError("no content type for this operation", operation)
if operation.command == 'i':
obj = query_model(session, model).\
filter(getattr(model, get_pk(model)) == operation.row_id).first()
pull_obj = container.query(model).\
filter(attr('__pk__') == operation.row_id).first()
if pull_obj is None:
raise OperationError(
"no object backing the operation in container", operation)
if obj is None:
session.add(pull_obj)
else:
# Don't raise an exception if the incoming object is
# exactly the same as the local one.
if properties_dict(obj) == properties_dict(pull_obj):
logger.warning(u"insert attempted when an identical object "
u"already existed in local database: "
u"model {0} pk {1}".format(model.__name__,
operation.row_id))
else:
raise OperationError(
u"insert attempted when the object already existed: "
u"model {0} pk {1}".format(model.__name__,
operation.row_id))
elif operation.command == 'u':
obj = query_model(session, model).\
filter(getattr(model, get_pk(model)) == operation.row_id).first()
if obj is None:
# For now, the record will be created again, but is an
# error because nothing should be deleted without
# using dbsync
# raise OperationError(
# "the referenced object doesn't exist in database", operation)
logger.warning(
u"The referenced object doesn't exist in database. "
u"Node %s. Operation %s",
node_id,
operation)
pull_obj = container.query(model).\
filter(attr('__pk__') == operation.row_id).first()
if pull_obj is None:
raise OperationError(
"no object backing the operation in container", operation)
session.merge(pull_obj)
elif operation.command == 'd':
obj = query_model(session, model, only_pk=True).\
filter(getattr(model, get_pk(model)) == operation.row_id).first()
if obj is None:
# The object is already deleted in the server
# The final state in node and server are the same. But
# it's an error because nothing should be deleted
# without using dbsync
logger.warning(
"The referenced object doesn't exist in database. "
u"Node %s. Operation %s",
node_id,
operation)
else:
session.delete(obj)
else:
raise OperationError(
"the operation doesn't specify a valid command ('i', 'u', 'd')",
operation)
