def request_push(
push_url, extra_data=None, encode=None, decode=None, headers=None, timeout=None, extensions=True, session=None
):
message = PushMessage()
message.latest_version_id = core.get_latest_version_id(session=session)
compress(session=session)
message.add_unversioned_operations(session=session, include_extensions=extensions)
message.set_node(session.query(Node).order_by(Node.node_id.desc()).first())
data = message.to_json()
data.update({"extra_data": extra_data or {}})
code, reason, response = post_request(push_url, data, encode, decode, headers, timeout)
if (code // 100 != 2) or response is None:
if suggests_pull is not None and suggests_pull(code, reason, response):
raise PullSuggested(code, reason, response)
raise PushRejected(code, reason, response)
new_version_id = response.get("new_version_id")
if new_version_id is None:
raise PushRejected(code, reason, {"error": "server didn't respond with new version id", "response": response})
# Who should set the dates? Maybe send a complete Version from the
# server. For now the field is ignored, so it doesn't matter.
session.add(Version(version_id=new_version_id, created=datetime.datetime.now()))
for op in message.operations:
op.version_id = new_version_id
# return the response for the programmer to do what she wants
# afterwards
return response
def trim(session=None):
"""
Clears space by deleting operations and versions that are no
longer needed.
This might cause the server to answer incorrectly to pull requests
from nodes that were late to register. To go around that, a repair
should be enforced after the node's register.
Another problem with this procedure is that it won't clear space
if there's at least one abandoned node registered. The task of
keeping the nodes registry clean of those is left to the
programmer.
"""
versions = [maybe(session.query(Version).\
filter(Version.node_id == node.node_id).\
order_by(Version.version_id.desc()).first(),
attr('version_id'),
None)
for node in session.query(Node)]
if not versions:
last_id = core.get_latest_version_id(session=session)
# all operations are versioned according to dbsync.server.track
session.query(Operation).delete()
session.query(Version).filter(Version.version_id != last_id).delete()
return
if None in versions: return # dead nodes block the trim
minversion = min(versions)
session.query(Operation).filter(Operation.version_id <= minversion).delete()
session.query(Version).filter(Version.version_id < minversion).delete()
def handle_repair(data=None, session=None):
"Handle repair request. Return whole server database."
include_extensions = 'exclude_extensions' not in (data or {})
latest_version_id = core.get_latest_version_id(session=session)
message = BaseMessage()
for model in core.synched_models.models.iterkeys():
for obj in query_model(session, model):
message.add_object(obj, include_extensions=include_extensions)
response = message.to_json()
response['latest_version_id'] = latest_version_id
return response
def handle_repair(data=None, session=None):
"Handle repair request. Return whole server database."
include_extensions = 'exclude_extensions' not in (data or {})
latest_version_id = core.get_latest_version_id(session=session)
message = BaseMessage()
for model in list(core.synched_models.models.keys()):
for obj in query_model(session, model):
message.add_object(obj, include_extensions=include_extensions)
response = message.to_json()
response['latest_version_id'] = latest_version_id
return response
def __init__(self, raw_data=None):
"""
*raw_data* must be a python dictionary. If not given, the
message should be filled with the or
add_unversioned_operations method.
"""
super(PullRequestMessage, self).__init__(raw_data)
if raw_data is not None:
self._build_from_raw(raw_data)
else:
self.latest_version_id = get_latest_version_id()
self.operations = []
def __init__(self, raw_data=None):
"""
*raw_data* must be a python dictionary. If not given, the
message should be filled with the or
add_unversioned_operations method.
"""
super(PullRequestMessage, self).__init__(raw_data)
if raw_data is not None:
self._build_from_raw(raw_data)
else:
self.latest_version_id = get_latest_version_id()
self.operations = []
def request_push(push_url: str,
extra_data=None,
encode=None,
decode=None,
headers=None,
timeout=None,
extensions=True,
session=None) -> Dict[str, Any]:
message = PushMessage()
message.latest_version_id = core.get_latest_version_id(session=session)
compress(session=session)
message.add_unversioned_operations(session=session,
include_extensions=extensions)
if not message.operations:
return {}
message.set_node(session.query(Node).order_by(Node.node_id.desc()).first())
data = message.to_json()
data.update({'extra_data': extra_data or {}})
code, reason, response = post_request(push_url, data, encode, decode,
headers, timeout)
if (code // 100 != 2) or response is None:
if suggests_pull is not None and suggests_pull(code, reason, response):
raise PullSuggested(code, reason, response)
raise PushRejected(code, reason, response)
new_version_id = response.get('new_version_id')
if new_version_id is None:
raise PushRejected(
code, reason, {
'error': "server didn't respond with new version id",
'response': response
})
# Who should set the dates? Maybe send a complete Version from the
# server. For now the field is ignored, so it doesn't matter.
session.add(
Version(version_id=new_version_id, created=datetime.datetime.now()))
for op in message.operations:
op.version_id = new_version_id
# return the response for the programmer to do what she wants
# afterwards
return response
def create_push_message(self,
session: Optional[
sqlalchemy.orm.session.Session] = None,
extensions=True,
do_compress=True) -> PushMessage:
# TODO: mit do_compress=True muss noch getestet werden, welche Szenarien die referentielle Integritaet
# verletzen koennen. Denn wenn die Tabellen in richtiger Reihenfolge synchronisiert werden
# koennte man auf das Aussetzen der RI verzichten
if not session:
session = self.Session() # TODO: p
# breakpoint()
message = PushMessage()
message.latest_version_id = core.get_latest_version_id(session=session)
if do_compress:
compress(session=session)
message.add_unversioned_operations(session=session,
include_extensions=extensions)
return message
def trim(session=None):
"Trims the internal synchronization tables, to free space."
last_id = core.get_latest_version_id(session=session)
session.query(Operation).filter(Operation.version_id != None).delete()
session.query(Version).filter(Version.version_id != last_id).delete()
async def handle_push(connection: Connection,
session: sqlalchemy.orm.Session) -> Optional[int]:
msgs_got = 0
version: Optional[Version] = None
async for msg in connection.socket:
msgs_got += 1
msg_json = json.loads(msg)
pushmsg = PushMessage(msg_json)
# print(f"pushmsg: {msg}")
if not pushmsg.operations:
logger.warn("empty operations list in client PushMessage")
for op in pushmsg.operations:
logger.info(f"operation: {op}")
# await connection.socket.send(f"answer is:{msg}")
logger.info(f"message key={pushmsg.key}")
latest_version_id = core.get_latest_version_id(session=session)
logger.info(
f"** version on server:{latest_version_id}, version in pushmsg:{pushmsg.latest_version_id}"
)
if latest_version_id != pushmsg.latest_version_id:
exc = f"version identifier isn't the latest one; " \
f"incoming: {pushmsg.latest_version_id}, on server:{latest_version_id}"
if latest_version_id is None:
logger.warn(exc)
raise PushRejected(exc)
if pushmsg.latest_version_id is None:
logger.warn(exc)
raise PullSuggested(exc)
if pushmsg.latest_version_id < latest_version_id:
logger.warn(exc)
raise PullSuggested(exc)
raise PushRejected(exc)
if not pushmsg.islegit(session):
raise PushRejected("message isn't properly signed")
for listener in before_push:
listener(session, pushmsg)
# I) detect unique constraint conflicts and resolve them if possible
unique_conflicts = find_unique_conflicts(pushmsg, session)
conflicting_objects = set()
for uc in unique_conflicts:
obj = uc['object']
conflicting_objects.add(obj)
for key, value in zip(uc['columns'], uc['new_values']):
setattr(obj, key, value)
for obj in conflicting_objects:
make_transient(obj) # remove from session
for model in set(type(obj) for obj in conflicting_objects):
pk_name = get_pk(model)
pks = [
getattr(obj, pk_name) for obj in conflicting_objects
if type(obj) is model
]
session.query(model).filter(getattr(model, pk_name).in_(pks)). \
delete(synchronize_session=False) # remove from the database
session.add_all(conflicting_objects) # reinsert
session.flush()
# II) perform the operations
operations = [
o for o in pushmsg.operations if o.tracked_model is not None
]
post_operations: List[Tuple[Operation, SQLClass,
Optional[SQLClass]]] = []
try:
op: Operation
for op in operations:
(obj,
old_obj) = await op.perform_async(pushmsg, session,
pushmsg.node_id,
connection.socket)
if obj is not None:
# if the op has been skipped, it wont be appended for post_operation handling
post_operations.append((op, obj, old_obj))
resp = dict(type="info",
op=dict(
row_id=op.row_id,
version=op.version,
command=op.command,
content_type_id=op.content_type_id,
))
call_after_tracking_fn(session, op, obj)
await connection.socket.send(json.dumps(resp))
except OperationError as e:
logger.exception(
"Couldn't perform operation in push from node %s.",
pushmsg.node_id)
raise PushRejected("at least one operation couldn't be performed",
*e.args)
# III) insert a new version
if post_operations: # only if operations have been done -> create the new version
version = Version(created=datetime.datetime.now(),
node_id=pushmsg.node_id)
session.add(version)
# IV) insert the operations, discarding the 'order' column
accomplished_operations = [
op for (op, obj, old_obj) in post_operations
]
for op in sorted(accomplished_operations, key=attr('order')):
new_op = Operation()
for k in [k for k in properties_dict(op) if k != 'order']:
setattr(new_op, k, getattr(op, k))
session.add(new_op)
new_op.version = version
session.flush()
for op, obj, old_obj in post_operations:
op.call_after_operation_fn(session, obj, old_obj)
# from woodmaster.model.sql.model import WoodPile, Measurement
# orphans = session.query(Measurement).filter(Measurement.woodpile_id == None).all()
# print(f"orphans:{orphans}")
for listener in after_push:
listener(session, pushmsg)
# return the new version id back to the client
logger.info(f"version is: {version}")
if version:
await connection.socket.send(
json.dumps(
dict(type="result", new_version_id=version.version_id)))
return {'new_version_id': version.version_id}
else:
await connection.socket.send(
json.dumps(dict(type="result", new_version_id=None)))
logger.info("sent nothing message")
await connection.socket.close()
logger.info("push ready")
async def merge(pull_message, session=None, websocket=None):
"""
Merges a message from the server with the local database.
*pull_message* is an instance of dbsync.messages.pull.PullMessage.
"""
logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~begin merge")
if not isinstance(pull_message, PullMessage):
raise TypeError("need an instance of dbsync.messages.pull.PullMessage "
"to perform the local merge operation")
valid_cts = set(ct for ct in core.synched_models.ids)
unversioned_ops = compress(session=session)
pull_ops = list(filter(attr('content_type_id').in_(valid_cts),
pull_message.operations))
pull_ops = compressed_operations(pull_ops)
logger.info(f"pull_ops:{len(pull_ops)} items")
# I) first phase: resolve unique constraint conflicts if
# possible. Abort early if a human error is detected
unique_conflicts, unique_errors = find_unique_conflicts(
pull_ops, unversioned_ops, pull_message, session)
if unique_errors:
raise UniqueConstraintError(unique_errors)
conflicting_objects = set()
logger.info(f"{len(unique_conflicts)} conflicts found")
for uc in unique_conflicts:
obj = uc['object']
conflicting_objects.add(obj)
for key, value in zip(uc['columns'], uc['new_values']):
setattr(obj, key, value)
# Resolve potential cyclical conflicts by deleting and reinserting
for obj in conflicting_objects:
make_transient(obj) # remove from session
for model in set(type(obj) for obj in conflicting_objects):
pk_name = get_pk(model)
pks = [getattr(obj, pk_name)
for obj in conflicting_objects
if type(obj) is model]
session.query(model).filter(getattr(model, pk_name).in_(pks)).\
delete(synchronize_session=False) # remove from the database
session.add_all(conflicting_objects) # reinsert them
session.flush()
# II) second phase: detect conflicts between pulled operations and
# unversioned ones
direct_conflicts = find_direct_conflicts(pull_ops, unversioned_ops)
# in which the delete operation is registered on the pull message
dependency_conflicts = find_dependency_conflicts(
pull_ops, unversioned_ops, session)
# in which the delete operation was performed locally
reversed_dependency_conflicts = find_reversed_dependency_conflicts(
pull_ops, unversioned_ops, pull_message)
insert_conflicts = find_insert_conflicts(pull_ops, unversioned_ops)
# III) third phase: perform pull operations, when allowed and
# while resolving conflicts
def extract(op, conflicts):
return [local for (remote, local) in conflicts if remote is op]
def purgelocal(local):
session.delete(local)
exclude = lambda tup: tup[1] is not local
mfilter(exclude, direct_conflicts)
mfilter(exclude, dependency_conflicts)
mfilter(exclude, reversed_dependency_conflicts)
mfilter(exclude, insert_conflicts)
unversioned_ops.remove(local)
st = time.time()
logger.info(f"--> start performing pull_ops with {len(pull_ops)} ops")
for i, pull_op in enumerate(pull_ops):
# flag to control whether the remote operation is free of obstacles
can_perform = True
# flag to detect the early exclusion of a remote operation
reverted = False
# the class of the operation
class_ = pull_op.tracked_model
direct = extract(pull_op, direct_conflicts)
if direct:
if pull_op.command == 'd':
can_perform = False
for local in direct:
pair = (pull_op.command, local.command)
if pair == ('u', 'u'):
can_perform = False # favor local changes over remote ones
elif pair == ('u', 'd'):
pull_op.command = 'i' # negate the local delete
purgelocal(local)
elif pair == ('d', 'u'):
local.command = 'i' # negate the remote delete
session.flush()
reverted = True
else: # ('d', 'd')
purgelocal(local)
dependency = extract(pull_op, dependency_conflicts)
if dependency and not reverted:
can_perform = False
order = min(op.order for op in unversioned_ops)
# first move all operations further in order, to make way
# for the new one
for op in unversioned_ops:
op.order = op.order + 1
session.flush()
# then create operation to reflect the reinsertion and
# maintain a correct operation history
session.add(Operation(row_id=pull_op.row_id,
content_type_id=pull_op.content_type_id,
command='i',
order=order))
reversed_dependency = extract(pull_op, reversed_dependency_conflicts)
for local in reversed_dependency:
# reinsert record
local.command = 'i'
# local.perform(pull_message, session)
await local.perform_async(pull_message, session, websocket=websocket)
# delete trace of deletion
purgelocal(local)
insert = extract(pull_op, insert_conflicts)
for local in insert:
session.flush()
next_id = max(max_remote(class_, pull_message),
max_local(class_, session)) + 1
update_local_id(local.row_id, next_id, class_, session)
local.row_id = next_id
if can_perform:
# pull_op.perform(pull_message, session)
logger.info(f"calling pull_op.perform_aync[{pull_op.tracked_model}], {i}/{len(pull_ops)} t+{time.time() - st}")
await pull_op.perform_async(pull_message, session, websocket=websocket)
session.flush()
# logger.warn(f"waiting... ")
# time.sleep(0.2)
# websocket.recv()
ft = time.time()
logger.info(f"finished processing pull ops: {ft-st} seconds")
# IV) fourth phase: insert versions from the pull_message
# TODO: purge old versions locally
# should normally be harmless, but this has to be done carefully
pull_version = None
for pull_version in pull_message.versions:
session.add(pull_version)
session.flush()
latest_version=get_latest_version_id(session=session)
logger.info(f"latest version after all {latest_version}/{pull_version}")
session.commit()
def handle_push(data: Dict[str, Any],
session: Optional[Session] = None) -> Dict[str, int]:
"""
Handle the push request and return a dictionary object to be sent
back to the node.
If the push is rejected, this procedure will raise a
dbsync.server.handlers.PushRejected exception.
*data* must be a dictionary-like object, usually the product of
parsing a JSON string.
"""
message: PushMessage
try:
message = PushMessage(data)
except KeyError:
raise PushRejected("request object isn't a valid PushMessage", data)
latest_version_id = core.get_latest_version_id(session=session)
if latest_version_id != message.latest_version_id:
exc = "version identifier isn't the latest one; "\
"given: %s" % message.latest_version_id
if latest_version_id is None:
raise PushRejected(exc)
if message.latest_version_id is None:
raise PullSuggested(exc)
if message.latest_version_id < latest_version_id:
raise PullSuggested(exc)
raise PushRejected(exc)
if not message.operations:
return {}
# raise PushRejected("message doesn't contain operations")
if not message.islegit(session):
raise PushRejected("message isn't properly signed")
for listener in before_push:
listener(session, message)
# I) detect unique constraint conflicts and resolve them if possible
unique_conflicts = find_unique_conflicts(message, session)
conflicting_objects = set()
for uc in unique_conflicts:
obj = uc['object']
conflicting_objects.add(obj)
for key, value in zip(uc['columns'], uc['new_values']):
setattr(obj, key, value)
for obj in conflicting_objects:
make_transient(obj) # remove from session
for model in set(type(obj) for obj in conflicting_objects):
pk_name = get_pk(model)
pks = [
getattr(obj, pk_name) for obj in conflicting_objects
if type(obj) is model
]
session.query(model).filter(getattr(model, pk_name).in_(pks)).\
delete(synchronize_session=False) # remove from the database
session.add_all(conflicting_objects) # reinsert
session.flush()
# II) perform the operations
operations = [o for o in message.operations if o.tracked_model is not None]
try:
for op in operations:
op.perform(message, session, message.node_id)
except OperationError as e:
logger.exception("Couldn't perform operation in push from node %s.",
message.node_id)
raise PushRejected("at least one operation couldn't be performed",
*e.args)
# III) insert a new version
version = Version(created=datetime.datetime.now(), node_id=message.node_id)
session.add(version)
# IV) insert the operations, discarding the 'order' column
for op in sorted(operations, key=attr('order')):
new_op = Operation()
for k in [k for k in properties_dict(op) if k != 'order']:
setattr(new_op, k, getattr(op, k))
session.add(new_op)
new_op.version = version
session.flush()
for listener in after_push:
listener(session, message)
# return the new version id back to the node
return {'new_version_id': version.version_id}
def handle_push(data, session=None):
"""
Handle the push request and return a dictionary object to be sent
back to the node.
If the push is rejected, this procedure will raise a
dbsync.server.handlers.PushRejected exception.
*data* must be a dictionary-like object, usually the product of
parsing a JSON string.
"""
message = None
try:
message = PushMessage(data)
except KeyError:
raise PushRejected("request object isn't a valid PushMessage", data)
latest_version_id = core.get_latest_version_id(session=session)
if latest_version_id != message.latest_version_id:
exc = "version identifier isn't the latest one; "\
"given: %s" % message.latest_version_id
if latest_version_id is None:
raise PushRejected(exc)
if message.latest_version_id is None:
raise PullSuggested(exc)
if message.latest_version_id < latest_version_id:
raise PullSuggested(exc)
raise PushRejected(exc)
if not message.operations:
raise PushRejected("message doesn't contain operations")
if not message.islegit(session):
raise PushRejected("message isn't properly signed")
for listener in before_push:
listener(session, message)
# I) detect unique constraint conflicts and resolve them if possible
unique_conflicts = find_unique_conflicts(message, session)
conflicting_objects = set()
for uc in unique_conflicts:
obj = uc['object']
conflicting_objects.add(obj)
for key, value in izip(uc['columns'], uc['new_values']):
setattr(obj, key, value)
for obj in conflicting_objects:
make_transient(obj) # remove from session
for model in set(type(obj) for obj in conflicting_objects):
pk_name = get_pk(model)
pks = [getattr(obj, pk_name)
for obj in conflicting_objects
if type(obj) is model]
session.query(model).filter(getattr(model, pk_name).in_(pks)).\
delete(synchronize_session=False) # remove from the database
session.add_all(conflicting_objects) # reinsert
session.flush()
# II) perform the operations
operations = filter(lambda o: o.tracked_model is not None, message.operations)
try:
for op in operations:
op.perform(message, session, message.node_id)
except OperationError as e:
logger.exception(u"Couldn't perform operation in push from node %s.",
message.node_id)
raise PushRejected("at least one operation couldn't be performed",
*e.args)
# III) insert a new version
version = Version(created=datetime.datetime.now(), node_id=message.node_id)
session.add(version)
# IV) insert the operations, discarding the 'order' column
for op in sorted(operations, key=attr('order')):
new_op = Operation()
for k in ifilter(lambda k: k != 'order', properties_dict(op)):
setattr(new_op, k, getattr(op, k))
session.add(new_op)
new_op.version = version
session.flush()
for listener in after_push:
listener(session, message)
# return the new version id back to the node
return {'new_version_id': version.version_id}
def trim(session=None):
"Trims the internal synchronization tables, to free space."
last_id = core.get_latest_version_id(session=session)
session.query(Operation).filter(Operation.version_id != None).delete()
session.query(Version).filter(Version.version_id != last_id).delete()
