def test_message_does_not_contaminate_database():
addstuff()
session = Session()
message = PullMessage()
version = session.query(models.Version).first()
message.add_version(version)
# test that the are no unversioned operations
assert not session.query(models.Operation).\
filter(models.Operation.version_id == None).all()
async def handle_pull(connection: Connection):
"""
Handle the pull request and return a dictionary object to be sent
back to the node.
*data* must be a dictionary-like object, usually one obtained from
decoding a JSON dictionary in the POST body.
"""
swell = False,
include_extensions = True
data_str = await connection.socket.recv()
data = json.loads(data_str)
try:
request_message = PullRequestMessage(data)
except KeyError:
raise PullRejected("request object isn't a valid PullRequestMessage",
data)
message = PullMessage()
st = time.time()
message.fill_for(request_message,
swell=swell,
include_extensions=include_extensions,
connection=connection)
logger.info(
f"composed message with fill_for() in {time.time()-st} seconds")
# sends the whole bunch to the client,
# there it is received by client's run_pull and handled by pull.py/merge
st = time.time()
await connection.socket.send(
json.dumps(message.to_json(), indent=4, cls=SyncdbJSONEncoder))
logger.info(f"sent msg to client in {time.time() - st} seconds")
# fetch messages from client
logger.debug(f"server listening for messages after sending object")
async for msg_ in connection.socket:
logger.debug(f"server getting for msg: {msg_}")
msg = json.loads(msg_)
# logger.debug(f"msg: {msg}")
if msg['type'] == "request_field_payload":
# sends payload data to client here
logger.info(f"obj from client:{msg}")
await send_field_payload(connection, msg)
elif msg['type'] == 'result':
new_version_id = msg['new_version_id']
if new_version_id is None:
break
else:
logger.debug(f"response from server:{msg}")
def test_create_message():
addstuff()
session = Session()
message = PullMessage()
version = session.query(models.Version).first()
message.add_version(version)
assert message.to_json() == PullMessage(message.to_json()).to_json()
def handle_pull(data, swell=False, include_extensions=True):
"""
Handle the pull request and return a dictionary object to be sent
back to the node.
*data* must be a dictionary-like object, usually one obtained from
decoding a JSON dictionary in the POST body.
"""
try:
request_message = PullRequestMessage(data)
except KeyError:
raise PullRejected("request object isn't a valid PullRequestMessage",
data)
message = PullMessage()
message.fill_for(request_message,
swell=swell,
include_extensions=include_extensions)
return message.to_json()
def handle_pull(data, swell=False, include_extensions=True):
"""
Handle the pull request and return a dictionary object to be sent
back to the node.
*data* must be a dictionary-like object, usually one obtained from
decoding a JSON dictionary in the POST body.
"""
try:
request_message = PullRequestMessage(data)
except KeyError:
raise PullRejected("request object isn't a valid PullRequestMessage", data)
message = PullMessage()
message.fill_for(
request_message,
swell=swell,
include_extensions=include_extensions)
return message.to_json()
def test_encode_message():
addstuff()
session = Session()
message = PullMessage()
version = session.query(models.Version).first()
message.add_version(version)
assert message.to_json() == json.loads(json.dumps(message.to_json()))
def test_encode_message():
addstuff()
session = Session()
message = PullMessage()
version = session.query(models.Version).first()
message.add_version(version)
msg_json = json.dumps(message.to_json(), cls=SyncdbJSONEncoder)
assert message.to_json() == json.loads(msg_json)
def test_message_query():
addstuff()
session = Session()
message = PullMessage()
version = session.query(models.Version).first()
message.add_version(version)
# test equal representation, because the test models are well printed
for b in session.query(B):
assert repr(b) == repr(message.query(B).filter(
attr('id') == b.id).all()[0])
for op in session.query(models.Operation):
assert repr(op) == repr(message.query(models.Operation).filter(
attr('order') == op.order).all()[0])
try:
message.query(1)
raise Exception("Message query did not fail")
except TypeError:
pass
def pull(pull_url,
extra_data=None,
encode=None,
decode=None,
headers=None,
monitor=None,
timeout=None,
include_extensions=True):
"""
Attempts a pull from the server. Returns the response body.
Additional data can be passed to the request by giving
*extra_data*, a dictionary of values.
If not interrupted, the pull will perform a local merge. If the
response from the server isn't appropriate, it will raise a
dbysnc.client.pull.BadResponseError.
By default, the *encode* function is ``json.dumps``, the *decode*
function is ``json.loads``, and the *headers* are appropriate HTTP
headers for JSON.
*monitor* should be a routine that receives a dictionary with
information of the state of the request and merge procedure.
*include_extensions* dictates whether the extension functions will
be called during the merge or not. Default is ``True``.
"""
assert isinstance(pull_url, basestring), "pull url must be a string"
assert bool(pull_url), "pull url can't be empty"
if extra_data is not None:
assert isinstance(extra_data, dict), "extra data must be a dictionary"
request_message = PullRequestMessage()
for op in compress():
request_message.add_operation(op)
data = request_message.to_json()
data.update({'extra_data': extra_data or {}})
code, reason, response = post_request(pull_url, data, encode, decode,
headers, timeout, monitor)
if (code // 100 != 2):
if monitor:
monitor({'status': "error", 'reason': reason.lower()})
raise BadResponseError(code, reason, response)
if response is None:
if monitor:
monitor({'status': "error", 'reason': "invalid response format"})
raise BadResponseError(code, reason, response)
message = None
try:
message = PullMessage(response)
except KeyError:
if monitor:
monitor({'status': "error", 'reason': "invalid message format"})
raise BadResponseError("response object isn't a valid PullMessage",
response)
if monitor:
monitor({'status': "merging", 'operations': len(message.operations)})
merge(message, include_extensions=include_extensions)
if monitor:
monitor({'status': "done"})
# return the response for the programmer to do what she wants
# afterwards
return response
async def run_pull(
self,
session: Optional[sqlalchemy.orm.session.Session] = None,
extra_data: Dict[str, Any] = None,
monitor: Optional[Callable[[Dict[str, Any]], None]] = None):
include_extensions = False
if extra_data is None:
extra_data = {}
logger.info(f"run_pull begin")
# new_version_id: Optional[int]
# message = self.create_push_message()
if not session:
session = self.Session()
if extra_data is not None:
assert isinstance(extra_data,
dict), "extra data must be a dictionary"
request_message = PullRequestMessage()
for op in compress():
request_message.add_operation(op)
data = request_message.to_json()
data.update({'extra_data': extra_data or {}})
msg = json.dumps(data, cls=SyncdbJSONEncoder)
st = time.time()
logger.info("requesting PullMessage")
await self.websocket.send(msg)
logger.info(f"sent msg {time.time() - st}")
# XXX: must be possible to fetch in a loop because this response can be fairly huge
response_str = await self.websocket.recv()
logger.info(
f"@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@received : {len(response_str)} in {time.time()-st} seconds"
)
response = json.loads(response_str)
message = None
try:
message = PullMessage(response)
# logger.info(f"got PullMessage: {message} from response: {response_str}")
except KeyError:
if monitor:
monitor({
'status': "error",
'reason': "invalid message format"
})
raise BadResponseError("response object isn't a valid PullMessage",
response)
logger.info(
f"pull message contains {len(message.operations)} operations")
if monitor:
monitor({
'status': "merging",
'operations': len(message.operations)
})
logger.info("merging PullMessage...")
await merge(message,
include_extensions=include_extensions,
websocket=self.websocket) #TODO: request_payload etc.
if monitor:
monitor({'status': "done"})
# return the response for the programmer to do what she wants
# afterwards
return response