Esempio n. 1
0
def test_message_does_not_contaminate_database():
    addstuff()
    session = Session()
    message = PullMessage()
    version = session.query(models.Version).first()
    message.add_version(version)
    # test that the are no unversioned operations
    assert not session.query(models.Operation).\
        filter(models.Operation.version_id == None).all()
Esempio n. 2
0
async def handle_pull(connection: Connection):
    """
    Handle the pull request and return a dictionary object to be sent
    back to the node.

    *data* must be a dictionary-like object, usually one obtained from
    decoding a JSON dictionary in the POST body.
    """
    swell = False,
    include_extensions = True
    data_str = await connection.socket.recv()
    data = json.loads(data_str)
    try:
        request_message = PullRequestMessage(data)
    except KeyError:
        raise PullRejected("request object isn't a valid PullRequestMessage",
                           data)

    message = PullMessage()
    st = time.time()
    message.fill_for(request_message,
                     swell=swell,
                     include_extensions=include_extensions,
                     connection=connection)
    logger.info(
        f"composed message with fill_for() in {time.time()-st} seconds")

    # sends the whole bunch to the client,
    # there it is received by client's run_pull and handled by pull.py/merge
    st = time.time()
    await connection.socket.send(
        json.dumps(message.to_json(), indent=4, cls=SyncdbJSONEncoder))
    logger.info(f"sent msg to client in {time.time() - st} seconds")
    # fetch messages from client
    logger.debug(f"server listening for messages after sending object")
    async for msg_ in connection.socket:
        logger.debug(f"server getting for msg: {msg_}")
        msg = json.loads(msg_)
        # logger.debug(f"msg: {msg}")
        if msg['type'] == "request_field_payload":
            # sends payload data to client here
            logger.info(f"obj from client:{msg}")
            await send_field_payload(connection, msg)
        elif msg['type'] == 'result':
            new_version_id = msg['new_version_id']
            if new_version_id is None:
                break
        else:
            logger.debug(f"response from server:{msg}")
Esempio n. 3
0
def test_create_message():
    addstuff()
    session = Session()
    message = PullMessage()
    version = session.query(models.Version).first()
    message.add_version(version)
    assert message.to_json() == PullMessage(message.to_json()).to_json()
Esempio n. 4
0
def handle_pull(data, swell=False, include_extensions=True):
    """
    Handle the pull request and return a dictionary object to be sent
    back to the node.

    *data* must be a dictionary-like object, usually one obtained from
    decoding a JSON dictionary in the POST body.
    """
    try:
        request_message = PullRequestMessage(data)
    except KeyError:
        raise PullRejected("request object isn't a valid PullRequestMessage",
                           data)

    message = PullMessage()
    message.fill_for(request_message,
                     swell=swell,
                     include_extensions=include_extensions)
    return message.to_json()
Esempio n. 5
0
def handle_pull(data, swell=False, include_extensions=True):
    """
    Handle the pull request and return a dictionary object to be sent
    back to the node.

    *data* must be a dictionary-like object, usually one obtained from
    decoding a JSON dictionary in the POST body.
    """
    try:
        request_message = PullRequestMessage(data)
    except KeyError:
        raise PullRejected("request object isn't a valid PullRequestMessage", data)

    message = PullMessage()
    message.fill_for(
        request_message,
        swell=swell,
        include_extensions=include_extensions)
    return message.to_json()
Esempio n. 6
0
def test_encode_message():
    addstuff()
    session = Session()
    message = PullMessage()
    version = session.query(models.Version).first()
    message.add_version(version)
    assert message.to_json() == json.loads(json.dumps(message.to_json()))
Esempio n. 7
0
def test_encode_message():
    addstuff()
    session = Session()
    message = PullMessage()
    version = session.query(models.Version).first()
    message.add_version(version)
    msg_json = json.dumps(message.to_json(), cls=SyncdbJSONEncoder)
    assert message.to_json() == json.loads(msg_json)
Esempio n. 8
0
def test_message_query():
    addstuff()
    session = Session()
    message = PullMessage()
    version = session.query(models.Version).first()
    message.add_version(version)
    # test equal representation, because the test models are well printed
    for b in session.query(B):
        assert repr(b) == repr(message.query(B).filter(
                attr('id') == b.id).all()[0])
    for op in session.query(models.Operation):
        assert repr(op) == repr(message.query(models.Operation).filter(
                attr('order') == op.order).all()[0])
    try:
        message.query(1)
        raise Exception("Message query did not fail")
    except TypeError:
        pass
Esempio n. 9
0
def pull(pull_url,
         extra_data=None,
         encode=None,
         decode=None,
         headers=None,
         monitor=None,
         timeout=None,
         include_extensions=True):
    """
    Attempts a pull from the server. Returns the response body.

    Additional data can be passed to the request by giving
    *extra_data*, a dictionary of values.

    If not interrupted, the pull will perform a local merge. If the
    response from the server isn't appropriate, it will raise a
    dbysnc.client.pull.BadResponseError.

    By default, the *encode* function is ``json.dumps``, the *decode*
    function is ``json.loads``, and the *headers* are appropriate HTTP
    headers for JSON.

    *monitor* should be a routine that receives a dictionary with
    information of the state of the request and merge procedure.

    *include_extensions* dictates whether the extension functions will
    be called during the merge or not. Default is ``True``.
    """
    assert isinstance(pull_url, basestring), "pull url must be a string"
    assert bool(pull_url), "pull url can't be empty"
    if extra_data is not None:
        assert isinstance(extra_data, dict), "extra data must be a dictionary"
    request_message = PullRequestMessage()
    for op in compress():
        request_message.add_operation(op)
    data = request_message.to_json()
    data.update({'extra_data': extra_data or {}})

    code, reason, response = post_request(pull_url, data, encode, decode,
                                          headers, timeout, monitor)
    if (code // 100 != 2):
        if monitor:
            monitor({'status': "error", 'reason': reason.lower()})
        raise BadResponseError(code, reason, response)
    if response is None:
        if monitor:
            monitor({'status': "error", 'reason': "invalid response format"})
        raise BadResponseError(code, reason, response)
    message = None
    try:
        message = PullMessage(response)
    except KeyError:
        if monitor:
            monitor({'status': "error", 'reason': "invalid message format"})
        raise BadResponseError("response object isn't a valid PullMessage",
                               response)

    if monitor:
        monitor({'status': "merging", 'operations': len(message.operations)})
    merge(message, include_extensions=include_extensions)
    if monitor:
        monitor({'status': "done"})
    # return the response for the programmer to do what she wants
    # afterwards
    return response
Esempio n. 10
0
    async def run_pull(
            self,
            session: Optional[sqlalchemy.orm.session.Session] = None,
            extra_data: Dict[str, Any] = None,
            monitor: Optional[Callable[[Dict[str, Any]], None]] = None):
        include_extensions = False
        if extra_data is None:
            extra_data = {}

        logger.info(f"run_pull begin")
        # new_version_id: Optional[int]
        # message = self.create_push_message()
        if not session:
            session = self.Session()

        if extra_data is not None:
            assert isinstance(extra_data,
                              dict), "extra data must be a dictionary"
        request_message = PullRequestMessage()
        for op in compress():
            request_message.add_operation(op)
        data = request_message.to_json()
        data.update({'extra_data': extra_data or {}})
        msg = json.dumps(data, cls=SyncdbJSONEncoder)
        st = time.time()
        logger.info("requesting PullMessage")
        await self.websocket.send(msg)
        logger.info(f"sent msg {time.time() - st}")

        # XXX: must be possible to fetch in a loop because this response can be fairly huge
        response_str = await self.websocket.recv()
        logger.info(
            f"@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@received : {len(response_str)} in {time.time()-st} seconds"
        )
        response = json.loads(response_str)
        message = None
        try:
            message = PullMessage(response)
            # logger.info(f"got PullMessage: {message} from response: {response_str}")
        except KeyError:
            if monitor:
                monitor({
                    'status': "error",
                    'reason': "invalid message format"
                })
            raise BadResponseError("response object isn't a valid PullMessage",
                                   response)

        logger.info(
            f"pull message contains {len(message.operations)} operations")

        if monitor:
            monitor({
                'status': "merging",
                'operations': len(message.operations)
            })

        logger.info("merging PullMessage...")
        await merge(message,
                    include_extensions=include_extensions,
                    websocket=self.websocket)  #TODO: request_payload etc.
        if monitor:
            monitor({'status': "done"})
        # return the response for the programmer to do what she wants
        # afterwards
        return response