def recv(self):
"""Receive a SockJS wrapped msg."""
raw = self.websocket.recv()
if not raw.startswith('a'):
raise ValueError('Invalid response: %r' % raw)
wrapped = ejson.loads(raw[1:])
return [ejson.loads(msg) for msg in wrapped]
async def _handle_messages(
self, websocket: websockets.protocol.WebSocketCommonProtocol,
_path: str):
async def send(data: object) -> None:
await websocket.send(ejson.dumps(data))
async def fail():
await send({
"msg": "failed",
"version": "1",
})
data = ejson.loads(await websocket.recv())
if data["msg"] != "connect":
return await fail()
await websocket.send(
ejson.dumps({
"msg": "connected",
"session": str(uuid.uuid4())
}))
async for message in websocket:
data = ejson.loads(message)
if data["msg"] == "method":
assert data["method"] in self._method_handlers
await send({
"id":
data["id"],
"msg":
"result",
"result":
self._method_handlers[data["method"]](*data["params"]),
})
continue
await fail()
def ddp_frames_from_message(self, message):
"""Yield DDP messages from a raw WebSocket message."""
# parse message set
try:
msgs = ejson.loads(message)
except ValueError:
self.reply("error", error=400, reason="Data is not valid EJSON")
raise StopIteration
if not isinstance(msgs, list):
self.reply("error", error=400, reason="Invalid EJSON messages")
raise StopIteration
# process individual messages
while msgs:
# pop raw message from the list
raw = msgs.pop(0)
# parse message payload
try:
data = ejson.loads(raw)
except (TypeError, ValueError):
data = None
if not isinstance(data, dict):
self.reply("error", error=400, reason="Invalid SockJS DDP payload", offendingMessage=raw)
yield data
if msgs:
# yield to other greenlets before processing next msg
gevent.sleep()
def recv(self):
"""Receive a SockJS wrapped msg."""
raw = self.websocket.recv()
if not raw.startswith('a'):
raise ValueError('Invalid response: %r' % raw)
wrapped = ejson.loads(raw[1:])
return [ejson.loads(msg) for msg in wrapped]
def ddp_frames_from_message(self, message):
"""Yield DDP messages from a raw WebSocket message."""
# parse message set
try:
msgs = ejson.loads(message)
except ValueError:
self.reply(
'error', error=400, reason='Data is not valid EJSON',
)
raise StopIteration
if not isinstance(msgs, list):
self.reply(
'error', error=400, reason='Invalid EJSON messages',
)
raise StopIteration
# process individual messages
while msgs:
# pop raw message from the list
raw = msgs.pop(0)
# parse message payload
try:
data = ejson.loads(raw)
except (TypeError, ValueError):
data = None
if not isinstance(data, dict):
self.reply(
'error', error=400,
reason='Invalid SockJS DDP payload',
offendingMessage=raw,
)
yield data
if msgs:
# yield to other greenlets before processing next msg
gevent.sleep()
def test_binary(self):
if six.PY3:
s = ejson.dumps([six.binary_type('foo', 'ascii'), 'foo'])
self.assertEqual('[{"$binary": "Zm9v"}, "foo"]', s)
self.assertEqual([six.binary_type('foo', 'ascii'), 'foo'], ejson.loads(s))
else:
s = ejson.dumps(['foo', six.text_type('foo')])
self.assertEqual('["foo", "foo"]', s)
self.assertEqual(['foo', 'foo'], ejson.loads('[{"$binary": "Zm9v"}, "foo"]'))
async def _handle_messages(self, websocket: WebSocketServerProtocol,
_path: str):
async def send(data: object) -> None:
await websocket.send(ejson.dumps(data))
self._subscription_task = asyncio.create_task(
self._send_subscription_messages(send=send, ))
async def fail():
await send({
"msg": "failed",
"version": "1",
})
data = ejson.loads(await websocket.recv())
if data["msg"] != "connect":
return await fail()
await websocket.send(
ejson.dumps({
"msg": "connected",
"session": str(uuid.uuid4())
}))
async for message in websocket:
data = ejson.loads(message)
if data["msg"] == "method":
assert data["method"] in self._method_handlers
await send({
"id":
data["id"],
"msg":
"result",
"result":
self._method_handlers[data["method"]](*data["params"]),
})
continue
if data["msg"] == "sub":
self._subscriptions[data["name"]] = data["id"]
await send({
"msg": "ready",
"subs": [data["id"]],
})
continue
if data["msg"] == "unsub":
topic = [
topic for topic, id in self._subscriptions.items()
if id == data["id"]
][0]
del self._subscriptions[topic]
# Nothing to respond with in this case.
continue
await fail()
def poll(self, conn):
"""Poll DB socket and process async tasks."""
while 1:
state = conn.poll()
if state == psycopg2.extensions.POLL_OK:
while conn.notifies:
notify = conn.notifies.pop()
self.logger.info(
"Got NOTIFY (pid=%d, payload=%r)",
notify.pid, notify.payload,
)
# read the header and check seq/fin.
hdr, chunk = notify.payload.split('|', 1)
# print('RECEIVE: %s' % hdr)
header = ejson.loads(hdr)
uuid = header['uuid']
size, chunks = self.chunks.setdefault(uuid, [0, {}])
if header['fin']:
size = self.chunks[uuid][0] = header['seq']
# stash the chunk
chunks[header['seq']] = chunk
if len(chunks) != size:
# haven't got all the chunks yet
continue # process next NOTIFY in loop
# got the last chunk -> process it.
data = ''.join(
chunk for _, chunk in sorted(chunks.items())
)
del self.chunks[uuid] # don't forget to cleanup!
data = ejson.loads(data)
sender = data.pop('_sender', None)
tx_id = data.pop('_tx_id', None)
for connection_id in data.pop('_connection_ids'):
try:
websocket = self.connections[connection_id]
except KeyError:
continue # connection not in this process
if connection_id == sender:
websocket.send(data, tx_id=tx_id)
else:
websocket.send(data)
break
elif state == psycopg2.extensions.POLL_WRITE:
gevent.select.select([], [conn.fileno()], [])
elif state == psycopg2.extensions.POLL_READ:
gevent.select.select([conn.fileno()], [], [])
else:
self.logger.warn('POLL_ERR: %s', state)
def poll(self, conn):
"""Poll DB socket and process async tasks."""
while 1:
state = conn.poll()
if state == psycopg2.extensions.POLL_OK:
while conn.notifies:
notify = conn.notifies.pop()
self.logger.info(
"Got NOTIFY (pid=%d, payload=%r)",
notify.pid,
notify.payload,
)
# read the header and check seq/fin.
hdr, chunk = notify.payload.split('|', 1)
# print('RECEIVE: %s' % hdr)
header = ejson.loads(hdr)
uuid = header['uuid']
size, chunks = self.chunks.setdefault(uuid, [0, {}])
if header['fin']:
size = self.chunks[uuid][0] = header['seq']
# stash the chunk
chunks[header['seq']] = chunk
if len(chunks) != size:
# haven't got all the chunks yet
continue # process next NOTIFY in loop
# got the last chunk -> process it.
data = ''.join(chunk
for _, chunk in sorted(chunks.items()))
del self.chunks[uuid] # don't forget to cleanup!
data = ejson.loads(data)
sender = data.pop('_sender', None)
tx_id = data.pop('_tx_id', None)
for connection_id in data.pop('_connection_ids'):
try:
websocket = self.connections[connection_id]
except KeyError:
continue # connection not in this process
if connection_id == sender:
websocket.send(data, tx_id=tx_id)
else:
websocket.send(data)
break
elif state == psycopg2.extensions.POLL_WRITE:
gevent.select.select([], [conn.fileno()], [])
elif state == psycopg2.extensions.POLL_READ:
gevent.select.select([conn.fileno()], [], [])
else:
self.logger.warn('POLL_ERR: %s', state)
def test_decimal():
# Given that I have a decimal object
obj = decimal.Decimal("0.14285714285714285")
# When I try to serialize it
serialized = ejson.dumps(obj)
# Then I see the proper string description of the object
serialized.should.contain('"__class__": "decimal.Decimal"')
serialized.should.contain('"__value__": "0.14285714285714285"')
# When I try to deserialize, I see that it also works
ejson.loads(serialized).should.equal(obj)
def test_time():
# Given that I have a time object to serialize
obj = datetime.time(1, 12, 50, 123)
# When I serialize it
serialized = ejson.dumps(obj)
# Then I see that it was correctly serialized
serialized.should.contain('"__class__": "datetime.time"')
serialized.should.contain('"__value__": "01:12:50.000123"')
# When I try to deserialize, I see that it also works
ejson.loads(serialized).should.equal(obj)
def test_time_with_timezone():
# Given that I have a time object to serialize
obj = datetime.time(1, 12, 50, 123,
ejson.serializers.TZInfoHelper(-120, "FNT"))
# When I serialize it
serialized = ejson.dumps(obj)
# Then I see that it was correctly serialized
serialized.should.contain('"__class__": "datetime.time"')
serialized.should.contain('"__value__": "01:12:50.000123-02:00"')
# When I try to deserialize, I see that it also works
ejson.loads(serialized).should.equal(obj)
def test_date():
# Given that I have an instance of the date object that must be serialized
obj = datetime.date(2013, 0o3, 30)
# When I serialize it
serialized = ejson.dumps(obj)
# Then I see the proper representation of the instance that will allow us
# to deserialize it later
serialized.should.contain('"__class__": "datetime.date"')
serialized.should.contain('"__value__": "2013-03-30"')
# When I try to load this info again, Then I see that it also works
ejson.loads(serialized).should.equal(obj)
def on_message(self, message):
"""Process a message received from remote."""
if self.ws.closed:
return None
try:
self.logger.debug('< %s %r', self, message)
# parse message set
try:
msgs = ejson.loads(message)
except ValueError as err:
self.error(400, 'Data is not valid EJSON')
return
if not isinstance(msgs, list):
self.error(400, 'Invalid EJSON messages')
return
# process individual messages
while msgs:
# parse message payload
raw = msgs.pop(0)
try:
data = ejson.loads(raw)
except (TypeError, ValueError) as err:
self.error(400, 'Data is not valid EJSON')
continue
if not isinstance(data, dict):
self.error(400, 'Invalid EJSON message payload', raw)
continue
try:
msg = data.pop('msg')
except KeyError:
self.error(400, 'Bad request', offendingMessage=data)
continue
# dispatch message
try:
self.dispatch(msg, data)
except MeteorError as err:
self.error(err)
except Exception as err:
traceback.print_exc()
self.error(err)
# emit request_finished signal to close DB connections
signals.request_finished.send(sender=self.__class__)
if msgs:
# yield to other greenlets before processing next msg
gevent.sleep()
except geventwebsocket.WebSocketError as err:
self.ws.close()
def on_message(self, message):
"""Process a message received from remote."""
if self.ws.closed:
return None
try:
self.logger.debug('< %s %r', self, message)
# parse message set
try:
msgs = ejson.loads(message)
except ValueError as err:
self.error(400, 'Data is not valid EJSON')
return
if not isinstance(msgs, list):
self.error(400, 'Invalid EJSON messages')
return
# process individual messages
while msgs:
# parse message payload
raw = msgs.pop(0)
try:
data = ejson.loads(raw)
except (TypeError, ValueError) as err:
self.error(400, 'Data is not valid EJSON')
continue
if not isinstance(data, dict):
self.error(400, 'Invalid EJSON message payload', raw)
continue
try:
msg = data.pop('msg')
except KeyError:
self.error(400, 'Bad request', offendingMessage=data)
continue
# dispatch message
try:
self.dispatch(msg, data)
except MeteorError as err:
self.error(err)
except Exception as err:
traceback.print_exc()
self.error(err)
# emit request_finished signal to close DB connections
signals.request_finished.send(sender=self.__class__)
if msgs:
# yield to other greenlets before processing next msg
gevent.sleep()
except geventwebsocket.WebSocketError as err:
self.ws.close()
def on_message(self, msg):
msg = ejson.loads(msg)
if msg.get('msg') == 'added':
self.on_added(msg['collection'], msg['id'], msg.get('fields', {}))
if msg.get('msg') == 'changed':
self.on_changed(msg['collection'], msg['id'],
msg.get('fields', {}), msg.get('cleared', []))
if msg.get('msg') == 'removed':
self.on_removed(msg['collection'], msg['id'])
if msg.get('msg') == 'ready':
self.on_ready(msg['subs'])
if msg.get('msg') == 'addedBefore':
raise NotImplementedError
if msg.get('msg') == 'movedBefore':
raise NotImplementedError
if msg.get('msg') == 'result':
self.on_result(msg['id'], msg.get('error'), msg.get('result', {}))
if msg.get('msg') == 'updated':
self.on_updated(msg['methods'])
return msg
def consumer(self): # all data gets must go inside a try
while True:
msg = yield self.queue.get()
if msg == 'stop':
return
data = ejson.loads(msg)
message = data['msg']
if message == 'method':
if data['method'] not in methods:
self.send_nomethod(data['id'], 'method does not exist')
else:
method = getattr(self, data['method'])
result = yield method(**data['params'])
self.send_result(data['id'], result)
elif message == 'sub':
if data['name'] not in subs:
self.send_nosub(data['id'], 'sub does not exist')
else:
query = getattr(self, data['name'])(**data['params'])
tornado.ioloop.IOLoop.current().spawn_callback(
self.feed, data['id'], query)
#prefixed = 'sub_' + data['name']
#try:
# query = getattr(self, prefixed)(**data['params'])
# tornado.ioloop.IOLoop.current().spawn_callback(self.feed, data['id'], query)
#except AttributeError:
# self.send_nosub(data['id'], 'sub does not exist')
elif message == 'unsub':
id = data['id']
feed = self.registered_feeds[id]
feed.close()
del self.registered_feeds[id]
self.queue.task_done()
async def connect(self):
'''This coroutine establishes a connection to a server.
It blocks until the connection is established or an exception is raised.
Raises ddp_asyncio.ConnectionError if the server reports a failure (usually caused by incomplatible versions of the DDP protocol.)
'''
self._websocket = await websockets.connect(self.url)
msg = {'msg': 'connect', 'version': '1', 'support': ['1']}
await self._websocket.send(ejson.dumps(msg))
while self._websocket.open:
msg = await self._websocket.recv()
msg = ejson.loads(msg)
_type = msg.get('msg')
if _type == 'failed':
raise ConnectionError(
'The server is not compatible with version 1 of the DDP protocol.'
)
elif _type == 'connected':
# Ensure all Collections are in their default states
for col in self._cols.values():
col._data = {}
self.is_connected = True
self._disconnection_event.clear()
self._event_loop.create_task(self.__handler__())
return
def process(event_name, data):
"""Iterates over the event handler registry and execute each found
handler.
It takes the event name and its its `data`, passing the return of
`ejson.loads(data)` to the found handlers.
"""
deserialized = loads(data)
event_cls = find_event(event_name)
event = event_cls(event_name, deserialized)
try:
event.clean()
except ValidationError as exc:
if os.environ.get('EVENTLIB_RAISE_ERRORS'):
raise
else:
logger.warning(
"The event system just got an exception while cleaning "
"data for the event '{}'\ndata: {}\nexc: {}".format(
event_name, data, str(exc)))
return
for handler in find_handlers(event_name):
try:
handler(deserialized)
except Exception as exc:
logger.warning(
(u'One of the handlers for the event "{}" has failed with the '
u'following exception: {}').format(event_name, str(exc)))
if getsetting('DEBUG'):
raise exc
event._broadcast()
def test_date(self):
s1 = ejson.dumps(datetime.date(2015, 1, 25))
self.assertEqual('{"$date": 1422144000000}', s1)
self.assertEqual(datetime.datetime(2015, 1, 25, tzinfo=ejson.timezone.utc), ejson.loads(s1))
s2 = ejson.dumps(datetime.datetime(2015, 1, 25, 10, 30, 1, tzinfo=ejson.timezone.utc))
self.assertEqual('{"$date": 1422181801000}', s2)
self.assertEqual(datetime.datetime(2015, 1, 25, 10, 30, 1, tzinfo=ejson.timezone.utc), ejson.loads(s2))
def poll(self):
"""Poll DB socket and process async tasks."""
while 1:
state = self.conn.poll()
if state == psycopg2.extensions.POLL_OK:
while self.conn.notifies:
notify = self.conn.notifies.pop()
name = notify.channel
self.logger.info(
"Got NOTIFY (pid=%d, name=%r, payload=%r)",
notify.pid, name, notify.payload,
)
try:
self._sub_lock.acquire()
self.logger.info(self.all_subs)
subs = self.all_subs[name]
data = ejson.loads(notify.payload)
sub_ids = data.pop('_sub_ids')
self.logger.info('Subscribers: %r', sub_ids)
self.logger.info(subs)
for id_, func in subs.items():
if id_ not in sub_ids:
continue # not for this subscription
gevent.spawn(func, id_, name, data)
finally:
self._sub_lock.release()
break
elif state == psycopg2.extensions.POLL_WRITE:
gevent.select.select([], [self.conn.fileno()], [])
elif state == psycopg2.extensions.POLL_READ:
gevent.select.select([self.conn.fileno()], [], [])
else:
self.logger.warn('POLL_ERR: %s', state)
def process(event_name, data):
"""Iterates over the event handler registry and execute each found
handler.
It takes the event name and its its `data`, passing the return of
`ejson.loads(data)` to the found handlers.
"""
deserialized = loads(data)
event_cls = find_event(event_name)
event = event_cls(event_name, deserialized)
try:
event.clean()
except ValidationError as exc:
if os.environ.get('EVENTLIB_RAISE_ERRORS'):
raise
else:
logger.warning(
"The event system just got an exception while cleaning "
"data for the event '{}'\ndata: {}\nexc: {}".format(
event_name, data, str(exc)))
return
for handler in find_handlers(event_name):
try:
handler(deserialized)
except Exception as exc:
logger.warning(
(u'One of the handlers for the event "{}" has failed with the '
u'following exception: {}').format(event_name, str(exc)))
if getsetting('DEBUG'):
raise exc
event._broadcast()
def recvloop(self):
while self.websocket.open:
while len(self.callcache) > 0:
self.call(*self.callcache.pop(0))
msg = yield from self.websocket.recv()
if not msg: continue
msg = ejson.loads(msg)
if msg.get('msg') == 'connected':
self.connected = True
self.session = msg['session']
elif msg.get('msg') == 'ping':
yield from self.websocket.send(
ejson.dumps({
'msg': 'pong',
'id': msg.get('id')
}))
elif msg.get('msg') == 'ready':
for sub in self.subs.values():
if sub.id in msg['subs']:
yield from sub.ready_cb(sub)
elif msg.get('msg') == 'added':
sub = self.subs.get(msg['collection'])
if sub:
sub.data[msg['id']] = msg['fields']
yield from sub.added_cb(sub, msg['id'], msg['fields'])
elif msg.get('msg') == 'changed':
sub = self.subs.get(msg['collection'])
if sub:
if msg.get('fields'):
sub.data[msg['id']].update(msg['fields'])
yield from sub.changed_cb(sub, msg['id'],
msg['fields'])
elif msg.get('cleared'):
for key in msg['cleared']:
del sub.data[key]
yield from sub.changed_cb(sub, msg['id'],
msg['cleared'])
elif msg.get('msg') == 'removed':
sub = self.subs.get(msg['collection'])
if sub:
del sub.data[msg['id']]
yield from sub.removed_cb(sub, msg['id'])
elif msg.get('msg') == 'result':
c = self.calls.get(msg['id'])
if c:
yield from c.onfinished(msg.get('result'),
msg.get('error'))
self.connected = False
while True:
yield from self.connect()
return
def sub_unique_objects(self, obj, params=None, pub=None, *args, **kwargs):
"""Return objects that are only visible through given subscription."""
if params is None:
params = ejson.loads(obj.params_ejson)
if pub is None:
pub = self.get_pub_by_name(obj.publication)
queries = collections.OrderedDict(
(col, qs)
for (qs, col) in (self.qs_and_collection(qs)
for qs in pub.user_queries(obj.user, *params)))
# mergebox via MVCC! For details on how this is possible, read this:
# https://devcenter.heroku.com/articles/postgresql-concurrency
to_send = collections.OrderedDict((
col,
col.objects_for_user(user=obj.user_id, qs=qs, *args, **kwargs),
) for col, qs in queries.items())
for other in Subscription.objects.filter(
connection=obj.connection_id,
collections__collection_name__in=[col.name for col in queries],
).exclude(pk=obj.pk, ).order_by('pk').distinct():
other_pub = self.get_pub_by_name(other.publication)
for qs in other_pub.user_queries(other.user, *other.params):
qs, col = self.qs_and_collection(qs)
if col not in to_send:
continue
to_send[col] = to_send[col].exclude(
pk__in=col.objects_for_user(user=other.user_id,
qs=qs,
*args,
**kwargs).values('pk'), )
for col, qs in to_send.items():
yield col, qs.distinct()
def test_escape_escape(self):
self.assertEqual(
{
"$date":
datetime.datetime(
1970, 1, 1, 0, 0, 32, tzinfo=ejson.timezone.utc)
}, ejson.loads('{"$escape": {"$date": {"$date": 32000}}}'))
def load(self):
self.clear()
if os.path.isfile(self._file):
with file(self._file, 'r') as f:
raw = f.read()
if raw:
data = ejson.loads(raw)
self.update(data)
def _handle_message(self, msg, socket):
if msg.tp == sockjs.MSG_OPEN:
socket._ddp_session = None
elif msg.tp == sockjs.MSG_MESSAGE:
try:
try:
mbody = ejson.loads(msg.data)
if type(mbody) != dict or mbody.get("msg") == None:
self.logger.debug(
"Discarding non-object DDP message {msg}"
.format(msg=msg.data),
)
socket.send(ejson.dumps({
"msg": "error",
"reason": "Bad request",
"offendingMessage": mbody,
}))
return
except ValueError:
self.logger.debug(
"Discarding message with invalid JSON {msg}"
.format(msg=msg.data),
)
socket.send(json.dumps({
"msg": "error",
"reason": "Bad request",
}))
return
if mbody["msg"] == "connect":
if socket._ddp_session:
socket.send(json.dumps({
"msg": "error",
"reason": "Already connected",
"offendingMessage": mbody,
}))
return
asyncio.async(self._handle_connect(mbody, socket),
loop=self.loop)
return
if not socket._ddp_session:
socket.send(ejson.dumps({
"msg": "error",
"reason": "Must connect first",
"offendingMessage": mbody,
}))
return
socket._ddp_session.process_message(mbody)
except Exception:
self.logger.error(
"Internal exception while processing message {msg}\n{err}"
.format(msg=msg.data, err=traceback.format_exc())
)
def send(self, data):
"""Send raw `data` to WebSocket client."""
if data[1:]:
msg = ejson.loads(data[1:])
self.logger.debug('> %s %r', self, data)
try:
self.ws.send(data)
except geventwebsocket.WebSocketError:
self.ws.close()
def test_datetime_with_microseconds():
# Given that I have an instance of the datetime object with timezone
# information, must be serialized
obj = datetime.datetime(
2013, 0o3, 30, 12, 12, 12, 123456,
ejson.serializers.TZInfoHelper(60 * 3, "America/Sao_Paulo"))
# When I serialize it
serialized = ejson.dumps(obj)
# Then I see the proper representation of the instance that will allow us
# to deserialize it later
serialized.should.contain('"__class__": "datetime.datetime"')
serialized.should.contain(
'"__value__": "2013-03-30T12:12:12.123456+03:00"')
# When I try to deserialize, Then I see it works again
ejson.loads(serialized).should.equal(obj)
def deserialize(msg_raw):
msg = ejson.loads(msg_raw)
msg_type = msg.get("msg", False)
if not msg_type:
raise Exception("No message type specified")
if msg_type not in msg_types:
raise Exception("Invalid message type")
obj = msg_types[msg_type]
return obj(*[msg.get(arg) for arg in obj._serialize_args])
def test_escape_decode(self):
self.assertEqual({
"foo": "bar",
"bat": {
"$escape": {
"baz": "bazam"
}
}
}, ejson.loads('{"bat": {"$escape": {"$escape": {"baz": "bazam"}}}, "foo": "bar"}'))
def on_message(self, message):
"""Process a message received from remote."""
if self.ws.closed:
return None
try:
self.logger.debug('< %s %r', self, message)
# parse message set
try:
msgs = ejson.loads(message)
except ValueError, err:
self.error(400, 'Data is not valid EJSON')
return
if not isinstance(msgs, list):
self.error(400, 'Invalid EJSON messages')
return
# process individual messages
while msgs:
# parse message payload
raw = msgs.pop(0)
try:
data = ejson.loads(raw)
except ValueError, err:
self.error(400, 'Data is not valid EJSON')
continue
if not isinstance(data, dict):
self.error(400, 'Invalid EJSON message payload', raw)
continue
try:
msg = data.pop('msg')
except KeyError:
self.error(
400, 'Bad request', offendingMessage=data,
)
continue
# dispatch message
try:
self.dispatch(msg, data)
except MeteorError, err:
self.error(err)
def test_deserialization():
ejson.cleanup_deserialization_registry()
@ejson.register_deserializer(Person)
def deserialize_person(data):
return Person(data['name'], data['age'])
obj = ejson.loads('{"age": 25, "name": "Lincoln"}')
person = ejson.deserialize(Person, obj)
person.should.be.a(Person)
person.name.should.be.equals('Lincoln')
person.age.should.be.equals(25)
def test_auto_deserialization():
ejson.cleanup_deserialization_registry()
@ejson.register_deserializer(Person)
def deserialize_person(data):
return Person(data['name'], data['age'])
person = ejson.loads(
'{"__class__": "tests.unit.test_ejson.Person", "__value__": {"age": 25, "name": "Lincoln"}}')
person.should.be.a(Person)
person.name.should.be.equals('Lincoln')
person.age.should.be.equals(25)
def test_deserialization():
ejson.cleanup_deserialization_registry()
@ejson.register_deserializer(Person)
def deserialize_person(data):
return Person(data['name'], data['age'])
obj = ejson.loads('{"age": 25, "name": "Lincoln"}')
person = ejson.deserialize(Person, obj)
person.should.be.a(Person)
person.name.should.be.equals('Lincoln')
person.age.should.be.equals(25)
def recvloop(self):
while self.websocket.open:
while len(self.callcache) > 0:
self.call(*self.callcache.pop(0))
msg = yield from self.websocket.recv()
if not msg: continue
msg = ejson.loads(msg)
if msg.get('msg') == 'connected':
self.connected = True
self.session = msg['session']
elif msg.get('msg') == 'ping':
yield from self.websocket.send(ejson.dumps({'msg': 'pong', 'id': msg.get('id')}))
elif msg.get('msg') == 'ready':
for sub in self.subs.values():
if sub.id in msg['subs']:
yield from sub.ready_cb(sub)
elif msg.get('msg') == 'added':
sub = self.subs.get(msg['collection'])
if sub:
sub.data[msg['id']] = msg['fields']
yield from sub.added_cb(sub, msg['id'], msg['fields'])
elif msg.get('msg') == 'changed':
sub = self.subs.get(msg['collection'])
if sub:
if msg.get('fields'):
sub.data[msg['id']].update(msg['fields'])
yield from sub.changed_cb(sub, msg['id'], msg['fields'])
elif msg.get('cleared'):
for key in msg['cleared']:
del sub.data[key]
yield from sub.changed_cb(sub, msg['id'], msg['cleared'])
elif msg.get('msg') == 'removed':
sub = self.subs.get(msg['collection'])
if sub:
del sub.data[msg['id']]
yield from sub.removed_cb(sub, msg['id'])
elif msg.get('msg') == 'result':
c = self.calls.get(msg['id'])
if c:
yield from c.onfinished(msg.get('result'), msg.get('error'))
self.connected = False
while True:
yield from self.connect()
return
async def _websocket_message_handler(self) -> None:
async for message in self:
recv = ejson.loads(message)
if recv["msg"] == "result":
self._invoke_method_handler(recv)
elif recv["msg"] == "ready":
self._subscription_ready_handler(recv)
elif recv["msg"] == "added" or recv["msg"] == "changed":
self._subscription_message_handler(recv)
else:
logger.error(f"Unhandled message from server:\n%s",
pprint.pformat(recv))
def test_auto_deserialization():
ejson.cleanup_deserialization_registry()
@ejson.register_deserializer(Person)
def deserialize_person(data):
return Person(data['name'], data['age'])
person = ejson.loads(
'{"__class__": "tests.unit.test_ejson.Person", "__value__": {"age": 25, "name": "Lincoln"}}'
)
person.should.be.a(Person)
person.name.should.be.equals('Lincoln')
person.age.should.be.equals(25)
async def __handler__(self):
'''Handles messages received from the server'''
while self._websocket.open:
try:
msg = await self._websocket.recv()
except websockets.exceptions.ConnectionClosed:
break
if not msg: continue
msg = ejson.loads(msg)
_type = msg.get('msg')
if _type == 'ping':
await self.__pong__(msg.get('id'))
elif _type == 'ready':
for _id in msg['subs']:
sub = self._subs.get(_id)
if sub: sub.__ready__()
elif _type == 'nosub':
sub = self._subs.get(msg['id'])
if sub:
sub.__error__(
msg.get('error',
'Denied by server for unspecified reason.'))
elif _type == 'added':
col = self.get_collection(msg['collection'])
col.__added__(msg['id'], msg['fields'])
elif _type == 'changed':
col = self.get_collection(msg['collection'])
col.__changed__(msg['id'], msg.get('fields', {}),
msg.get('cleared', []))
elif _type == 'removed':
col = self.get_collection(msg['collection'])
col.__removed__(msg['id'])
elif _type == 'result':
c = self._calls.get(msg['id'])
if c:
await c.__result__(msg.get('error'), msg.get('result'))
del self._calls[msg['id']]
self.is_connected = False
self._disconnection_event.set()
async def invoke_method(self, method: str, params: List[Any] = []) -> Any:
async with self._method_lock:
id = str(uuid.uuid4())
await super().send(
ejson.dumps({
"id": id,
"msg": "method",
"method": method,
"params": params,
}))
recv = ejson.loads(await super().recv())
if recv["id"] != id or recv["msg"] != "result":
raise websockets.exceptions.ProtocolError("Unexpected message")
return recv["result"]
def _assert_serialize_deserialize_equals_and_is_binary(
ejson_obj, json_obj, json_str, is_binary
):
assert json_str == ejson.stringify(ejson_obj)
assert json_str == ejson.dumps(ejson_obj)
assert json_obj == ejson.to_json_value(ejson_obj)
assert ejson_obj == ejson.parse(json_str)
assert ejson_obj == ejson.loads(json_str)
assert ejson_obj == ejson.from_json_value(json_obj)
ejson_obj2 = deepcopy(ejson_obj)
assert ejson.equals(ejson_obj, ejson_obj2)
assert ejson.equals(ejson_obj, ejson.clone(ejson_obj))
assert is_binary == ejson.is_binary(ejson_obj)
def sub_unique_objects(self, obj, params=None, pub=None, *args, **kwargs):
"""Return objects that are only visible through given subscription."""
if params is None:
params = ejson.loads(obj.params_ejson)
if pub is None:
pub = self.get_pub_by_name(obj.publication)
queries = collections.OrderedDict(
(col, qs) for (qs, col) in (
self.qs_and_collection(qs)
for qs
in pub.get_queries(*params)
)
)
# mergebox via MVCC! For details on how this is possible, read this:
# https://devcenter.heroku.com/articles/postgresql-concurrency
to_send = collections.OrderedDict(
(
col,
col.objects_for_user(
user=obj.user_id,
qs=qs,
*args, **kwargs
),
)
for col, qs
in queries.items()
)
for other in Subscription.objects.filter(
connection=obj.connection_id,
collections__collection_name__in=[col.name for col in queries],
).exclude(
pk=obj.pk,
).order_by('pk').distinct():
other_pub = self.get_pub_by_name(other.publication)
for qs in other_pub.get_queries(*other.params):
qs, col = self.qs_and_collection(qs)
if col not in to_send:
continue
to_send[col] = to_send[col].exclude(
pk__in=col.objects_for_user(
user=other.user_id,
qs=qs,
*args, **kwargs
).values('pk'),
)
for col, qs in to_send.items():
yield col, qs.distinct()
def update_subs(new_user_id):
"""Update subs to send added/removed for collections with user_rel."""
for sub in Subscription.objects.filter(connection=this.ws.connection):
params = loads(sub.params_ejson)
pub = API.get_pub_by_name(sub.publication)
# calculate the querysets prior to update
pre = collections.OrderedDict([
(col, query) for col, query
in API.sub_unique_objects(sub, params, pub)
])
# save the subscription with the updated user_id
sub.user_id = new_user_id
sub.save()
# calculate the querysets after the update
post = collections.OrderedDict([
(col, query) for col, query
in API.sub_unique_objects(sub, params, pub)
])
# first pass, send `added` for objs unique to `post`
for col_post, query in post.items():
try:
qs_pre = pre[col_post]
query = query.exclude(
pk__in=qs_pre.order_by().values('pk'),
)
except KeyError:
# collection not included pre-auth, everything is added.
pass
for obj in query:
this.ws.send(col_post.obj_change_as_msg(obj, ADDED))
# second pass, send `removed` for objs unique to `pre`
for col_pre, query in pre.items():
try:
qs_post = post[col_pre]
query = query.exclude(
pk__in=qs_post.order_by().values('pk'),
)
except KeyError:
# collection not included post-auth, everything is removed.
pass
for obj in query:
this.ws.send(col_pre.obj_change_as_msg(obj, REMOVED))
def validated_user(cls, token, purpose, minutes_valid):
"""Resolve and validate auth token, returns user object."""
try:
username, auth_hash = loads(token.decode('base64'))
except (ValueError, Error):
cls.auth_failed(token=token)
try:
user = cls.user_model.objects.get(**{
cls.user_model.USERNAME_FIELD: username,
'is_active': True,
})
user.backend = 'django.contrib.auth.backends.ModelBackend'
except cls.user_model.DoesNotExist:
cls.auth_failed(username=username, token=token)
if auth_hash not in iter_auth_hashes(user, purpose, minutes_valid):
cls.auth_failed(username=username, token=token)
return user
def validated_user(cls, token, purpose, days_valid):
"""Resolve and validate auth token, returns user object."""
try:
username, auth_hash = loads(token.decode('base64'))
except (ValueError, Error):
cls.auth_failed(token=token)
try:
user = cls.user_model.objects.get(**{
cls.user_model.USERNAME_FIELD: username,
'is_active': True,
})
user.backend = 'django.contrib.auth.backends.ModelBackend'
except cls.user_model.DoesNotExist:
cls.auth_failed(username=username, token=token)
if auth_hash not in iter_auth_hashes(user, purpose, days_valid):
cls.auth_failed(username=username, token=token)
return user
def listen_for_events():
"""Pubsub event listener
Listen for events in the pubsub bus and calls the process function
when somebody comes to play.
"""
import_event_modules()
conn = redis_connection.get_connection()
pubsub = conn.pubsub()
pubsub.subscribe("eventlib")
for message in pubsub.listen():
if message['type'] != 'message':
continue
data = loads(message["data"])
if 'name' in data:
event_name = data.pop('name')
process_external(event_name, data)
def validated_user_and_session(cls, token, purpose):
"""Resolve and validate auth token, returns user and session objects."""
try:
username, session_key, auth_hash = loads(token.decode('base64'))
except (ValueError, Error):
cls.auth_failed(token=token)
try:
user = cls.user_model.objects.get(**{
cls.user_model.USERNAME_FIELD: username,
})
user.backend = 'django.contrib.auth.backends.ModelBackend'
except cls.user_model.DoesNotExist:
cls.auth_failed(username=username, token=token)
if cls.get_auth_hash(user, purpose) != auth_hash:
cls.auth_failed(username=username, token=token)
session = SessionStore(
session_key=session_key,
)
if session.get_expiry_date() <= timezone.now():
cls.auth_failed(username=username, token=token)
return (user, session)
def read_json(path):
"""Read JSON encoded contents from specified path."""
with open(path, mode='r') as json_file:
return loads(json_file.read())
def recv(self):
"""Receive a message."""
raw = self.websocket.recv()
return ejson.loads(raw)
def test_forwarding_kwargs_to_loads():
my_float = lambda f: float(f) * 2
ejson.loads('{"num": 2.5}', parse_float=my_float).should.equal(
{'num': 5})
def test_deserialization_with_no_objects_registered():
ejson.cleanup_deserialization_registry()
obj = ejson.loads('{"age": 25, "name": "Lincoln"}')
ejson.deserialize.when.called_with(Person, obj).should.throw(
TypeError,
"There is no deserializer registered to handle instances of 'Person'")
def received_message(self, data):
"""Incomming messages"""
data = ejson.loads(str(data))
if not data.get('msg'):
return
elif data['msg'] == 'failed':
self._ddp_version_index += 1
self._retry_new_version = data.get('version', True)
self.emit('failed', data)
elif data['msg'] == 'connected':
self._session = data.get('session')
if self._is_reconnecting:
self.ddpsocket._debug_log("* RECONNECTED")
self.emit('reconnected')
self._is_reconnecting = False
else:
self.ddpsocket._debug_log("* CONNECTED")
self.emit('connected')
self._retry_new_version = False
# method result
elif data['msg'] == 'result':
# call the optional callback
callback = self._callbacks.get(data['id'])
if callback:
callback(data.get('error'), data.get('result'))
self._callbacks.pop(data['id'])
# missing subscription
elif data['msg'] == 'nosub':
callback = self._callbacks.get(data['id'])
if callback:
callback(data.get('error'), data['id'])
self._callbacks.pop(data['id'])
# document added to collection
elif data['msg'] == 'added':
self.emit('added', data['collection'],
data['id'], data.get('fields', {}))
# document changed in collection
elif data['msg'] == 'changed':
self.emit('changed', data['collection'], data['id'],
data.get('fields', {}), data.get('cleared', {}))
# document removed from collection
elif data['msg'] == 'removed':
self.emit('removed', data['collection'], data['id'])
# subcription ready
elif data['msg'] == 'ready':
for sub_id in data.get('subs', []):
callback = self._callbacks.get(sub_id)
if callback:
callback(data.get('error'), sub_id)
self._callbacks.pop(sub_id)
elif data['msg'] == 'ping':
msg = {'msg': 'pong'}
id = data.get('id')
if id is not None:
msg['id'] = id
self.ddpsocket.send(msg)
else:
pass
def test_unknown_type_decode(self):
with self.assertRaises(ejson.UnknownTypeError):
ejson.loads('{"foo": {"$type": "pow", "$value": 5}}')
def __init__(self, **kwargs):
"""
Initialisation for Django DDP server view.
`Meteor.settings` is sourced from the following (later take precedence):
1. django.conf.settings.METEOR_SETTINGS
2. os.environ['METEOR_SETTINGS']
3. MeteorView.meteor_settings (class attribute) or empty dict
4. MeteorView.as_view(meteor_settings=...)
Additionally, `Meteor.settings.public` is updated with values from
environemnt variables specified from the following sources:
1. django.conf.settings.METEOR_PUBLIC_ENVS
2. os.environ['METEOR_PUBLIC_ENVS']
3. MeteorView.meteor_public_envs (class attribute) or empty dict
4. MeteorView.as_view(meteor_public_envs=...)
"""
self.runtime_config = {}
self.meteor_settings = reduce(
dict_merge,
[
getattr(settings, 'METEOR_SETTINGS', {}),
loads(os.environ.get('METEOR_SETTINGS', '{}')),
self.meteor_settings or {},
kwargs.pop('meteor_settings', {}),
],
{},
)
self.meteor_public_envs = set()
self.meteor_public_envs.update(
getattr(settings, 'METEOR_PUBLIC_ENVS', []),
os.environ.get('METEOR_PUBLIC_ENVS', '').replace(',', ' ').split(),
self.meteor_public_envs or [],
kwargs.pop('meteor_public_envs', []),
)
public = self.meteor_settings.setdefault('public', {})
for env_name in self.meteor_public_envs:
try:
public[env_name] = os.environ[env_name]
except KeyError:
pass # environment variable not set
# super(...).__init__ assigns kwargs to instance.
super(MeteorView, self).__init__(**kwargs)
# read and process /etc/mime.types
mimetypes.init()
self.url_map = {}
# process `star_json`
self.star_json = read_json(self.json_path)
star_format = self.star_json['format']
if star_format != 'site-archive-pre1':
raise ValueError(
'Unknown Meteor star format: %r' % star_format,
)
programs = {
program['name']: program
for program in self.star_json['programs']
}
# process `bundle/programs/server/program.json` from build dir
server_json_path = os.path.join(
os.path.dirname(self.json_path),
os.path.dirname(programs['server']['path']),
'program.json',
)
server_json = read_json(server_json_path)
server_format = server_json['format']
if server_format != 'javascript-image-pre1':
raise ValueError(
'Unknown Meteor server format: %r' % server_format,
)
self.server_load_map = {}
for item in server_json['load']:
item['path_full'] = os.path.join(
os.path.dirname(server_json_path),
item['path'],
)
self.server_load_map[item['path']] = item
self.url_map[item['path']] = (
item['path_full'], 'text/javascript'
)
try:
item['source_map_full'] = os.path.join(
os.path.dirname(server_json_path),
item['sourceMap'],
)
self.url_map[item['sourceMap']] = (
item['source_map_full'], 'text/plain'
)
except KeyError:
pass
self.template_path = os.path.join(
os.path.dirname(server_json_path),
self.server_load_map[
'packages/boilerplate-generator.js'
][
'assets'
][
'boilerplate_web.browser.html'
],
)
# process `bundle/programs/web.browser/program.json` from build dir
web_browser_json_path = os.path.join(
os.path.dirname(self.json_path),
programs['web.browser']['path'],
)
web_browser_json = read_json(web_browser_json_path)
web_browser_format = web_browser_json['format']
if web_browser_format != 'web-program-pre1':
raise ValueError(
'Unknown Meteor web.browser format: %r' % (
web_browser_format,
),
)
self.client_map = {}
self.internal_map = {}
for item in web_browser_json['manifest']:
item['path_full'] = os.path.join(
os.path.dirname(web_browser_json_path),
item['path'],
)
if item['where'] == 'client':
if '?' in item['url']:
item['url'] = item['url'].split('?', 1)[0]
if item['url'].startswith('/'):
item['url'] = item['url'][1:]
self.client_map[item['url']] = item
self.url_map[item['url']] = (
item['path_full'],
mimetypes.guess_type(
item['path_full'],
)[0] or 'application/octet-stream',
)
elif item['where'] == 'internal':
self.internal_map[item['type']] = item
config = {
'css': [
{'url': item['path']}
for item in web_browser_json['manifest']
if item['type'] == 'css' and item['where'] == 'client'
],
'js': [
{'url': item['path']}
for item in web_browser_json['manifest']
if item['type'] == 'js' and item['where'] == 'client'
],
'meteorRuntimeConfig': '"%s"' % (
dumps(self.runtime_config)
),
'rootUrlPathPrefix': self.root_url_path_prefix,
'bundledJsCssPrefix': self.bundled_js_css_prefix,
'inlineScriptsAllowed': False,
'inline': None,
'head': read(
self.internal_map.get('head', {}).get('path_full', None),
default=u'',
),
'body': read(
self.internal_map.get('body', {}).get('path_full', None),
default=u'',
),
}
tmpl_raw = read(self.template_path, encoding='utf8')
compiler = pybars.Compiler()
tmpl = compiler.compile(tmpl_raw)
self.html = '<!DOCTYPE html>\n%s' % tmpl(config)
