class Store(object): KEEP_ALIVE_TIMEOUT = 60 # Seconds OPSLOG_SIZE = 1000000 # 1 MB def __init__(self): self.client = None self.db = None self.subscriptions = {} def connect(self, uri, db=None, w=1, j=True, **options): io_loop = options.get('io_loop', None) self.client = MotorClient(uri, w=w, j=j, **options).open_sync() self.client_sync = self.client.sync_client() db = db or uri_parser.parse_uri(uri)['database'] if not db: raise ConfigurationError('No database defined in uri') self.db = self.client[db] self.db_sync = self.client_sync[db] #PeriodicCallback(self.client.alive, Store.KEEP_ALIVE_TIMEOUT, # io_loop=io_loop).start() def opslog(self, collection): collection_opslog = '{0}.opslog'.format(collection) try: defer(self.db.create_collection, collection_opslog, capped=True, size=Store.OPSLOG_SIZE) # Prime opslog as tailable cursors die on empty collections defer(self.db[collection_opslog].insert, {}) except CollectionInvalid: pass return self.db[collection_opslog] def _monitor(self, collection, query_key, query): # TODO: Handle doc removal # TODO: Batch requests try: query = {'doc.{0}'.format(k): v for k, v in query.items()} query['_id'] = {'$gt': ObjectId.from_datetime(datetime.utcnow())} opslog = self.opslog(collection) cursor = opslog.find(query, tailable=True, await_data=True) item = tail(cursor.tail) while True: ops, err = next(item) if err: raise err print(ops) if not ops['doc'].get('_id'): _log.warn('Opslog for collection "{0}" contains a ' 'document with no _id'.format(collection)) continue if ops['op'] == 'insert': doc = ops['doc'] elif ops['op'] == 'update': doc = ops['updated'] response = json.dumps({ 'response': 'subscribe', 'query': query_key, 'collection': collection, 'result': [doc], }) for request in list(self.subscriptions[query_key]): if request.is_closed: self.subscriptions[query_key].remove(request) continue request.send(response) if not self.subscriptions[query_key]: break except Exception as e: _log.exception(e) finally: if query_key in self.subscriptions: del self.subscriptions[query_key] def subscribe(self, request, collection, query_key): # TODO: Inject security policies/adapters/transforms here query = json.loads(query_key) if query_key in self.subscriptions: self.subscriptions[query_key].add(request) else: Greenlet(self._monitor).switch(collection, query_key, query) self.subscriptions[query_key] = {request} docs = defer(self.db[collection].find(query).to_list, 1000) request.send(json.dumps({ 'response': 'subscribe', 'query': query_key, 'collection': collection, 'result': docs })) def __getattr__(self, name): return self[name] def __getitem__(self, name): return Collection(model, name)