def setup(self, environ): cfg = environ['pulsar.cfg'] self.store = create_store(cfg.data_store) pubsub = self.store.pubsub() return WsgiHandler([Router('/', get=self.home_page), WebSocket('/message', TweetsWsHandler(pubsub, self.channel))])
def create_store(cls, address, namespace=None, pool_size=2, **kw): if cls.redis_py_parser: kw['parser_class'] = redis_parser(True) if not namespace: namespace = cls.randomkey(6).lower() return create_store(address, namespace=namespace, pool_size=pool_size, **kw)
def monitor_start(self, monitor): '''When the monitor starts load all test classes into the queue''' # Create a datastore for this test suite if not self.cfg.task_backend: server = PulsarDS(bind='127.0.0.1:0', workers=0, key_value_save=[], name='%s_store' % self.name) yield server() address = 'pulsar://%s:%s' % server.cfg.addresses[0] else: address = self.cfg.task_backend store = create_store(address, pool_size=2, loop=monitor._loop) self.get_backend(store) loader = self.loader tags = self.cfg.labels exclude_tags = self.cfg.exclude_labels if self.cfg.show_leaks: show = show_leaks if self.cfg.show_leaks == 1 else hide_leaks self.cfg.set('when_exit', show) arbiter = pulsar.arbiter() arbiter.cfg.set('when_exit', show) try: tests = [] loader.runner.on_start() for tag, testcls in loader.testclasses(tags, exclude_tags): suite = loader.runner.loadTestsFromTestCase(testcls) if suite and suite._tests: tests.append((tag, testcls)) self._time_start = None if tests: self.logger.info('loading %s test classes', len(tests)) monitor.cfg.set('workers', min(self.cfg.workers, len(tests))) self._time_start = default_timer() queued = [] self._tests_done = set() self._tests_queued = None # # Bind to the task_done event self.backend.bind_event('task_done', partial(self._test_done, monitor)) for tag, testcls in tests: r = self.backend.queue_task('test', testcls=testcls, tag=tag) queued.append(r) queued = yield multi_async(queued) self.logger.debug('loaded %s test classes', len(tests)) self._tests_queued = set(queued) yield self._test_done(monitor) else: # pragma nocover raise ExitTest('Could not find any tests.') except ExitTest as e: # pragma nocover monitor.stream.writeln(str(e)) monitor._loop.stop() except Exception: # pragma nocover monitor.logger.critical('Error occurred while starting tests', exc_info=True) monitor._loop.call_soon(self._exit, 3)
def __init__(self, app, name, url): super().__init__(app, name, url) if app.green_pool: self._wait = app.green_pool.wait self.client = create_store(url).client() else: import redis self.client = redis.StrictRedis.from_url(url)
def monitor_start(self, monitor): '''When the monitor starts load all test classes into the queue''' # Create a datastore for this test suite if not self.cfg.task_backend: server = PulsarDS(bind='127.0.0.1:0', workers=0, key_value_save=[], name='%s_store' % self.name) yield server() address = 'pulsar://%s:%s' % server.cfg.addresses[0] else: address = self.cfg.task_backend store = create_store(address, pool_size=2, loop=monitor._loop) self.get_backend(store) loader = self.loader tags = self.cfg.labels exclude_tags = self.cfg.exclude_labels if self.cfg.show_leaks: show = show_leaks if self.cfg.show_leaks == 1 else hide_leaks self.cfg.set('when_exit', show) arbiter = pulsar.arbiter() arbiter.cfg.set('when_exit', show) try: tests = [] loader.runner.on_start() for tag, testcls in loader.testclasses(tags, exclude_tags): suite = loader.runner.loadTestsFromTestCase(testcls) if suite and suite._tests: tests.append((tag, testcls)) self._time_start = None if tests: self.logger.info('loading %s test classes', len(tests)) monitor.cfg.set('workers', min(self.cfg.workers, len(tests))) self._time_start = default_timer() queued = [] self._tests_done = set() self._tests_queued = None # # Bind to the task_done event self.backend.bind_event('task_done', partial(self._test_done, monitor)) for tag, testcls in tests: r = self.backend.queue_task('test', testcls=testcls, tag=tag) queued.append(r) queued = yield multi_async(queued) self.logger.debug('loaded %s test classes', len(tests)) self._tests_queued = set(queued) yield self._test_done(monitor) else: # pragma nocover raise ExitTest('Could not find any tests.') except ExitTest as e: # pragma nocover monitor.stream.writeln(str(e)) monitor.arbiter.stop() except Exception: # pragma nocover monitor.logger.critical('Error occurred while starting tests', exc_info=True) monitor._loop.call_soon(self._exit, 3)
def create_database(self, dbname=None, **kw): dbname = dbname or self.database store = create_store(self.dns, database='postgres', loop=self._loop) conn = store.sql_engine.connect() # when creating a database the connection must not be in a transaction conn.execute("commit") conn.execute("create database %s" % dbname) conn.close() return dbname
def on_loaded(self, app): '''Once the application has loaded, create the pub/sub handler used to publish messages to channels as well as subscribe to channels ''' pubsub_store = app.config['PUBSUB_STORE'] if pubsub_store: self.pubsub_store = create_store(pubsub_store) self.websocket.pubsub = self.pubsub_store.pubsub()
def setup(self, environ): cfg = environ['pulsar.cfg'] self.store = create_store(cfg.data_store) pubsub = self.store.pubsub() return WsgiHandler([ Router('/', get=self.home_page), FileRouter('/favicon.ico', FAVICON), WebSocket('/message', TweetsWsHandler(pubsub, self.channel)) ])
def __init__(self, app, scheme, url): super().__init__(app, scheme, url) if app.green_pool: from pulsar.apps.greenio import wait self._wait = wait self.client = create_store(url).client() else: import redis self.client = redis.StrictRedis.from_url(url)
def __init__(self, cfg, logger=None, app=None, **kw): self.store = create_store(cfg.data_store) self.cfg = cfg self.app = app self.green_pool = getattr(app, 'green_pool', GreenPool()) self.logger = logger or logging.getLogger('pulsar.queue') self._closing = False self._pubsub = PubSub(self) self.logger.debug('created %s', self)
def get_pubsub(self, websocket): if not self._store: cfg = websocket.cfg self._store = create_store(cfg.data_store, namespace='game_') self._client = self._store.client() self._pubsub = self._store.pubsub() gameserver = '%s:gameserver' % cfg.exc_id gameuser = '******' % cfg.exc_id yield from self._pubsub.subscribe(gameserver, gameuser)
def create(cls, app): protocol = module_attribute(app.config['PUBSUB_PROTOCOL'])() addr = app.config['PUBSUB_STORE'] if not addr or not app.green_pool: return store = create_store(addr) channels = store.channels(protocol=protocol, namespace=app.config['PUBSUB_PREFIX']) return cls(app, channels)
async def start_store(app, url, workers=0, **kw): '''Equivalent to :func:`.create_store` for most cases excepts when the ``url`` is for a pulsar store not yet started. In this case, a :class:`.PulsarDS` is started. ''' store = create_store(url, **kw) if store.name == 'pulsar': client = store.client() try: await client.ping() except ConnectionRefusedError: host = localhost(store._host) if not host: raise cfg = await send('arbiter', 'run', start_pulsar_ds, host, workers) store._host = cfg.addresses[0] dns = store.buildurl() store = create_store(dns, **kw) app.cfg.set('data_store', store.dns)
def get_pubsub(self, websocket): if not self._store: cfg = websocket.cfg self._store = create_store(cfg.data_store) self._client = self._store.client() self._pubsub = self._store.pubsub() webchat = '%s:webchat' % cfg.exc_id chatuser = '******' % cfg.exc_id yield from self._pubsub.subscribe(webchat, chatuser) return self._pubsub
async def start_store(app, url, workers=0, **kw): '''Equivalent to :func:`.create_store` for most cases excepts when the ``url`` is for a pulsar store not yet started. In this case, a :class:`.PulsarDS` is started. ''' store = create_store(url, **kw) if store.name == 'pulsar': client = store.client() try: await client.ping() except ConnectionRefusedError: host = localhost(store._host) if not host: raise cfg = await send('arbiter', 'run', start_pulsar_ds, host, workers) store._host = cfg.addresses[0] dns = store._buildurl() store = create_store(dns, **kw) app.cfg.set('data_store', store.dns)
def delete_database(self, dbname=None): # make sure no connections are opened self.sql_engine.dispose() dbname = dbname or self.database # switch to postgres database store = create_store(self.dns, database='postgres', loop=self._loop) conn = store.sql_engine.connect() conn.execute("commit") conn.execute("drop database %s" % dbname) conn.close() return dbname
def start_store(url, workers=0, **kw): '''Equivalent to :func:`.create_store` for most cases excepts when the ``url`` is for a pulsar store not yet started. In this case, a :class:`.PulsarDS` is started. ''' store = create_store(url, **kw) if store.name == 'pulsar': client = store.client() try: yield client.ping() except pulsar.ConnectionRefusedError: host = localhost(store._host) if not host: raise cfg = yield send('arbiter', 'run', start_pulsar_ds, host, workers) store._host = cfg.addresses[0] dns = store._buildurl() store = create_store(dns, **kw) coroutine_return(store)
def get_pubsub(self, websocket): '''Create the pubsub handler if not already available''' if not self._store: cfg = websocket.cfg self._store = create_store(cfg.data_store) self._client = self._store.client() self._pubsub = self._store.pubsub() webchat = '%s:webchat' % cfg.exc_id chatuser = '******' % cfg.exc_id self._pubsub.subscribe(webchat, chatuser) return self._pubsub
def create(cls, app): protocol = module_attribute(app.config['PUBSUB_PROTOCOL'])() addr = app.config['PUBSUB_STORE'] if not addr or not app.green_pool: return store = create_store(addr) channels = store.channels( protocol=protocol, namespace=app.config['PUBSUB_PREFIX'] ) return cls(app, channels)
def check_redis(): '''Check if redis server is available at the address specified by the :ref:`redis server <setting-redis_server>` config value. :rtype: boolean ''' addr = get_actor().cfg.get('redis_server') sync_store = create_store(addr, loop=new_event_loop()) try: sync_store.client().ping() return True except Exception: return False
def __init__(self, cfg, *, logger=None, **kw): # create the store for channels store = create_store(cfg.data_store, loop=cfg.params.pop('loop', None)) self.cfg = cfg self._loop = store._loop self.logger = logger self._closing_waiter = None if not cfg.message_broker: broker = store else: broker = create_store(cfg.message_broker, loop=self._loop) self.manager = (self.cfg.callable or Manager)(self) self.broker = register_broker(broker.name)(self, broker) self.channels = store.channels(protocol=self.broker, status_channel=ConsumerMessage.type, logger=self.logger) self.http = self.manager.http() self.green_pool = self.manager.green_pool() self.consumers = [] for consumer_path in self.cfg.consumers: consumer = module_attribute(consumer_path)(self) self.consumers.append(consumer) setattr(self, consumer.name, consumer)
def _autodiscover(self, binds): # Setup mdoels and engines if not binds: binds = {} elif isinstance(binds, str): binds = {'default': binds} if binds and 'default' not in binds: raise ImproperlyConfigured('default datastore not specified') self.metadata = MetaData() self._engines = {} self._nosql_engines = {} self._declarative_register = {} self.binds = {} # Create all sql engines in the binds dictionary # Quietly fails if the engine is not recognised, # it my be a NoSQL store for name, bind in tuple(binds.items()): key = None if name == 'default' else name try: self._engines[key] = create_engine(bind) except exc.NoSuchModuleError: self._nosql_engines[key] = create_store(bind) # if self._nosql_engines and not self.app.green_pool: raise ImproperlyConfigured('NoSql stores requires GREEN_POOL') for label, mod in module_iterator(self.app.config['EXTENSIONS']): # Loop through attributes in mod_models for name in dir(mod): value = getattr(mod, name) if isinstance(value, (Table, DeclarativeMeta)): for table in value.metadata.sorted_tables: if table.key not in self.metadata.tables: engine = None label = table.info.get('bind_label') keys = ('%s.%s' % (label, table.key), label, None) if label else (None,) for key in keys: engine = self.get_engine(key) if engine: break assert engine table.tometadata(self.metadata) self.binds[table] = engine if (isinstance(value, DeclarativeMeta) and hasattr(value, '__table__')): table = value.__table__ self._declarative_register[table.key] = value
def setup(self, environ): '''Called once only to setup the WSGI application handler. Check :ref:`lazy wsgi handler <wsgi-lazy-handler>` section for further information. ''' cfg = environ['pulsar.cfg'] loop = environ['pulsar.connection']._loop self.store = create_store(cfg.data_store, loop=loop) pubsub = self.store.pubsub(protocol=Protocol()) channel = '%s_webchat' % self.name pubsub.subscribe(channel) return WsgiHandler([Router('/', get=self.home_page), WebSocket('/message', Chat(pubsub, channel)), Router('/rpc', post=Rpc(pubsub, channel))])
def setup(self, environ): '''Called once only to setup the WSGI application handler. Check :ref:`lazy wsgi handler <wsgi-lazy-handler>` section for further information. ''' cfg = environ['pulsar.cfg'] loop = environ['pulsar.connection']._loop self.store = create_store(cfg.data_store, loop=loop) pubsub = self.store.pubsub(protocol=Protocol()) channel = '%s_webchat' % self.name pubsub.subscribe(channel) return WsgiHandler([Router('/', get=self.home_page), WebSocket('/message', Chat(pubsub, channel)), Router('/rpc', post=Rpc(pubsub, channel), response_content_types=JSON_CONTENT_TYPES)])
def check_server(name): '''Check if server ``name`` is available at the address specified ``<name>_server`` config value. :rtype: boolean ''' cfg = get_actor().cfg cfgname = '%s_server' % name addr = cfg.get('%s_server' % name) if ('%s://' % name) not in addr: addr = '%s://%s' % (name, addr) sync_store = create_store(addr, loop=new_event_loop()) try: sync_store.ping() return True except Exception: return False
async def __call__(self): cfg = get_actor().cfg addr = cfg.get('%s_server' % self.name) if addr: if ('%s://' % self.name) not in addr: addr = '%s://%s' % (self.name, addr) try: sync_store = create_store(addr) except ImproperlyConfigured: return False try: await sync_store.ping() return True except Exception: return False else: return False
def setup(self, environ): '''Called once only to setup the WSGI application handler. Check :ref:`lazy wsgi handler <wsgi-lazy-handler>` section for further information. ''' cfg = environ['pulsar.cfg'] loop = environ['pulsar.connection']._loop self.store = data.create_store(cfg.data_store, loop=loop) pubsub = self.store.pubsub(protocol=WsProtocol()) channel = '%s_messages' % self.name pubsub.subscribe(channel) middleware = [wsgi.Router('/', get=self.home_page), ws.WebSocket('/message', PhilosopherWs(pubsub, channel)), wsgi.FileRouter('/favicon.ico', FAVICON), wsgi.MediaRouter('media', ASSET_DIR)] return wsgi.WsgiHandler(middleware)
def setup(self, environ): '''Called once only to setup the WSGI application handler. Check :ref:`lazy wsgi handler <wsgi-lazy-handler>` section for further information. ''' cfg = environ['pulsar.cfg'] loop = environ['pulsar.connection']._loop self.store = create_store(cfg.data_store, loop=loop) pubsub = self.store.pubsub(protocol=Protocol()) channel = '%s_webchat' % self.name ensure_future(pubsub.subscribe(channel), loop=loop) return WsgiHandler([Router('/', get=self.home_page), WebSocket('/message', Chat(pubsub, channel)), Router('/rpc', post=Rpc(pubsub, channel), response_content_types=JSON_CONTENT_TYPES)], [AsyncResponseMiddleware, GZipMiddleware(min_length=20)])
def setup(self, environ): '''Called once only by the WSGI server. It returns a :class:`.WsgiHandler` with three routes: * The base route served by the :meth:`home_page` method * The websocket route * A route for static files ''' cfg = environ['pulsar.cfg'] # Create the store and the pubsub handler self.store = create_store(cfg.data_store) pubsub = self.store.pubsub() # subscribe to channel ensure_future(self.subscribe(pubsub)) return WsgiHandler([Router('/', get=self.home_page), MediaRouter('/static', STATIC_DIR), WebSocket('/message', TweetsWsHandler(pubsub, self.channel))])
def get_backend(self, store=None): if self.backend is None: if store is None: store = create_store(self.cfg.task_backend) else: self.cfg.set('task_backend', store.dns) task_backend = task_backends.get(store.name) if not task_backend: raise pulsar.ImproperlyConfigured( 'Task backend for %s not available' % store.name) self.backend = task_backend( store, logger=self.logger, name=self.name, task_paths=self.cfg.task_paths, schedule_periodic=self.cfg.schedule_periodic, max_tasks=self.cfg.max_requests, backlog=self.cfg.concurrent_tasks) self.logger.debug('created %s', self.backend) return self.backend
def setup(self, environ): '''Called once only to setup the WSGI application handler. Check :ref:`lazy wsgi handler <wsgi-lazy-handler>` section for further information. ''' request = wsgi_request(environ) cfg = request.cache.cfg loop = request.cache._loop self.store = create_store(cfg.data_store, loop=loop) pubsub = self.store.pubsub(protocol=Protocol()) channel = '%s_webchat' % self.name ensure_future(pubsub.subscribe(channel), loop=loop) return WsgiHandler([ Router('/', get=self.home_page), WebSocket('/message', Chat(pubsub, channel)), Router('/rpc', post=Rpc(pubsub, channel), response_content_types=JSON_CONTENT_TYPES) ], [AsyncResponseMiddleware, GZipMiddleware(min_length=20)])
def setup(self, environ): """Called once only to setup the WSGI application handler. Check :ref:`lazy wsgi handler <wsgi-lazy-handler>` section for further information. """ cfg = environ["pulsar.cfg"] loop = environ["pulsar.connection"]._loop self.store = create_store(cfg.data_store, loop=loop) pubsub = self.store.pubsub(protocol=Protocol()) channel = "%s_webchat" % self.name async(pubsub.subscribe(channel), loop=loop) return WsgiHandler( [ Router("/", get=self.home_page), WebSocket("/message", Chat(pubsub, channel)), Router("/rpc", post=Rpc(pubsub, channel), response_content_types=JSON_CONTENT_TYPES), ], [AsyncResponseMiddleware, GZipMiddleware(min_length=20)], async=True, )
def check_server(name): '''Check if server ``name`` is available at the address specified ``<name>_server`` config value. :rtype: boolean ''' cfg = get_actor().cfg addr = cfg.get('%s_server' % name) if addr: if ('%s://' % name) not in addr: addr = '%s://%s' % (name, addr) try: sync_store = create_store(addr, loop=new_event_loop()) except ImproperlyConfigured: return False try: sync_store._loop.run_until_complete(sync_store.ping()) return True except Exception: return False else: return False
def pubsub(self, key=None): '''Get a pub-sub handler for a given key A key is used to group together pub-subs so that bandwidths is reduced If no key is provided the handler is not included in the pubsub cache. ''' if not self._pubsub_store: if self.config['PUBSUB_STORE']: self._pubsub_store = create_store(self.config['PUBSUB_STORE']) self._pubsubs = {} else: self.logger.warning('No pubsub store configured. ' 'Cannot access pubsub handler') return if key: pubsub = self._pubsubs.get(key) if not pubsub: pubsub = self._pubsub_store.pubsub() self._pubsubs[key] = pubsub else: pubsub = self._pubsub_store.pubsub() return pubsub
def __call__(self, twitter, messages): if not self.store: self.store = create_store(twitter.cfg.data_store) self.pubsub = self.store.pubsub() for message in messages: self.pubsub.publish(self.channel, json.dumps(message))
def __call__(self, twitter, messages): if not self.store: self.store = create_store(twitter.cfg.data_store) self.pubsub = self.store.pubsub() ensure_future(self._publish(messages))
def create_store(cls, **kw): return create_store('mongodb://127.0.0.1:28017', **kw)
def __call__(self, messages): if not self.store: self.store = create_store(self.data_store) self.pubsub = self.store.pubsub() for message in messages: self.pubsub.publish(self.channel, json.dumps(message))
def __call__(self, twitter, messages): if not self.store: self.store = create_store(twitter.cfg.data_store) self.pubsub = self.store.pubsub() for message in messages: yield from self.pubsub.publish(self.channel, json.dumps(message))
def create_store(cls, pool_size=2, **kw): addr = '%s/%s' % (cls.cfg.couchdb_server, cls.name(cls.__name__)) return create_store(addr, pool_size=pool_size, **kw)
def create_store(cls, address, pool_size=2, **kw): return create_store(address, pool_size=pool_size, **kw)
def redis(self): if not hasattr(self, '_redis'): self._redis = create_store('redis://127.0.0.1:6379/15').client() return self._redis
def setUpClass(cls): cls.created = [] cls.store = create_store(cls.cfg.postgresql_server, database=cls.name('test')) assert cls.store.database == cls.name('test') return cls.createdb()
def __init__(self, dsn: str, logger: Optional[logging.Logger] = None): super().__init__(dsn, logger) _loop = asyncio.get_running_loop() self.redis_store = create_store(dsn, decode_responses=True, loop=_loop) self.transport = self.redis_store.client()
def on_loaded(self, app): pubsub_store = app.config['PUBSUB_STORE'] if pubsub_store: self.pubsub_store = create_store(pubsub_store) self.websocket.pubsub = self.pubsub_store.pubsub()
import time import sanic from sanic.response import json, text from pulsar.apps.data import create_store from pulsar.api import http_date app = sanic.Sanic("test") SERVER_NAME = 'Sanic/%s' % sanic.__version__ redis = create_store('redis://127.0.0.1:6379/15').client() @app.route("/") async def test(request): response = json({"test": True}) response.headers['Server'] = SERVER_NAME response.headers['Date'] = http_date(int(time.time())) return response @app.route("/payload/<size:int>") async def payload(request, size): response = text('d' * size) response.headers['Server'] = SERVER_NAME response.headers['Date'] = http_date(int(time.time())) return response @app.route("/ping-redis") async def ping_redis(request):
def __init__(self, dsn, prefix='PUBSUB', logger=None): super().__init__(dsn, prefix, logger) _loop = asyncio.get_running_loop() redis = create_store(dsn, decode_responses=True, loop=_loop) self.transport = redis.pubsub() self.transport.add_client(self.receiver)