Exemple #1
0
def redis(app=None):
    app = app_or_default(app)

    if not hasattr(app, "redbeat_redis") or app.redbeat_redis is None:
        app.redbeat_redis = StrictRedis.from_url(app.conf.REDBEAT_REDIS_URL, decode_responses=True)

    return app.redbeat_redis
Exemple #2
0
def get_redis(app=None):
    app = app_or_default(app)
    conf = ensure_conf(app)
    if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None:
        redis_options = conf.app.conf.get(
            'REDBEAT_REDIS_OPTIONS',
            conf.app.conf.get('BROKER_TRANSPORT_OPTIONS', {}))
        retry_period = redis_options.get('retry_period')
        if conf.redis_url.startswith(
                'redis-sentinel') and 'sentinels' in redis_options:
            from redis.sentinel import Sentinel
            sentinel = Sentinel(
                redis_options['sentinels'],
                socket_timeout=redis_options.get('socket_timeout'),
                password=redis_options.get('password'),
                decode_responses=True)
            connection = sentinel.master_for(
                redis_options.get('service_name', 'master'))
        else:
            connection = StrictRedis.from_url(conf.redis_url,
                                              decode_responses=True)

        if retry_period is None:
            app.redbeat_redis = connection
        else:
            app.redbeat_redis = RetryingConnection(retry_period, connection)

    return app.redbeat_redis
Exemple #3
0
def server():
    redis_server = StrictRedis.from_url(REDIS_URL)
    keys = redis_server.keys(
        SCHEDULER_QUEUE_KEY % {'spider': ATestSpider.name} + '*')
    if keys:
        redis_server.delete(*keys)
    return redis_server
Exemple #4
0
 def flushdb(self):
     """Destroy every shard's db
     """
     for pool in self.pool_map.values():
         con = StrictRedis.from_url(pool.url, connection_pool=pool)
         self.log.debug("flushing shard member: %s", con)
         con.flushdb()
         del con
Exemple #5
0
def get_redis(app=None):
    app = app_or_default(app)
    conf = ensure_conf(app)
    if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None:
        redis_options = conf.app.conf.get(
            'REDBEAT_REDIS_OPTIONS',
            conf.app.conf.get('BROKER_TRANSPORT_OPTIONS', {}))
        retry_period = redis_options.get('retry_period')
        if conf.redis_url.startswith(
                'redis-sentinel') and 'sentinels' in redis_options:
            from redis.sentinel import Sentinel
            sentinel = Sentinel(
                redis_options['sentinels'],
                socket_timeout=redis_options.get('socket_timeout'),
                password=redis_options.get('password'),
                db=redis_options.get('db', 0),
                decode_responses=True)
            connection = sentinel.master_for(
                redis_options.get('service_name', 'master'))
        elif conf.redis_url.startswith('rediss'):
            ssl_options = {'ssl_cert_reqs': ssl.CERT_REQUIRED}
            if isinstance(conf.redis_use_ssl, dict):
                ssl_options.update(conf.redis_use_ssl)
            connection = StrictRedis.from_url(conf.redis_url,
                                              decode_responses=True,
                                              **ssl_options)
        elif conf.redis_url.startswith('redis-cluster'):
            from rediscluster import RedisCluster
            if not redis_options.get('startup_nodes'):
                redis_options = {
                    'startup_nodes': [{
                        "host": "localhost",
                        "port": "30001"
                    }]
                }
            connection = RedisCluster(decode_responses=True, **redis_options)
        else:
            connection = StrictRedis.from_url(conf.redis_url,
                                              decode_responses=True)

        if retry_period is None:
            app.redbeat_redis = connection
        else:
            app.redbeat_redis = RetryingConnection(retry_period, connection)

    return app.redbeat_redis
Exemple #6
0
def redis(app=None):
    app = app_or_default(app)

    if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None:
        app.redbeat_redis = StrictRedis.from_url(app.conf.REDBEAT_REDIS_URL,
                                                 decode_responses=True)

    return app.redbeat_redis
Exemple #7
0
def redis(app=None):
    app = app_or_default(app)
    conf = ensure_conf(app)
    if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None:
        app.redbeat_redis = StrictRedis.from_url(conf.redis_url,
                                                 decode_responses=True)

    return app.redbeat_redis
Exemple #8
0
 def __init__(self, redis_connection=None, locker=None, *args, **kwargs):
     self.__redis_connection = redis_connection
     if self.__redis_connection is None:
          self.__redis_connection = StrictRedis.from_url(current_app.conf.CELERY_REDIS_SCHEDULER_URL)
     self._schedule = EntryProxy(self.__redis_connection)
     self._locker = locker
     if self._locker is None:
         self._locker = Redlock([current_app.conf.CELERY_REDIS_SCHEDULER_URL])
     super(ChardScheduler, self).__init__(*args, **kwargs)
def get_app(config=None):
    """
    App factory.

    :param dict config: configuration that can override config
        from `settings.py`
    :return: a new SuperdeskEve app instance
    """
    if config is None:
        config = {}

    config.setdefault('SOURCES', {})
    config['APP_ABSPATH'] = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))

    for key in dir(settings):
        if key.isupper():
            config.setdefault(key, getattr(settings, key))

    media_storage = SuperdeskGridFSMediaStorage
    if config.get('AMAZON_CONTAINER_NAME'):
        from superdesk.storage.amazon.amazon_media_storage import AmazonMediaStorage
        media_storage = AmazonMediaStorage

    app = Eve(
        auth=BearerAuth,
        settings=config,
        data=SuperdeskDataLayer,
        media=media_storage,
        json_encoder=MongoJSONEncoder,
        validator=SuperdeskValidator
    )

    superdesk.app = app
    _set_error_handlers(app)
    app.mail = Mail(app)
    if config.get('REDIS_URL'):
        app.redis = StrictRedis.from_url(config['REDIS_URL'], 0)

    for module_name in app.config['INSTALLED_APPS']:
        app_module = importlib.import_module(module_name)
        try:
            app_module.init_app(app)
        except AttributeError:
            pass

    for resource in config['DOMAIN']:
        app.register_resource(resource, config['DOMAIN'][resource])

    for blueprint in superdesk.BLUEPRINTS:
        prefix = app.api_prefix or None
        app.register_blueprint(blueprint, url_prefix=prefix)

    app.sentry = sentry
    sentry.init_app(app)

    return app
def get_app(config=None):
    """
    App factory.

    :param dict config: configuration that can override config
        from `settings.py`
    :return: a new SuperdeskEve app instance
    """
    if config is None:
        config = {}

    config['APP_ABSPATH'] = os.path.dirname(
        os.path.abspath(os.path.dirname(__file__)))

    for key in dir(settings):
        if key.isupper():
            config.setdefault(key, getattr(settings, key))

    media_storage = SuperdeskGridFSMediaStorage
    if config.get('AMAZON_CONTAINER_NAME'):
        from superdesk.storage.amazon.amazon_media_storage import AmazonMediaStorage
        media_storage = AmazonMediaStorage

    app = Eve(auth=BearerAuth,
              settings=config,
              data=SuperdeskDataLayer,
              media=media_storage,
              json_encoder=MongoJSONEncoder,
              validator=SuperdeskValidator)

    superdesk.app = app
    _set_error_handlers(app)
    app.mail = Mail(app)
    if config.get('REDIS_URL'):
        app.redis = StrictRedis.from_url(config['REDIS_URL'], 0)

    for module_name in app.config['INSTALLED_APPS']:
        app_module = importlib.import_module(module_name)
        try:
            app_module.init_app(app)
        except AttributeError:
            pass

    for resource in config['DOMAIN']:
        app.register_resource(resource, config['DOMAIN'][resource])

    for blueprint in superdesk.BLUEPRINTS:
        prefix = app.api_prefix or None
        app.register_blueprint(blueprint, url_prefix=prefix)

    app.sentry = sentry
    sentry.init_app(app)

    return app
Exemple #11
0
    def __init__(self, redis_connection=None, locker=None, *args, **kwargs):
        self.__redis_connection = redis_connection
        if self.__redis_connection is None:
            self.__redis_connection = StrictRedis.from_url(
                current_app.conf.CELERY_REDIS_SCHEDULER_URL)

        self._schedule = EntryProxy(self.__redis_connection)
        self._locker = locker
        if self._locker is None:
            self._locker = Redlock(
                [current_app.conf.CELERY_REDIS_SCHEDULER_URL])
        super(ProbitScheduler, self).__init__(*args, **kwargs)
Exemple #12
0
 def connection(self, node_id):
     if not self.__ready:
         return
     shard = self.shard_node(node_id)
     try:
         pool = self.pool_map.get(self.shard_map.get(shard))
         con = StrictRedis.from_url(pool.url, connection_pool=pool)
         yield con
     except Exception:
         self.log.exception("Something blew up in the Redis context "
                            "manager")
         raise
     finally:
         del con
class DataManager:

    client = StrictRedis.from_url(REDIS_PERSISTENT_DB, decode_responses=True)
    words_counter = 'words'

    def increment_keys(self, values):
        if values:
            for key, amount in values.items():
                self.increment_key(key=key, amount=amount)

    def increment_key(self, key, amount):
        self.client.hincrby(name=self.words_counter, key=key, amount=amount)

    def get_key(self, key):
        return self.client.hget(self.words_counter, key)
def get_redis(app=None):
    app = app_or_default(app)
    conf = ensure_conf(app)
    if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None:
        redis_options = conf.app.conf.get(
            'REDBEAT_REDIS_OPTIONS',
            conf.app.conf.get('BROKER_TRANSPORT_OPTIONS', {}))
        retry_period = redis_options.get('retry_period')
        connection = StrictRedis.from_url(conf.redis_url, decode_responses=True)
        if retry_period is None:
            app.redbeat_redis = connection
        else:
            app.redbeat_redis = RetryingConnection(retry_period, connection)

    return app.redbeat_redis
Exemple #15
0
def redis(app=None):
    app = app_or_default(app)
    conf = ensure_conf(app)
    if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None:
        BROKER_TRANSPORT_OPTIONS = conf.app.conf.get(
            'BROKER_TRANSPORT_OPTIONS')
        if conf.redis_url.startswith(
                'redis-sentinel') and 'sentinels' in BROKER_TRANSPORT_OPTIONS:
            from redis.sentinel import Sentinel
            sentinel = Sentinel(BROKER_TRANSPORT_OPTIONS['sentinels'],
                                socket_timeout=0.1)
            service_name = BROKER_TRANSPORT_OPTIONS.get(
                'service_name', 'master')
            app.redbeat_redis = sentinel.master_for(service_name,
                                                    socket_timeout=0.1)
        else:
            app.redbeat_redis = StrictRedis.from_url(conf.redis_url,
                                                     decode_responses=True)

    return app.redbeat_redis
Exemple #16
0
 def pipeline(self, node_id):
     if not self.__ready:
         return
     shard = self.shard_node(node_id)
     try:
         pool = self.pool_map.get(self.shard_map.get(shard))
         con = StrictRedis.from_url(pool.url, connection_pool=pool)
         with con.pipeline() as pipe:
             try:
                 yield pipe
                 pipe.execute()
             except:
                 self.log.exception("Something blew up inside a pipeline "
                                    "context")
                 pipe.reset()
                 raise
     except Exception:
         self.log.exception("Something blew up in the Redis context "
                            "manager")
         raise
     finally:
         del con
Exemple #17
0
    def open_spider(self, spider):
        if isinstance(spider, JdCategorySpider):
            self.client = MongoClient(spider.settings.get(
                'MONGODB_URI'))  # 也可以直接用从settings文件导入的MONGODB_URI
            self.collection = self.client['jd_spider']['category']
            self.redis = StrictRedis.from_url(spider.settings.get('REDIS_URL'))

            # 只插入一个item,用做测试
            item = {
                "outer_cate_title":
                "家用电器",
                "outer_cate_url":
                "https://jiadian.jd.com",
                "middle_cate_title":
                "电视",
                "middle_cate_url":
                "https://list.jd.com/list.html?cat=737,794,798",
                "inner_cate_title":
                "超薄电视",
                "inner_cate_url":
                "https://list.jd.com/list.html?cat=737,794,798&ev=4155_76344&sort=sort_rank_asc&trans=1&JL=2_1_0#J_crumbsBar"
            }
            data = pickle.dumps(item)
            self.redis.rpush(JdProductSpider.redis_key, data)
Exemple #18
0
from redis.exceptions import RedisError
from telegram.bot import Bot
from telegram.ext.commandhandler import CommandHandler
from telegram.ext.conversationhandler import ConversationHandler
from telegram.ext.dispatcher import Dispatcher
from telegram.ext.regexhandler import RegexHandler

from epl_bot.helpers import _get_upcoming_matches
from django.conf import settings

from epl_bot.models import Fixture, Bet, Score

logger = logging.getLogger(__name__)

MATCH, REBET, HOSTS, GUESTS = range(4)
redis = StrictRedis.from_url(settings.REDIS_URL)
match_id = ''


def bet(bot, update):
    upcoming = _get_upcoming_matches(0, 10)
    if upcoming:
        try:
            buttons = [[telegram.KeyboardButton(u"{}. {}".format(i, m.bet_text))]
                       for i, m in enumerate(upcoming, start=1)]
            reply_markup = telegram.ReplyKeyboardMarkup(buttons,
                                                        one_time_keyboard=True)
            bot.sendMessage(chat_id=update.message.chat_id,
                            text="Choose a match to bet:",
                            reply_markup=reply_markup)
try:
    import simplejson as json
except ImportError:
    import json

from celery.beat import Scheduler, ScheduleEntry
from celery.utils.log import get_logger
from celery import current_app
import celery.schedules

from redis.client import StrictRedis

from decoder import DateTimeDecoder, DateTimeEncoder

# share with result backend
rdb = StrictRedis.from_url(current_app.conf.CELERY_REDIS_SCHEDULER_URL)


class ValidationError(Exception):
    pass


class PeriodicTask(object):
    '''represents a periodic task
    '''
    name = None
    task = None

    type_ = None

    interval = None
Exemple #20
0
 def init_app(self, connection: str = None):
     self.client = StrictRedis.from_url(
         connection or settings.REDIS_CONNECTION_QUEUE_BASE,
         socket_timeout=settings.REDIS_SOCKET_TIMEOUT,
         socket_connect_timeout=settings.REDIS_SOCKET_CONNECT_TIMEOUT,
     )
Exemple #21
0
 def __init__(self, redis_url, lock_timeout=None, lock_sleep=None):
     super(CodisCache, self).__init__(lock_timeout=lock_timeout,
                                      lock_sleep=lock_sleep)
     self.client = StrictRedis.from_url(redis_url)
Exemple #22
0
 def __init__(self, redis_url='redis://localhost:6379/0', base_key=None):
     self.base_key = base_key
     self.redis_url = redis_url
     if redis_url not in self._redis_map:
         self._redis_map[redis_url] = StrictRedis.from_url(redis_url)
     self._redis = self._redis_map[redis_url]
Exemple #23
0
def get_redis_client():
    redis_client = StrictRedis.from_url(settings.REDIS_URL)
    return redis_client