def redis(app=None):
app = app_or_default(app)
if not hasattr(app, "redbeat_redis") or app.redbeat_redis is None:
app.redbeat_redis = StrictRedis.from_url(app.conf.REDBEAT_REDIS_URL, decode_responses=True)
return app.redbeat_redis
def get_redis(app=None):
app = app_or_default(app)
conf = ensure_conf(app)
if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None:
redis_options = conf.app.conf.get(
'REDBEAT_REDIS_OPTIONS',
conf.app.conf.get('BROKER_TRANSPORT_OPTIONS', {}))
retry_period = redis_options.get('retry_period')
if conf.redis_url.startswith(
'redis-sentinel') and 'sentinels' in redis_options:
from redis.sentinel import Sentinel
sentinel = Sentinel(
redis_options['sentinels'],
socket_timeout=redis_options.get('socket_timeout'),
password=redis_options.get('password'),
decode_responses=True)
connection = sentinel.master_for(
redis_options.get('service_name', 'master'))
else:
connection = StrictRedis.from_url(conf.redis_url,
decode_responses=True)
if retry_period is None:
app.redbeat_redis = connection
else:
app.redbeat_redis = RetryingConnection(retry_period, connection)
return app.redbeat_redis
def server():
redis_server = StrictRedis.from_url(REDIS_URL)
keys = redis_server.keys(
SCHEDULER_QUEUE_KEY % {'spider': ATestSpider.name} + '*')
if keys:
redis_server.delete(*keys)
return redis_server
def flushdb(self):
"""Destroy every shard's db
"""
for pool in self.pool_map.values():
con = StrictRedis.from_url(pool.url, connection_pool=pool)
self.log.debug("flushing shard member: %s", con)
con.flushdb()
del con
def get_redis(app=None):
app = app_or_default(app)
conf = ensure_conf(app)
if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None:
redis_options = conf.app.conf.get(
'REDBEAT_REDIS_OPTIONS',
conf.app.conf.get('BROKER_TRANSPORT_OPTIONS', {}))
retry_period = redis_options.get('retry_period')
if conf.redis_url.startswith(
'redis-sentinel') and 'sentinels' in redis_options:
from redis.sentinel import Sentinel
sentinel = Sentinel(
redis_options['sentinels'],
socket_timeout=redis_options.get('socket_timeout'),
password=redis_options.get('password'),
db=redis_options.get('db', 0),
decode_responses=True)
connection = sentinel.master_for(
redis_options.get('service_name', 'master'))
elif conf.redis_url.startswith('rediss'):
ssl_options = {'ssl_cert_reqs': ssl.CERT_REQUIRED}
if isinstance(conf.redis_use_ssl, dict):
ssl_options.update(conf.redis_use_ssl)
connection = StrictRedis.from_url(conf.redis_url,
decode_responses=True,
**ssl_options)
elif conf.redis_url.startswith('redis-cluster'):
from rediscluster import RedisCluster
if not redis_options.get('startup_nodes'):
redis_options = {
'startup_nodes': [{
"host": "localhost",
"port": "30001"
}]
}
connection = RedisCluster(decode_responses=True, **redis_options)
else:
connection = StrictRedis.from_url(conf.redis_url,
decode_responses=True)
if retry_period is None:
app.redbeat_redis = connection
else:
app.redbeat_redis = RetryingConnection(retry_period, connection)
return app.redbeat_redis
def redis(app=None):
app = app_or_default(app)
if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None:
app.redbeat_redis = StrictRedis.from_url(app.conf.REDBEAT_REDIS_URL,
decode_responses=True)
return app.redbeat_redis
def redis(app=None):
app = app_or_default(app)
conf = ensure_conf(app)
if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None:
app.redbeat_redis = StrictRedis.from_url(conf.redis_url,
decode_responses=True)
return app.redbeat_redis
def __init__(self, redis_connection=None, locker=None, *args, **kwargs):
self.__redis_connection = redis_connection
if self.__redis_connection is None:
self.__redis_connection = StrictRedis.from_url(current_app.conf.CELERY_REDIS_SCHEDULER_URL)
self._schedule = EntryProxy(self.__redis_connection)
self._locker = locker
if self._locker is None:
self._locker = Redlock([current_app.conf.CELERY_REDIS_SCHEDULER_URL])
super(ChardScheduler, self).__init__(*args, **kwargs)
def get_app(config=None):
"""
App factory.
:param dict config: configuration that can override config
from `settings.py`
:return: a new SuperdeskEve app instance
"""
if config is None:
config = {}
config.setdefault('SOURCES', {})
config['APP_ABSPATH'] = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
for key in dir(settings):
if key.isupper():
config.setdefault(key, getattr(settings, key))
media_storage = SuperdeskGridFSMediaStorage
if config.get('AMAZON_CONTAINER_NAME'):
from superdesk.storage.amazon.amazon_media_storage import AmazonMediaStorage
media_storage = AmazonMediaStorage
app = Eve(
auth=BearerAuth,
settings=config,
data=SuperdeskDataLayer,
media=media_storage,
json_encoder=MongoJSONEncoder,
validator=SuperdeskValidator
)
superdesk.app = app
_set_error_handlers(app)
app.mail = Mail(app)
if config.get('REDIS_URL'):
app.redis = StrictRedis.from_url(config['REDIS_URL'], 0)
for module_name in app.config['INSTALLED_APPS']:
app_module = importlib.import_module(module_name)
try:
app_module.init_app(app)
except AttributeError:
pass
for resource in config['DOMAIN']:
app.register_resource(resource, config['DOMAIN'][resource])
for blueprint in superdesk.BLUEPRINTS:
prefix = app.api_prefix or None
app.register_blueprint(blueprint, url_prefix=prefix)
app.sentry = sentry
sentry.init_app(app)
return app
def get_app(config=None):
"""
App factory.
:param dict config: configuration that can override config
from `settings.py`
:return: a new SuperdeskEve app instance
"""
if config is None:
config = {}
config['APP_ABSPATH'] = os.path.dirname(
os.path.abspath(os.path.dirname(__file__)))
for key in dir(settings):
if key.isupper():
config.setdefault(key, getattr(settings, key))
media_storage = SuperdeskGridFSMediaStorage
if config.get('AMAZON_CONTAINER_NAME'):
from superdesk.storage.amazon.amazon_media_storage import AmazonMediaStorage
media_storage = AmazonMediaStorage
app = Eve(auth=BearerAuth,
settings=config,
data=SuperdeskDataLayer,
media=media_storage,
json_encoder=MongoJSONEncoder,
validator=SuperdeskValidator)
superdesk.app = app
_set_error_handlers(app)
app.mail = Mail(app)
if config.get('REDIS_URL'):
app.redis = StrictRedis.from_url(config['REDIS_URL'], 0)
for module_name in app.config['INSTALLED_APPS']:
app_module = importlib.import_module(module_name)
try:
app_module.init_app(app)
except AttributeError:
pass
for resource in config['DOMAIN']:
app.register_resource(resource, config['DOMAIN'][resource])
for blueprint in superdesk.BLUEPRINTS:
prefix = app.api_prefix or None
app.register_blueprint(blueprint, url_prefix=prefix)
app.sentry = sentry
sentry.init_app(app)
return app
def __init__(self, redis_connection=None, locker=None, *args, **kwargs):
self.__redis_connection = redis_connection
if self.__redis_connection is None:
self.__redis_connection = StrictRedis.from_url(
current_app.conf.CELERY_REDIS_SCHEDULER_URL)
self._schedule = EntryProxy(self.__redis_connection)
self._locker = locker
if self._locker is None:
self._locker = Redlock(
[current_app.conf.CELERY_REDIS_SCHEDULER_URL])
super(ProbitScheduler, self).__init__(*args, **kwargs)
def connection(self, node_id):
if not self.__ready:
return
shard = self.shard_node(node_id)
try:
pool = self.pool_map.get(self.shard_map.get(shard))
con = StrictRedis.from_url(pool.url, connection_pool=pool)
yield con
except Exception:
self.log.exception("Something blew up in the Redis context "
"manager")
raise
finally:
del con
class DataManager:
client = StrictRedis.from_url(REDIS_PERSISTENT_DB, decode_responses=True)
words_counter = 'words'
def increment_keys(self, values):
if values:
for key, amount in values.items():
self.increment_key(key=key, amount=amount)
def increment_key(self, key, amount):
self.client.hincrby(name=self.words_counter, key=key, amount=amount)
def get_key(self, key):
return self.client.hget(self.words_counter, key)
def get_redis(app=None):
app = app_or_default(app)
conf = ensure_conf(app)
if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None:
redis_options = conf.app.conf.get(
'REDBEAT_REDIS_OPTIONS',
conf.app.conf.get('BROKER_TRANSPORT_OPTIONS', {}))
retry_period = redis_options.get('retry_period')
connection = StrictRedis.from_url(conf.redis_url, decode_responses=True)
if retry_period is None:
app.redbeat_redis = connection
else:
app.redbeat_redis = RetryingConnection(retry_period, connection)
return app.redbeat_redis
def redis(app=None):
app = app_or_default(app)
conf = ensure_conf(app)
if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None:
BROKER_TRANSPORT_OPTIONS = conf.app.conf.get(
'BROKER_TRANSPORT_OPTIONS')
if conf.redis_url.startswith(
'redis-sentinel') and 'sentinels' in BROKER_TRANSPORT_OPTIONS:
from redis.sentinel import Sentinel
sentinel = Sentinel(BROKER_TRANSPORT_OPTIONS['sentinels'],
socket_timeout=0.1)
service_name = BROKER_TRANSPORT_OPTIONS.get(
'service_name', 'master')
app.redbeat_redis = sentinel.master_for(service_name,
socket_timeout=0.1)
else:
app.redbeat_redis = StrictRedis.from_url(conf.redis_url,
decode_responses=True)
return app.redbeat_redis
def pipeline(self, node_id):
if not self.__ready:
return
shard = self.shard_node(node_id)
try:
pool = self.pool_map.get(self.shard_map.get(shard))
con = StrictRedis.from_url(pool.url, connection_pool=pool)
with con.pipeline() as pipe:
try:
yield pipe
pipe.execute()
except:
self.log.exception("Something blew up inside a pipeline "
"context")
pipe.reset()
raise
except Exception:
self.log.exception("Something blew up in the Redis context "
"manager")
raise
finally:
del con
def open_spider(self, spider):
if isinstance(spider, JdCategorySpider):
self.client = MongoClient(spider.settings.get(
'MONGODB_URI')) # 也可以直接用从settings文件导入的MONGODB_URI
self.collection = self.client['jd_spider']['category']
self.redis = StrictRedis.from_url(spider.settings.get('REDIS_URL'))
# 只插入一个item，用做测试
item = {
"outer_cate_title":
"家用电器",
"outer_cate_url":
"https://jiadian.jd.com",
"middle_cate_title":
"电视",
"middle_cate_url":
"https://list.jd.com/list.html?cat=737,794,798",
"inner_cate_title":
"超薄电视",
"inner_cate_url":
"https://list.jd.com/list.html?cat=737,794,798&ev=4155_76344&sort=sort_rank_asc&trans=1&JL=2_1_0#J_crumbsBar"
}
data = pickle.dumps(item)
self.redis.rpush(JdProductSpider.redis_key, data)
from redis.exceptions import RedisError
from telegram.bot import Bot
from telegram.ext.commandhandler import CommandHandler
from telegram.ext.conversationhandler import ConversationHandler
from telegram.ext.dispatcher import Dispatcher
from telegram.ext.regexhandler import RegexHandler
from epl_bot.helpers import _get_upcoming_matches
from django.conf import settings
from epl_bot.models import Fixture, Bet, Score
logger = logging.getLogger(__name__)
MATCH, REBET, HOSTS, GUESTS = range(4)
redis = StrictRedis.from_url(settings.REDIS_URL)
match_id = ''
def bet(bot, update):
upcoming = _get_upcoming_matches(0, 10)
if upcoming:
try:
buttons = [[telegram.KeyboardButton(u"{}. {}".format(i, m.bet_text))]
for i, m in enumerate(upcoming, start=1)]
reply_markup = telegram.ReplyKeyboardMarkup(buttons,
one_time_keyboard=True)
bot.sendMessage(chat_id=update.message.chat_id,
text="Choose a match to bet:",
reply_markup=reply_markup)
try:
import simplejson as json
except ImportError:
import json
from celery.beat import Scheduler, ScheduleEntry
from celery.utils.log import get_logger
from celery import current_app
import celery.schedules
from redis.client import StrictRedis
from decoder import DateTimeDecoder, DateTimeEncoder
# share with result backend
rdb = StrictRedis.from_url(current_app.conf.CELERY_REDIS_SCHEDULER_URL)
class ValidationError(Exception):
pass
class PeriodicTask(object):
'''represents a periodic task
'''
name = None
task = None
type_ = None
interval = None
def init_app(self, connection: str = None):
self.client = StrictRedis.from_url(
connection or settings.REDIS_CONNECTION_QUEUE_BASE,
socket_timeout=settings.REDIS_SOCKET_TIMEOUT,
socket_connect_timeout=settings.REDIS_SOCKET_CONNECT_TIMEOUT,
)
def __init__(self, redis_url, lock_timeout=None, lock_sleep=None):
super(CodisCache, self).__init__(lock_timeout=lock_timeout,
lock_sleep=lock_sleep)
self.client = StrictRedis.from_url(redis_url)
def __init__(self, redis_url='redis://localhost:6379/0', base_key=None):
self.base_key = base_key
self.redis_url = redis_url
if redis_url not in self._redis_map:
self._redis_map[redis_url] = StrictRedis.from_url(redis_url)
self._redis = self._redis_map[redis_url]
def get_redis_client():
redis_client = StrictRedis.from_url(settings.REDIS_URL)
return redis_client