def cache(self):
# Remove annoying depcration warning from flask-cache
from flask.exthook import ExtDeprecationWarning
warnings.simplefilter('ignore', ExtDeprecationWarning)
if hasattr(self, '_cache'):
return self._cache
if CACHE_BACKEND_URI == 'memory://':
cache = Cache(app, config={'CACHE_TYPE': 'simple'})
elif CACHE_BACKEND_URI.startswith('redis://'):
try:
from redis import from_url as redis_from_url
redis_from_url(CACHE_BACKEND_URI)
except:
print('BAD REDIS URL PROVIDED BY (CACHE_BACKEND_URI)')
exit(1)
cache = Cache(app, config={
'CACHE_TYPE': 'redis',
'CACHE_REDIS_URL': CACHE_BACKEND_URI,
'CACHE_DEFAULT_TIMEOUT': 0 # NEVER EXPIRES
})
cache.init_app(self.app)
self._cache = cache
return self._cache
def redis(app, config, args, kwargs):
try:
from redis import from_url as redis_from_url
except ImportError:
raise RuntimeError('no redis module found')
kwargs.update(
dict(host=config.get('CACHE_REDIS_HOST', 'localhost'),
port=config.get('CACHE_REDIS_PORT', 6379)))
password = config.get('CACHE_REDIS_PASSWORD')
if password:
kwargs['password'] = password
key_prefix = config.get('CACHE_KEY_PREFIX')
if key_prefix:
kwargs['key_prefix'] = key_prefix
db_number = config.get('CACHE_REDIS_DB')
if db_number:
kwargs['db'] = db_number
redis_url = config.get('CACHE_REDIS_URL')
if redis_url:
kwargs['host'] = redis_from_url(redis_url, db=kwargs.pop('db', None))
return RedisCache(*args, **kwargs)
def __init__(self, app, entry_point_group=None):
"""Initialize state."""
self.app = app
self.scopes = {}
# Initialize OAuth2 provider
oauth2.init_app(app)
# Configures the OAuth2 provider to use the SQLALchemy models for
# getters and setters for user, client and tokens.
bind_sqlalchemy(oauth2, db.session, client=Client)
# Flask-OAuthlib does not support CACHE_REDIS_URL
if app.config['OAUTH2_CACHE_TYPE'] == 'redis' and app.config.get(
'CACHE_REDIS_URL'):
from redis import from_url as redis_from_url
app.config.setdefault(
'OAUTH2_CACHE_REDIS_HOST',
redis_from_url(app.config['CACHE_REDIS_URL'])
)
# Configures an OAuth2Provider instance to use configured caching
# system to get and set the grant token.
bind_cache_grant(app, oauth2, lambda: OAuthUserProxy(current_user))
# Disables oauthlib's secure transport detection in in debug mode.
if app.debug or app.testing:
os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1'
if entry_point_group:
self.load_entry_point_group(entry_point_group)
def __init__(self, app, entry_point_group=None):
"""Initialize state."""
self.app = app
self.scopes = {}
# Initialize OAuth2 provider
oauth2.init_app(app)
# Flask-OAuthlib does not support CACHE_REDIS_URL
if app.config['OAUTH2_CACHE_TYPE'] == 'redis' and app.config.get(
'CACHE_REDIS_URL'):
from redis import from_url as redis_from_url
app.config.setdefault(
'OAUTH2_CACHE_REDIS_HOST',
redis_from_url(app.config['CACHE_REDIS_URL'])
)
# Configures an OAuth2Provider instance to use configured caching
# system to get and set the grant token.
bind_cache_grant(app, oauth2, lambda: OAuthUserProxy(current_user))
# Disables oauthlib's secure transport detection in in debug mode.
if app.debug or app.testing:
os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1'
if entry_point_group:
self.load_entry_point_group(entry_point_group)
def redis(app, config, args, kwargs):
kwargs.update(dict(
host=config.get('CACHE_REDIS_HOST', 'localhost'),
port=config.get('CACHE_REDIS_PORT', 6379),
))
password = config.get('CACHE_REDIS_PASSWORD')
if password:
kwargs['password'] = password
key_prefix = config.get('CACHE_KEY_PREFIX')
if key_prefix:
kwargs['key_prefix'] = key_prefix
db_number = config.get('CACHE_REDIS_DB')
if db_number:
kwargs['db'] = db_number
redis_url = config.get('CACHE_REDIS_URL')
if redis_url:
kwargs['host'] = redis_from_url(
redis_url,
db=kwargs.pop('db', None),
)
return RedisCache(*args, **kwargs)
def factory(cls, app, config, args, kwargs):
try:
from redis import from_url as redis_from_url
except ImportError as e:
raise RuntimeError("no redis module found") from e
kwargs.update(
dict(
host=config.get("CACHE_REDIS_HOST", "localhost"),
port=config.get("CACHE_REDIS_PORT", 6379),
db=config.get("CACHE_REDIS_DB", 0),
))
password = config.get("CACHE_REDIS_PASSWORD")
if password:
kwargs["password"] = password
key_prefix = config.get("CACHE_KEY_PREFIX")
if key_prefix:
kwargs["key_prefix"] = key_prefix
redis_url = config.get("CACHE_REDIS_URL")
if redis_url:
kwargs["host"] = redis_from_url(redis_url,
db=kwargs.pop("db", None))
new_class = cls(*args, **kwargs)
return new_class
def redis(app, config, args, kwargs):
try:
from redis import from_url as redis_from_url
except ImportError:
raise RuntimeError("no redis module found")
kwargs.update(
dict(
host=config.get("CACHE_REDIS_HOST", "localhost"),
port=config.get("CACHE_REDIS_PORT", 6379),
)
)
password = config.get("CACHE_REDIS_PASSWORD")
if password:
kwargs["password"] = password
key_prefix = config.get("CACHE_KEY_PREFIX")
if key_prefix:
kwargs["key_prefix"] = key_prefix
db_number = config.get("CACHE_REDIS_DB")
if db_number:
kwargs["db"] = db_number
redis_url = config.get("CACHE_REDIS_URL")
if redis_url:
kwargs["host"] = redis_from_url(redis_url, db=kwargs.pop("db", None))
return RedisCache(*args, **kwargs)
def redis(config, args, kwargs):
try:
from redis import from_url as redis_from_url
except ImportError:
raise RuntimeError("no redis module found")
kwargs.update(
dict(
host=config.get("CACHE_REDIS_HOST", "localhost"),
port=config.get("CACHE_REDIS_PORT", 6379),
))
password = config.get("CACHE_REDIS_PASSWORD")
if password:
kwargs["password"] = password
key_prefix = config.get("CACHE_KEY_PREFIX")
if key_prefix:
kwargs["key_prefix"] = key_prefix
db_number = config.get("CACHE_REDIS_DB")
if db_number:
kwargs["db"] = db_number
redis_url = config.get("CACHE_REDIS_URL")
if redis_url:
kwargs["host"] = redis_from_url(redis_url, db=kwargs.pop("db", None))
return Redis(*args, **kwargs)
def factory(cls, app, config, args, kwargs):
key_prefix = config.get('CACHE_KEY_PREFIX')
if key_prefix:
kwargs['key_prefix'] = key_prefix
kwargs['host'] = redis_from_url(config['CACHE_REDIS_URL'],
socket_timeout=1)
return IndicoRedisCache(*args, **kwargs)
def make_indico_redis_cache(app, config, args, kwargs):
from redis import from_url as redis_from_url
key_prefix = config.get('CACHE_KEY_PREFIX')
if key_prefix:
kwargs['key_prefix'] = key_prefix
kwargs['host'] = redis_from_url(config['CACHE_REDIS_URL'], socket_timeout=1)
return IndicoRedisCache(*args, **kwargs)
def redis(app, config, args, kwargs):
kwargs.update(dict(
host=config.get('CACHE_REDIS_HOST', 'localhost'),
port=config.get('CACHE_REDIS_PORT', 6379),
))
password = config.get('CACHE_REDIS_PASSWORD')
if password:
kwargs['password'] = password
key_prefix = config.get('CACHE_KEY_PREFIX')
if key_prefix:
kwargs['key_prefix'] = key_prefix
db_number = config.get('CACHE_REDIS_DB')
if db_number:
kwargs['db'] = db_number
redis_url = config.get('CACHE_REDIS_URL')
if redis_url:
kwargs['host'] = redis_from_url(
redis_url,
db=kwargs.pop('db', None),
)
return RedisCache(*args, **kwargs)
def setup_app():
"""Setup OAuth2 provider."""
# Initialize OAuth2 provider
oauth2.init_app(current_app)
# Configures the OAuth2 provider to use the SQLALchemy models for getters
# and setters for user, client and tokens.
bind_sqlalchemy(oauth2, db.session, client=Client)
# Flask-OAuthlib does not support CACHE_REDIS_URL
if cfg['OAUTH2_CACHE_TYPE'] == 'redis' and \
cfg.get('CACHE_REDIS_URL'):
from redis import from_url as redis_from_url
cfg.setdefault(
'OAUTHLIB_CACHE_REDIS_HOST',
redis_from_url(cfg['CACHE_REDIS_URL'])
)
# Configures an OAuth2Provider instance to use configured caching system
# to get and set the grant token.
bind_cache_grant(current_app, oauth2, OAuthUserProxy.get_current_user)
# Disables oauthlib's secure transport detection in in debug mode.
if current_app.debug or current_app.testing:
os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1'
def setup_client(self, app=None, uri=None):
if app is None:
app = current_app
if uri is None:
uri = app.config.get('REDIS_URI')
if uri:
self.client = redis_from_url(uri)
elif app.configured and not app.testing:
raise ValueError('Redis extension: REDIS_URI is not defined in '
'application configuration')
def setup_client(self, app=None, uri=None):
if app is None:
app = current_app
if uri is None:
uri = app.config.get('REDIS_URI')
if uri:
self.client = redis_from_url(uri)
elif app.configured and not app.testing:
raise ValueError('Redis extension: REDIS_URI is not defined in '
'application configuration')
def _init_local_cache():
from common import settings
global _cache_backend
if settings.CACHE_TYPE == 'simple':
from flask_caching.backends.simple import SimpleCache
_cache_backend = SimpleCache()
elif settings.CACHE_TYPE == 'redis':
from redis import from_url as redis_from_url
from flask_caching.backends.rediscache import RedisCache
_cache_backend = RedisCache(key_prefix=settings.CACHE_KEY_PREFIX,
host=redis_from_url(
settings.CACHE_REDIS_URL))
def _get_cache_client():
"""
# we use bare python redis client to get list of all keys
# since this is not supported in the flask cache
:return: redis connection
"""
config = app.cache.config
cache_type = config['CACHE_TYPE']
if cache_type != 'redis':
print('NOT SUPPORTED CACHE BACKEND, ONLY SUPPORTED IS (redis)')
exit(1)
try:
return redis_from_url(app.cache.config['CACHE_REDIS_URL'])
except:
print('BAD REDIS URL PROVIDED BY (CACHE_BACKEND_URI)')
exit(1)
def redis(app, config, kwargs):
kwargs.update(dict(
host=config.get("CACHE_REDIS_HOST", "localhost"),
port=config.get("CACHE_REDIS_PORT", 6379),
))
password = config.get("CACHE_REDIS_PASSWORD")
if password:
kwargs["password"] = password
db_number = config.get("CACHE_REDIS_DB")
if db_number:
kwargs["db"] = db_number
redis_url = config.get("CACHE_REDIS_URL")
if redis_url:
kwargs["host"] = redis_from_url(redis_url,
db=kwargs.pop('db', None)
)
return Redis(**kwargs)
def redis(app, config, args, kwargs):
kwargs.update(dict(host=config.get("CACHE_REDIS_HOST", "localhost"), port=config.get("CACHE_REDIS_PORT", 6379)))
password = config.get("CACHE_REDIS_PASSWORD")
if password:
kwargs["password"] = password
key_prefix = config.get("CACHE_KEY_PREFIX")
if key_prefix:
kwargs["key_prefix"] = key_prefix
db_number = config.get("CACHE_REDIS_DB")
if db_number:
kwargs["db"] = db_number
redis_url = config.get("CACHE_REDIS_URL")
if redis_url:
kwargs["host"] = redis_from_url(redis_url, db=kwargs.pop("db", None))
return RedisCache(*args, **kwargs)
app = Flask(__name__)
app.config.from_pyfile('moviesneaker.cfg')
db = SQLAlchemy(app)
from models import *
DEBUG = False
if os.path.exists('/home/dotcloud/environment.json'):
with open('/home/dotcloud/environment.json') as f:
env = json.load(f)
else: # we're debugging
DEBUG = True
env = None
if env:
redis = redis_from_url(env["DOTCLOUD_DATA_REDIS_URL"])
else: # we're debugging
redis = Redis()
showtime_parse_queue = Queue('showtime_parse', redis)
rp = redis.connection_pool.connection_kwargs
cache = Cache(app,
config={
'CACHE_TYPE': 'redis',
'CACHE_REDIS_HOST': rp.get('host'),
'CACHE_REDIS_POST': rp.get('port'),
'CACHE_REDIS_PASSWORD': rp.get('password')
})
### Various utilities I'll break out later
from redis import from_url as redis_from_url
if os.path.exists('/home/dotcloud/environment.json'):
with open('/home/dotcloud/environment.json') as f:
env = json.load(f)
else:
env = None
BASE = "https://graph.facebook.com/%s"
DEFAULT_TIMEZONE = "America/Los_Angeles"
DATE_FORMAT = '%Y-%m-%dT%H:%M:%S'
app = Flask(__name__)
app.debug = True
if env:
redis = redis_from_url(env["DOTCLOUD_DATA_REDIS_URL"])
else:
redis = Redis()
def get_events(access_token,limit=50):
response = requests.get(BASE%'me/events?limit=%(limit)s&access_token=%(access_token)s'%locals())
return json.loads(response.text)
def localize_date_str(datestr,date_fmt,timezone):
return timezone.localize(datetime.strptime(datestr,date_fmt))
def parse_event(fbevent,default_timezone=DEFAULT_TIMEZONE,date_fmt=DATE_FORMAT):
event = icalendar.Event()
event.add('summary',fbevent['name'])
timezone = pytz.timezone(fbevent['timezone'] if fbevent.has_key('timezone') else default_timezone)
event.add('dtstart',localize_date_str(fbevent['start_time'],date_fmt,timezone))
import sys
from flask import Flask
from flask_socketio import SocketIO
from flask_pymongo import PyMongo
import os
from redis import from_url as redis_from_url
app = Flask(__name__)
app.config['SECRET_KEY'] = os.getenv('SECRET')
app.config['MONGO_URI'] = os.getenv('MONGODB_URI')
app.config['REDIS_URL'] = os.getenv('REDIS_URL')
mongo = PyMongo(app)
redis = redis_from_url(app.config['REDIS_URL'])
socketio = SocketIO(app, cors_allowed_origins="*")
from .routes import *
if not startDate:
raise ValueError(f"Invalid period {period}")
return df[(df['Date'] > startDate) & (df['Date'] <= endDate)]
@timing
def get_periodStart_to_endDate(self, period='M', endDate='2012-06-30'):
# TODO refactor redis pattern
redisKey = PERIOD_START_TO_END + period + endDate
if not self.redis.exists(redisKey):
print(
f"Rebuild data by period and start to end date {period} {endDate}"
)
df = self._get_periodStart_to_endDate(period, endDate)
self.redis.set(redisKey,
self.context.serialize(df).to_buffer().to_pybytes())
self.redis.expire(redisKey, REDIS_EXPIRE)
return df
else:
print("Loading period start to end date from Redis: ", redisKey)
return self.context.deserialize(self.redis.get(redisKey))
if __name__ == '__main__':
import os
from redis import from_url as redis_from_url
redis = redis_from_url(os.environ.get("REDIS_URL"))
tm = Turtle_Manager(redis)
df = tm.df_all
import os
import time
import requests
import rpqueue
from redis import from_url as redis_from_url
redis = redis_from_url(os.environ.get("REDIS_URL", "redis://localhost:6379"))
rpqueue.set_redis_connection(redis)
DEFAULT_RETRY_DELAY = 10
@rpqueue.task()
def callback(request_id,
message,
callback_url,
scheduled_at,
last_retry=None,
retry_delay=None,
_attempts=None):
if retry_delay is None:
retry_delay = DEFAULT_RETRY_DELAY
try:
response = requests.post(callback_url, data=message)
response.raise_for_status()
except requests.RequestException:
callback.retry(request_id=request_id,
message=message,
callback_url=callback_url,
