def mem(self):
if getattr(self, 'client', None) is None:
if self.username and self.password:
self.client = Client(
servers=self.servers,
username=self.username,
password=self.password,
binary=True
)
else:
self.client = Client(self.servers)
return self.client
def get_memcached_client(config=None):
try:
from pylibmc import Client
except ImportError:
# try to use an alternative client
from memcache import Client as BaseClient
class Client(BaseClient):
def add_multi(self,
mapping,
time=0,
key_prefix='',
min_compress_len=0,
noreply=False):
"""
Behaviour of set_multi is some different but there is no other alternative
"""
return self.set_multi(mapping, time, key_prefix,
min_compress_len, noreply)
def delete(self, key, time=None, noreply=False):
"""
The logic of delete method is some different in memcache and pylibmc client
pylibmc client returns True only if key exists in cache
memcache client always returns True
This patch fixes the situation
"""
return self._deletetouch([b'DELETED'], "delete", key, time,
noreply)
memcached_params = dict({'servers': ['localhost']}, **config or {})
return Client(**memcached_params)
def __init__(self,
*,
namespace="dramatiq-results",
pool=None,
pool_size=8,
**parameters):
self.namespace = namespace
self.pool = pool or ClientPool(Client(**parameters), pool_size)
def __init__(self, *args, **kw):
self._client = Client(*args, **kw)
self.pool = ThreadMappedPool(self._client)
# using a locker to avoid race conditions
# when several clients for the same user
# get/set the cached data
self._locker = threading.RLock()
subscribe(REQUEST_ENDS, self._cleanup_pool)
def getmc():
try:
from pylibmc import Client
from config import cacheServers
adv_setting = {"cas": True, "tcp_nodelay": True, "ketama": True}
return Client(cacheServers, binary=True, behaviors=adv_setting)
except Exception as e:
print e
print "import pylibmc failed, perhaps your os is not supported"
def get_bmemcached_client(servers, timeout=None, **kwargs):
from bmemcached import Client
if timeout:
kwargs['socket_timeout'] = timeout
client = Client(servers, **kwargs)
client.TooBig = None
return client
def cache(localconfig=None):
"cache object"
if localconfig:
server = localconfig.get('baruwa.memcached.host', '127.0.0.1')
else:
server = config.get('baruwa.memcached.host', '127.0.0.1')
beh = {"tcp_nodelay": True, "ketama": True}
conn = Client([server], binary=True, behaviors=beh)
return conn
def __init__(self, **options):
self.servers = [
server.strip() for server in options['servers'].split(',')
]
self.prefix = options['prefix']
self._client = Client(self.servers)
self.pool = ThreadMappedPool(self._client)
# using a locker to avoid race conditions
# when several clients for the same user
# get/set the cached data
self._locker = threading.RLock()
def getmc():
"""
@return: Memcached ClientPool
"""
try:
from pylibmc import Client, ClientPool
from config import cacheServers, MC_POOL_SIZE
adv_setting = {"cas": True, "tcp_nodelay": True, "ketama": True}
mc = Client(cacheServers, binary=True, behaviors=adv_setting)
return ClientPool(mc, MC_POOL_SIZE)
except ImportError:
return ClientPoolEmptyOnWIndows()
def get_pylibmc_client(servers, timeout=None, binary=True, **kwargs):
from pylibmc import Client
try:
from pylibmc import TooBig
except ImportError:
from pylibmc import Error, ServerError
TooBig = (Error, ServerError)
if timeout:
kwargs['behaviors'] = {'connect_timeout': timeout}
client = Client(servers, binary=binary, **kwargs)
client.TooBig = TooBig
return client
def _setup_cache(app):
"""
If a test is being run or we don't want cache, NullCache will be initialized just as a dummy.
If running locally without the 'DISABLE_CACHE' env variable and without a memcached instance running,
MemcachedCache and it's underlying pylibmc will give no warning on connection, but will throw
exceptions when trying to work with the cache. A few connection retires will be made in that
scenario, and eventually the cache will be replaced with a NullCache. Binary communications must
be used for SASL.
"""
# initialize the retry count if it's our first time here
if not hasattr(app, 'cache_retry'):
app.cache_retry = 0
# Setup cache
if app.config['TESTING'] or os.environ.get('DISABLE_CACHE',
None) is not None:
app.cache = NullCache()
app.logger.debug('Cache initialized as NullCache')
else:
MEMCACHED_SERVERS = os.environ.get('MEMCACHEDCLOUD_SERVERS',
'127.0.0.1:11211')
try:
memcached_client = Client(
servers=MEMCACHED_SERVERS.split(','),
username=os.environ.get('MEMCACHEDCLOUD_USERNAME'),
password=os.environ.get('MEMCACHEDCLOUD_PASSWORD'),
binary=True)
app.cache = MemcachedCache(memcached_client)
app.logger.debug(
'Cache initialized as MemcachedCache with servers: %s',
MEMCACHED_SERVERS)
except Exception as e:
# very unlikely to have an exception here. pylibmc mostly throws when trying to communicate, not connect
app.logger.error('Error initializing MemcachedCache: %s', e)
app.logger.error('Initializing cache as NullCache. Fix ASAP!')
app.cache = NullCache()
from urllib2 import Request, urlopen
import HTMLParser
import helpers
app = Flask(__name__)
if os.path.exists(app.config.root_path + '/config.cfg') is False:
print "copy config.default.cfg to config.cfg and add your settings"
app.config.from_pyfile(app.config.root_path + '/config.default.cfg')
else:
app.config.from_pyfile(app.config.root_path + '/config.cfg')
babel = Babel(app)
Compress(app)
client = Client(app.config['MEMCACHED'])
mc_pool = ClientPool(client, app.config['POOL_SIZE'])
def cache_name(pagename, lang=''):
if not lang:
lang = 'sv' if 'sv' in request.url_rule.rule else 'en'
return '%s_%s' % (pagename, lang)
def check_cache(page, lang=''):
# If the cache should not be used, return None
if app.config['TEST']:
return None
try:
with mc_pool.reserve() as client:
with open(os.path.join(APP_STATIC, 'post_data_stable.xml')) as f:
POST_DATA_STABLE = f.read().replace('\n', '')
with open(os.path.join(APP_STATIC, 'post_data_beta.xml')) as f:
POST_DATA_BETA = f.read().replace('\n', '')
with open(os.path.join(APP_STATIC, 'post_data_dev.xml')) as f:
POST_DATA_DEV = f.read().replace('\n', '')
post_data = OrderedDict([('stable', POST_DATA_STABLE),
('beta', POST_DATA_BETA),
('dev', POST_DATA_DEV)])
if MEMCACHE_SERVERS:
client = Client([MEMCACHE_SERVERS], behaviors={"tcp_nodelay": True},
binary=True, username=MEMCACHE_USERNAME, password=MEMCACHE_PASSWORD)
else:
client = Client(['127.0.0.1:11211'])
cache = MemcachedCache(client, default_timeout=60)
def cached(timeout=60):
def decorator(f):
@wraps(f)
def decorated_function(*args, **kwargs):
key = args[0]
rv = cache.get(key)
if rv is not None:
print 'hit\t\t', key
return rv
print 'miss\t\t', key
def __init__(self, *, pool_size=8, **parameters):
behaviors = parameters["behaviors"] = parameters.get("behaviors", {})
behaviors["cas"] = True
self.client = client = Client(**parameters)
self.pool = ClientPool(client, pool_size)
def __init__(self, servers, services, ttl, binary):
self.services = services
self.ttl = ttl
self._cache = Client(servers, binary=binary)
self._cache.behaviors = {"no_block": True}
def create_app():
"""Instanciate app."""
app = Flask(__name__)
if os.path.exists(app.config.root_path + '/config.cfg') is False:
print("copy config.default.cfg to config.cfg and add your settings")
app.config.from_pyfile(app.config.root_path + '/config.default.cfg')
else:
app.config.from_pyfile(app.config.root_path + '/config.cfg')
babel = Babel(app)
Compress(app)
@babel.localeselector
def get_locale():
"""Get correct language from url."""
locale = request.path[1:].split('/', 1)[0]
if locale in ['sv', 'en']:
return locale
else:
locale = 'sv'
for lang in list(request.accept_languages.values()):
if lang[:2] in ['sv', 'en']:
locale = lang[:2]
break
g.locale = locale
return locale
client = Client(app.config['MEMCACHED'])
@app.before_request
def func():
g.babel = Babel
g.language = get_locale()
g.config = app.config
g.mc_pool = ClientPool(client, app.config['POOL_SIZE'])
@app.context_processor
def inject_custom():
d = {
'lurl_for':
lambda ep, **kwargs: url_for(ep + '_' + g.language, **kwargs)
}
return d
@app.template_filter('deescape')
def deescape_filter(s):
html_parser = html.parser.HTMLParser()
return html_parser.unescape(s)
@app.template_filter('cclink')
def cclink_filter(s):
return re.sub(
r'(CC-BY\S*)',
'<a href="https://creativecommons.org/licenses/" target="_blank">\\1</a>',
s)
from . import helpers
app.jinja_env.globals.update(get_life_range=helpers.get_life_range)
app.jinja_env.globals.update(make_namelist=helpers.make_namelist)
app.jinja_env.globals.update(make_datelist=helpers.make_datelist)
app.jinja_env.globals.update(
make_simplenamelist=helpers.make_simplenamelist)
app.jinja_env.globals.update(make_placelist=helpers.make_placelist)
app.jinja_env.globals.update(make_placenames=helpers.make_placenames)
app.jinja_env.globals.update(
make_alphabetical_bucket=helpers.make_alphabetical_bucket)
app.jinja_env.globals.update(get_date=helpers.get_date)
app.jinja_env.globals.update(join_name=helpers.join_name)
app.jinja_env.globals.update(swedish_translator=helpers.swedish_translator)
app.jinja_env.globals.update(sorted=sorted)
app.jinja_env.globals.update(len=len)
app.jinja_env.globals.update(get_lang_text=helpers.get_lang_text)
app.jinja_env.globals.update(get_shorttext=helpers.get_shorttext)
app.jinja_env.globals.update(get_org_name=helpers.get_org_name)
app.jinja_env.globals.update(rewrite_von=helpers.rewrite_von)
app.jinja_env.globals.update(lowersorted=helpers.lowersorted)
app.jinja_env.globals.update(get_current_date=helpers.get_current_date)
app.jinja_env.globals.update(karp_fe_url=helpers.karp_fe_url)
from . import views
app.register_blueprint(views.bp)
app.register_error_handler(Exception, views.page_not_found)
app.wsgi_app = flask_reverse_proxy.ReverseProxied(app.wsgi_app)
return app
