def __init__(self, host_list):
super(BMemcachedCli, self).__init__()
# look hosts for username/password for SASL authentication - when found, assume all servers use the same
auth = [None, None]
for idx in range(len(host_list)):
if host_list[idx].find('@') > -1:
auth = host_list[idx].split('@')[0].split(':')
host_list[idx] = host_list[idx].split('@')[
1] # remove auth from server entry
self.memcache = Client(host_list, auth[0], auth[1])
self._check_connection(
host_list
) # Make sure we actually have connections, since our library doesn't make this apparent
if auth[0]:
self.prompt = '([B]memcached) '
else:
self.prompt = '(memcached) '
# Time for some hardcore introspection and dynamic method creation
for name in dir(self.memcache):
# List our methods, ignoring private methods and ones that should be hidden to our CLI
if not name.startswith('_') and not self._is_hidden(name):
attr = getattr(self.memcache, name)
if callable(attr):
# Make sure we only keep callable methods, let's pin the doc strings to them while we are at it
setattr(self.__class__, 'do_' + name, self._make_cmd(name))
doc = (getattr(attr, '__doc__') or '').strip()
if doc: # Not everything has a docstring
setattr(self.__class__, 'help_' + name,
self._make_help(doc))
def __init__(
self,
host,
port: int = 11211,
user: Optional[str] = None,
password: Optional[str] = None,
):
"""Init Cache Layer."""
self.client = Client((f"{host}:{port}", ), user, password)
class CacheLayer(object):
"""Memcache Wrapper."""
def __init__(
self,
host,
port: int = 11211,
user: Optional[str] = None,
password: Optional[str] = None,
):
"""Init Cache Layer."""
self.client = Client((f"{host}:{port}",), user, password)
def get_image_from_cache(self, img_hash: str) -> Tuple[bytes, ImageType]:
"""
Get image body from cache layer.
Attributes
----------
img_hash : str
file url.
Returns
-------
img : bytes
image body.
ext : str
image ext
"""
content, ext = self.client.get(img_hash)
return content, ext
def set_image_cache(
self, img_hash: str, body: Tuple[bytes, ImageType], timeout: int = 432000
) -> bool:
"""
Set base64 encoded image body in cache layer.
Attributes
----------
img_hash : str
file url.
body : tuple
image body + ext
Returns
-------
bool
"""
try:
return self.client.set(img_hash, body, time=timeout)
except Exception:
return False
class MemcacheStorage(Storage):
'''
Storage class for a memcached cluster.
'''
def __init__(self, servers):
from bmemcached import Client
self.mc = Client(servers)
self.last_stats = {}
def get(self, key):
return self.mc.get(key)
def put(self, key, data):
return self.mc.set(key, data) != 0
def __init__(self, host_list):
super(BMemcachedCli, self).__init__()
# look hosts for username/password for SASL authentication - when found, assume all servers use the same
auth = [None, None]
for idx in xrange(len(host_list)):
if host_list[idx].find('@') > -1:
auth = host_list[idx].split('@')[0].split(':')
host_list[idx] = host_list[idx].split('@')[1] # remove auth from server entry
self.memcache = Client(host_list, auth[0], auth[1])
self._check_connection(
host_list) # Make sure we actually have connections, since our library doesn't make this apparent
if auth[0]:
self.prompt = '([B]memcached) '
else:
self.prompt = '(memcached) '
# Time for some hardcore introspection and dynamic method creation
for name in dir(self.memcache):
# List our methods, ignoring private methods and ones that should be hidden to our CLI
if not name.startswith('_') and not self._is_hidden(name):
attr = getattr(self.memcache, name)
if callable(attr):
# Make sure we only keep callable methods, let's pin the doc strings to them while we are at it
setattr(self.__class__, 'do_' + name, self._make_cmd(name))
doc = (getattr(attr, '__doc__') or '').strip()
if doc: # Not everything has a docstring
setattr(self.__class__, 'help_' + name, self._make_help(doc))
def get_cached_post(url, postdata, host='127.0.0.1', port=11211):
"""Returns url data from url with post request"""
servers = ["%s:%d" % (host, port)]
m = hashlib.sha256()
m.update(url.encode('utf8'))
m.update(str(postdata).encode('utf8'))
key = m.hexdigest()
client = Client(servers)
c_data = client.get(key)
if c_data:
data = decompress(c_data)
else:
r = requests.post(url, postdata)
data = r.text
client.set(key, compress(data))
hp = lxml.etree.HTMLParser(encoding='utf-8')
root = lxml.html.fromstring(data, parser=hp)
return root
def get_cached_url(url,
timeout=DEFAULT_CACHE_TIMEOUT,
host='127.0.0.1',
port=11211):
"""Returns url data from url or from local memcached"""
servers = ["%s:%d" % (host, port)]
m = hashlib.sha256()
m.update(url.encode('utf8'))
key = m.hexdigest()
client = Client(servers)
c_data = client.get(key)
if c_data:
data = decompress(c_data)
else:
o = urlopen(url)
rurl = o.geturl()
data = o.read()
client.set(key, compress(data))
hp = lxml.etree.HTMLParser(encoding='utf-8')
root = lxml.html.fromstring(data, parser=hp)
return root
class BMemcachedCli(cmd.Cmd, object):
def __init__(self, host_list):
super(BMemcachedCli, self).__init__()
# look hosts for username/password for SASL authentication - when found, assume all servers use the same
auth = [None, None]
for idx in xrange(len(host_list)):
if host_list[idx].find('@') > -1:
auth = host_list[idx].split('@')[0].split(':')
host_list[idx] = host_list[idx].split('@')[
1] # remove auth from server entry
self.memcache = Client(host_list, auth[0], auth[1])
self._check_connection(
host_list
) # Make sure we actually have connections, since our library doesn't make this apparent
if auth[0]:
self.prompt = '([B]memcached) '
else:
self.prompt = '(memcached) '
# Time for some hardcore introspection and dynamic method creation
for name in dir(self.memcache):
# List our methods, ignoring private methods and ones that should be hidden to our CLI
if not name.startswith('_') and not self._is_hidden(name):
attr = getattr(self.memcache, name)
if callable(attr):
# Make sure we only keep callable methods, let's pin the doc strings to them while we are at it
setattr(self.__class__, 'do_' + name, self._make_cmd(name))
doc = (getattr(attr, '__doc__') or '').strip()
if doc: # Not everything has a docstring
setattr(self.__class__, 'help_' + name,
self._make_help(doc))
@staticmethod
def _make_cmd(name):
# Whats a little functional passing between friends?
# This is our core factory for creating dynamic methods
def _get_stats(self, line):
try:
# Rabbit Holes can get pretty deep, better to keep a lid on it
pp = pprint.PrettyPrinter(depth=4)
pp.pprint(self.memcache.stats(line))
except Exception, e:
print_error(e)
def handler(self, line):
parts = line.split()
try:
# This is where the magic happens, get our function by name, pass the arguments,
# then pretty-print the results
pprint.pprint(getattr(self.memcache, name)(*parts))
except Exception, e:
print_error(e)
class BMemcachedCli(cmd.Cmd, object):
def __init__(self, host_list):
super(BMemcachedCli, self).__init__()
# look hosts for username/password for SASL authentication - when found, assume all servers use the same
auth = [None, None]
for idx in xrange(len(host_list)):
if host_list[idx].find('@') > -1:
auth = host_list[idx].split('@')[0].split(':')
host_list[idx] = host_list[idx].split('@')[1] # remove auth from server entry
self.memcache = Client(host_list, auth[0], auth[1])
self._check_connection(
host_list) # Make sure we actually have connections, since our library doesn't make this apparent
if auth[0]:
self.prompt = '([B]memcached) '
else:
self.prompt = '(memcached) '
# Time for some hardcore introspection and dynamic method creation
for name in dir(self.memcache):
# List our methods, ignoring private methods and ones that should be hidden to our CLI
if not name.startswith('_') and not self._is_hidden(name):
attr = getattr(self.memcache, name)
if callable(attr):
# Make sure we only keep callable methods, let's pin the doc strings to them while we are at it
setattr(self.__class__, 'do_' + name, self._make_cmd(name))
doc = (getattr(attr, '__doc__') or '').strip()
if doc: # Not everything has a docstring
setattr(self.__class__, 'help_' + name, self._make_help(doc))
@staticmethod
def _make_cmd(name):
# Whats a little functional passing between friends?
# This is our core factory for creating dynamic methods
def _get_stats(self, line):
try:
# Rabbit Holes can get pretty deep, better to keep a lid on it
pp = pprint.PrettyPrinter(depth=4)
pp.pprint(self.memcache.stats(line))
except Exception, e:
print_error(e)
def handler(self, line):
parts = line.split()
try:
# This is where the magic happens, get our function by name, pass the arguments,
# then pretty-print the results
pprint.pprint(getattr(self.memcache, name)(*parts))
except Exception, e:
print_error(e)
class BMemcachedCli(cmd.Cmd, object):
def __init__(self, host_list):
super(BMemcachedCli, self).__init__()
# look hosts for username/password for SASL authentication - when found, assume all servers use the same
auth = [None, None]
for idx in range(len(host_list)):
if host_list[idx].find('@') > -1:
auth = host_list[idx].split('@')[0].split(':')
host_list[idx] = host_list[idx].split('@')[
1] # remove auth from server entry
self.memcache = Client(host_list, auth[0], auth[1])
self._check_connection(
host_list
) # Make sure we actually have connections, since our library doesn't make this apparent
if auth[0]:
self.prompt = '([B]memcached) '
else:
self.prompt = '(memcached) '
# Time for some hardcore introspection and dynamic method creation
for name in dir(self.memcache):
# List our methods, ignoring private methods and ones that should be hidden to our CLI
if not name.startswith('_') and not self._is_hidden(name):
attr = getattr(self.memcache, name)
if callable(attr):
# Make sure we only keep callable methods, let's pin the doc strings to them while we are at it
setattr(self.__class__, 'do_' + name, self._make_cmd(name))
doc = (getattr(attr, '__doc__') or '').strip()
if doc: # Not everything has a docstring
setattr(self.__class__, 'help_' + name,
self._make_help(doc))
@staticmethod
def _make_cmd(name):
# Whats a little functional passing between friends?
# This is our core factory for creating dynamic methods
def _get_stats(self, line):
try:
# Rabbit Holes can get pretty deep, better to keep a lid on it
pp = pprint.PrettyPrinter(depth=4)
pp.pprint(self.memcache.stats(line))
except Exception as e:
print_error(e)
def handler(self, line):
parts = line.split()
try:
# This is where the magic happens, get our function by name, pass the arguments,
# then pretty-print the results
pprint.pprint(getattr(self.memcache, name)(*parts))
except Exception as e:
print_error(e)
return handler
@staticmethod
# This just lets cmd know how to print help commands for arbitrary methods
def _make_help(doc):
def help(self):
print(doc)
return help
def _check_connection(self, host_list):
# Get Stats for all hosts, make sure we connect to all of them
unreachable_hosts = []
reachable_hosts = []
stats = self.memcache.stats()
# Stats returns a list with each host getting a tuple with two elements, a string and a
# dictionary (We just hit 3 of the python data structures in one return, wtf)
# The string contains the hostname and some other junk
# The dictionary contains all of the stats for that host
for stat in stats:
reachable_hosts.append(stat)
# Compare our list of reachable hosts with the ones we were passed
unreachable_hosts = [
host for host in host_list if host not in reachable_hosts
]
if unreachable_hosts:
for host in unreachable_hosts:
print_error("Unable to connect to memcache server: %s" % host)
sys.exit(1)
# Helper method for trimming out arbitrary commands from our library
def _is_hidden(self, name):
hidden_commands = ['get_multi', 'set_multi']
for command in hidden_commands:
if command in name:
return True
return False
# Make it so we can exit easily
def do_exit(self, line):
return True
# Alias EOF to exit
do_EOF = do_exit
# We don't want to repeat the last command if a blank line is entered (default behavior), so we pass
def emptyline(self):
pass
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from bmemcached import Client
import os
import time
from . import report
from .logger import logger
_CLIENT = Client(
os.environ.get('MEMCACHEDCLOUD_SERVERS', 'localhost:11211').split(','),
os.environ.get('MEMCACHEDCLOUD_USERNAME', ''),
os.environ.get('MEMCACHEDCLOUD_PASSWORD', ''),
)
def get_client():
return _CLIENT
def update():
logger.info('Update the data')
bcache = get_client()
bugs = report.get_bugs()
data = report.get_stats(bugs)
if not bcache.replace('data', data, time=0, compress_level=0):
bcache.set('data', data, time=0, compress_level=0)
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from bmemcached import Client
import hashlib
from itertools import chain
import os
import time
from . import config, signatures
from .logger import logger
__CLIENT = Client(
os.environ.get('MEMCACHEDCLOUD_SERVERS').split(','),
os.environ.get('MEMCACHEDCLOUD_USERNAME'),
os.environ.get('MEMCACHEDCLOUD_PASSWORD'))
def get_client():
return __CLIENT
def get_value(hgurls, sgns):
data = signatures.get_for_urls_sgns(hgurls, sgns, [], sumup=True)
data, links, versions = signatures.prepare_bug_for_html(data)
return (data, links, versions)
def get_hash(key):
key = key.encode('utf-8')
md5 = hashlib.md5(key).hexdigest()
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from bmemcached import Client
import hashlib
from itertools import chain
import os
import time
from . import config, signatures
from .logger import logger
__CLIENT = Client(
os.environ.get('MEMCACHEDCLOUD_SERVERS',
config.get_memcached('servers')).split(','),
os.environ.get('MEMCACHEDCLOUD_USERNAME',
config.get_memcached('username')),
os.environ.get('MEMCACHEDCLOUD_PASSWORD',
config.get_memcached('password')))
def get_client():
return __CLIENT
def get_value(hgurls, sgns, extra):
data = signatures.get_for_urls_sgns(hgurls,
sgns, [],
extra=extra,
sumup=True)
data, links, versions, affected, has_extra = signatures.prepare_bug_for_html(
def __init__(self, servers):
from bmemcached import Client
self.mc = Client(servers)
self.last_stats = {}
