def bench_cache_lifetime(minutes):
"Attempts to find how long a given memcached key can be expected to live"
from pylons import g
from r2.lib.cache import Memcache
# we'll create an independent connection to memcached for this
# test
mc = Memcache(g.memcaches)
# set N keys, and tell them not to live for longer than this test
mc.set_multi(dict(('bench_cache_%d' % x, x) for x in xrange(minutes)),
time=minutes * 60)
# and every minute, check to see that the keys are still present,
# until we find one missing
for x in xrange(minutes):
if mc.get('bench_cache_%d' % x, None) is not None:
sleep(60)
else:
# we found one missing
return x - 1
else:
# we're out of numbers, and we didn't find any missing
# keys. Since we only set N keys, we can't check for anything
# else
print(("Cache lifetime is greater than %d minutes. Try again with a" +
" higher 'minutes' value") % minutes)
return None
def bench_cache_lifetime(minutes):
"Attempts to find how long a given memcached key can be expected to live"
from pylons import g
from r2.lib.cache import Memcache
# we'll create an independent connection to memcached for this
# test
mc = Memcache(g.memcaches)
# set N keys, and tell them not to live for longer than this test
mc.set_multi(dict( ('bench_cache_%d' % x, x)
for x in xrange(minutes) ),
time=minutes*60)
# and every minute, check to see that the keys are still present,
# until we find one missing
for x in xrange(minutes):
if mc.get('bench_cache_%d' % x, None) is not None:
sleep(60)
else:
# we found one missing
return x-1
else:
# we're out of numbers, and we didn't find any missing
# keys. Since we only set N keys, we can't check for anything
# else
print (("Cache lifetime is greater than %d minutes. Try again with a"+
" higher 'minutes' value") % minutes)
return None
def monitor_cache_lifetime(minutes, retest = 10, ntest = -1,
cache_name = "memcaches", verbose = False):
# list of list of active memcache test keys
keys = []
period = 60 # 1 minute cycle time
data = DataLogger()
# we'll create an independent connection to memcached for this test
mc = Memcache(getattr(g, cache_name))
counter = 0
while ntest:
if counter == 0 or (retest and counter % retest == 0):
randstr = random.random()
newkeys = [("%s_lifetime_%s_%d" % (cache_name, randstr, x), x+1)
for x in xrange(minutes)]
# set N keys, and tell them not to live for longer than this test
mc.set_multi(dict(newkeys),
#time = minutes * period)
time = (minutes+1) * period)
# add the list in reverse order since we'll be poping.
newkeys.reverse()
keys.append(newkeys)
# wait for the next key to (potentially) expire
counter += 1
time.sleep(period)
for k in keys:
key, age = k.pop()
if mc.get(key) is None or k == []:
if verbose:
print "cache expiration: %d seconds" % (period * age)
data.add(period * age)
AppServiceMonitor.set_cache_lifetime(data, key = cache_name)
# wipe out the list for removal by the subsequent filter
while k: k.pop()
# clear out any empty key lists
if [] in keys:
keys = filter(None, keys)
ntest -= 1
def monitor_cache_lifetime(minutes,
retest=10,
ntest=-1,
cache_name="memcaches",
verbose=False):
# list of list of active memcache test keys
keys = []
period = 60 # 1 minute cycle time
data = DataLogger()
# we'll create an independent connection to memcached for this test
mc = Memcache(getattr(g, cache_name))
counter = 0
while ntest:
if counter == 0 or (retest and counter % retest == 0):
randstr = random.random()
newkeys = [("%s_lifetime_%s_%d" % (cache_name, randstr, x), x + 1)
for x in xrange(minutes)]
# set N keys, and tell them not to live for longer than this test
mc.set_multi(
dict(newkeys),
#time = minutes * period)
time=(minutes + 1) * period)
# add the list in reverse order since we'll be poping.
newkeys.reverse()
keys.append(newkeys)
# wait for the next key to (potentially) expire
counter += 1
time.sleep(period)
for k in keys:
key, age = k.pop()
if mc.get(key) is None or k == []:
if verbose:
print "cache expiration: %d seconds" % (period * age)
data.add(period * age)
AppServiceMonitor.set_cache_lifetime(data, key=cache_name)
# wipe out the list for removal by the subsequent filter
while k:
k.pop()
# clear out any empty key lists
if [] in keys:
keys = filter(None, keys)
ntest -= 1
def __init__(self, output_db):
"""Initialise with path to output SQLite DB file"""
# If the output file exists, delete it so that the db is
# created from scratch
if os.path.exists(output_db):
os.unlink(output_db)
self.db = create_engine("sqlite:///%s" % output_db)
# Python's encoding handling is reallly annoying
# http://stackoverflow.com/questions/3033741/sqlalchemy-automatically-converts-str-to-unicode-on-commit
self.db.raw_connection().connection.text_factory = str
self.init_db()
pylons.g.cache = CacheChain(
(SelfEmptyingCache(max_size=1000), Memcache(pylons.g.memcaches)))
import os
import sys
from datetime import datetime
# XXX: We have to cap the cache size to stop the export from eating
# all available memory. We also have to adjust the cache before
# importing r2.lib.db.thing, which takes its own reference to whatever
# cache is currently set.
import pylons
from r2.lib.cache import Memcache, SelfEmptyingCache, CacheChain
pylons.g.cache = CacheChain((SelfEmptyingCache(max_size=1000), Memcache(pylons.g.memcaches)))
from r2.lib.db import tdb_sql as tdb
from r2.lib.db.thing import NotFound, Relation
from r2.models import Link, Comment, Account, Vote, Subreddit
from sqlalchemy import *
class Exporter:
def __init__(self, output_db):
"""Initialise with path to output SQLite DB file"""
# If the output file exists, delete it so that the db is
# created from scratch
if os.path.exists(output_db):
os.unlink(output_db)
self.db = create_engine("sqlite:///%s" % output_db)
# Python's encoding handling is reallly annoying
# http://stackoverflow.com/questions/3033741/sqlalchemy-automatically-converts-str-to-unicode-on-commit
self.db.raw_connection().connection.text_factory = str
def __init__(self, global_conf, app_conf, paths, **extra):
"""
Globals acts as a container for objects available throughout
the life of the application.
One instance of Globals is created by Pylons during
application initialization and is available during requests
via the 'g' variable.
``global_conf``
The same variable used throughout ``config/middleware.py``
namely, the variables from the ``[DEFAULT]`` section of the
configuration file.
``app_conf``
The same ``kw`` dictionary used throughout
``config/middleware.py`` namely, the variables from the
section in the config file for your application.
``extra``
The configuration returned from ``load_config`` in
``config/middleware.py`` which may be of use in the setup of
your global variables.
"""
def to_bool(x):
return (x.lower() == 'true') if x else None
def to_iter(name, delim = ','):
return (x.strip() for x in global_conf.get(name, '').split(delim))
# slop over all variables to start with
for k, v in global_conf.iteritems():
if not k.startswith("_") and not hasattr(self, k):
if k in self.int_props:
v = int(v)
elif k in self.bool_props:
v = to_bool(v)
elif k in self.tuple_props:
v = tuple(to_iter(k))
setattr(self, k, v)
# initialize caches
mc = Memcache(self.memcaches, debug=True)
self.cache = CacheChain((LocalCache(), mc))
self.permacache = Memcache(self.permacaches, debug=True)
self.rendercache = Memcache(self.rendercaches, debug=True)
self.make_lock = make_lock_factory(mc)
self.rec_cache = Memcache(self.rec_cache, debug=True)
# set default time zone if one is not set
self.tz = pytz.timezone(global_conf.get('timezone'))
#make a query cache
self.stats_collector = QueryStats()
# set the modwindow
self.MODWINDOW = timedelta(self.MODWINDOW)
self.REDDIT_MAIN = bool(os.environ.get('REDDIT_MAIN'))
# turn on for language support
self.languages, self.lang_name = _get_languages()
all_languages = self.lang_name.keys()
all_languages.sort()
self.all_languages = all_languages
# load the md5 hashes of files under static
static_files = os.path.join(paths.get('static_files'), 'static')
self.static_md5 = {}
if os.path.exists(static_files):
for f in os.listdir(static_files):
if f.endswith('.md5'):
key = f[0:-4]
f = os.path.join(static_files, f)
with open(f, 'r') as handle:
md5 = handle.read().strip('\n')
self.static_md5[key] = md5
#set up the logging directory
log_path = self.log_path
process_iden = global_conf.get('scgi_port', 'default')
if log_path:
if not os.path.exists(log_path):
os.makedirs(log_path)
for fname in os.listdir(log_path):
if fname.startswith(process_iden):
full_name = os.path.join(log_path, fname)
os.remove(full_name)
#setup the logger
self.log = logging.getLogger('reddit')
self.log.addHandler(logging.StreamHandler())
if self.debug:
self.log.setLevel(logging.DEBUG)
#read in our CSS so that it can become a default for subreddit
#stylesheets
stylesheet_path = os.path.join(paths.get('static_files'),
self.static_path.lstrip('/'),
self.stylesheet)
with open(stylesheet_path) as s:
self.default_stylesheet = s.read()
self.reddit_host = socket.gethostname()
self.reddit_pid = os.getpid()
