def bench_cache_lifetime(minutes): "Attempts to find how long a given memcached key can be expected to live" from pylons import g from r2.lib.cache import Memcache # we'll create an independent connection to memcached for this # test mc = Memcache(g.memcaches) # set N keys, and tell them not to live for longer than this test mc.set_multi(dict(('bench_cache_%d' % x, x) for x in xrange(minutes)), time=minutes * 60) # and every minute, check to see that the keys are still present, # until we find one missing for x in xrange(minutes): if mc.get('bench_cache_%d' % x, None) is not None: sleep(60) else: # we found one missing return x - 1 else: # we're out of numbers, and we didn't find any missing # keys. Since we only set N keys, we can't check for anything # else print(("Cache lifetime is greater than %d minutes. Try again with a" + " higher 'minutes' value") % minutes) return None
def bench_cache_lifetime(minutes): "Attempts to find how long a given memcached key can be expected to live" from pylons import g from r2.lib.cache import Memcache # we'll create an independent connection to memcached for this # test mc = Memcache(g.memcaches) # set N keys, and tell them not to live for longer than this test mc.set_multi(dict( ('bench_cache_%d' % x, x) for x in xrange(minutes) ), time=minutes*60) # and every minute, check to see that the keys are still present, # until we find one missing for x in xrange(minutes): if mc.get('bench_cache_%d' % x, None) is not None: sleep(60) else: # we found one missing return x-1 else: # we're out of numbers, and we didn't find any missing # keys. Since we only set N keys, we can't check for anything # else print (("Cache lifetime is greater than %d minutes. Try again with a"+ " higher 'minutes' value") % minutes) return None
def monitor_cache_lifetime(minutes, retest = 10, ntest = -1, cache_name = "memcaches", verbose = False): # list of list of active memcache test keys keys = [] period = 60 # 1 minute cycle time data = DataLogger() # we'll create an independent connection to memcached for this test mc = Memcache(getattr(g, cache_name)) counter = 0 while ntest: if counter == 0 or (retest and counter % retest == 0): randstr = random.random() newkeys = [("%s_lifetime_%s_%d" % (cache_name, randstr, x), x+1) for x in xrange(minutes)] # set N keys, and tell them not to live for longer than this test mc.set_multi(dict(newkeys), #time = minutes * period) time = (minutes+1) * period) # add the list in reverse order since we'll be poping. newkeys.reverse() keys.append(newkeys) # wait for the next key to (potentially) expire counter += 1 time.sleep(period) for k in keys: key, age = k.pop() if mc.get(key) is None or k == []: if verbose: print "cache expiration: %d seconds" % (period * age) data.add(period * age) AppServiceMonitor.set_cache_lifetime(data, key = cache_name) # wipe out the list for removal by the subsequent filter while k: k.pop() # clear out any empty key lists if [] in keys: keys = filter(None, keys) ntest -= 1
def monitor_cache_lifetime(minutes, retest=10, ntest=-1, cache_name="memcaches", verbose=False): # list of list of active memcache test keys keys = [] period = 60 # 1 minute cycle time data = DataLogger() # we'll create an independent connection to memcached for this test mc = Memcache(getattr(g, cache_name)) counter = 0 while ntest: if counter == 0 or (retest and counter % retest == 0): randstr = random.random() newkeys = [("%s_lifetime_%s_%d" % (cache_name, randstr, x), x + 1) for x in xrange(minutes)] # set N keys, and tell them not to live for longer than this test mc.set_multi( dict(newkeys), #time = minutes * period) time=(minutes + 1) * period) # add the list in reverse order since we'll be poping. newkeys.reverse() keys.append(newkeys) # wait for the next key to (potentially) expire counter += 1 time.sleep(period) for k in keys: key, age = k.pop() if mc.get(key) is None or k == []: if verbose: print "cache expiration: %d seconds" % (period * age) data.add(period * age) AppServiceMonitor.set_cache_lifetime(data, key=cache_name) # wipe out the list for removal by the subsequent filter while k: k.pop() # clear out any empty key lists if [] in keys: keys = filter(None, keys) ntest -= 1
def __init__(self, output_db): """Initialise with path to output SQLite DB file""" # If the output file exists, delete it so that the db is # created from scratch if os.path.exists(output_db): os.unlink(output_db) self.db = create_engine("sqlite:///%s" % output_db) # Python's encoding handling is reallly annoying # http://stackoverflow.com/questions/3033741/sqlalchemy-automatically-converts-str-to-unicode-on-commit self.db.raw_connection().connection.text_factory = str self.init_db() pylons.g.cache = CacheChain( (SelfEmptyingCache(max_size=1000), Memcache(pylons.g.memcaches)))
import os import sys from datetime import datetime # XXX: We have to cap the cache size to stop the export from eating # all available memory. We also have to adjust the cache before # importing r2.lib.db.thing, which takes its own reference to whatever # cache is currently set. import pylons from r2.lib.cache import Memcache, SelfEmptyingCache, CacheChain pylons.g.cache = CacheChain((SelfEmptyingCache(max_size=1000), Memcache(pylons.g.memcaches))) from r2.lib.db import tdb_sql as tdb from r2.lib.db.thing import NotFound, Relation from r2.models import Link, Comment, Account, Vote, Subreddit from sqlalchemy import * class Exporter: def __init__(self, output_db): """Initialise with path to output SQLite DB file""" # If the output file exists, delete it so that the db is # created from scratch if os.path.exists(output_db): os.unlink(output_db) self.db = create_engine("sqlite:///%s" % output_db) # Python's encoding handling is reallly annoying # http://stackoverflow.com/questions/3033741/sqlalchemy-automatically-converts-str-to-unicode-on-commit self.db.raw_connection().connection.text_factory = str
def __init__(self, global_conf, app_conf, paths, **extra): """ Globals acts as a container for objects available throughout the life of the application. One instance of Globals is created by Pylons during application initialization and is available during requests via the 'g' variable. ``global_conf`` The same variable used throughout ``config/middleware.py`` namely, the variables from the ``[DEFAULT]`` section of the configuration file. ``app_conf`` The same ``kw`` dictionary used throughout ``config/middleware.py`` namely, the variables from the section in the config file for your application. ``extra`` The configuration returned from ``load_config`` in ``config/middleware.py`` which may be of use in the setup of your global variables. """ def to_bool(x): return (x.lower() == 'true') if x else None def to_iter(name, delim = ','): return (x.strip() for x in global_conf.get(name, '').split(delim)) # slop over all variables to start with for k, v in global_conf.iteritems(): if not k.startswith("_") and not hasattr(self, k): if k in self.int_props: v = int(v) elif k in self.bool_props: v = to_bool(v) elif k in self.tuple_props: v = tuple(to_iter(k)) setattr(self, k, v) # initialize caches mc = Memcache(self.memcaches, debug=True) self.cache = CacheChain((LocalCache(), mc)) self.permacache = Memcache(self.permacaches, debug=True) self.rendercache = Memcache(self.rendercaches, debug=True) self.make_lock = make_lock_factory(mc) self.rec_cache = Memcache(self.rec_cache, debug=True) # set default time zone if one is not set self.tz = pytz.timezone(global_conf.get('timezone')) #make a query cache self.stats_collector = QueryStats() # set the modwindow self.MODWINDOW = timedelta(self.MODWINDOW) self.REDDIT_MAIN = bool(os.environ.get('REDDIT_MAIN')) # turn on for language support self.languages, self.lang_name = _get_languages() all_languages = self.lang_name.keys() all_languages.sort() self.all_languages = all_languages # load the md5 hashes of files under static static_files = os.path.join(paths.get('static_files'), 'static') self.static_md5 = {} if os.path.exists(static_files): for f in os.listdir(static_files): if f.endswith('.md5'): key = f[0:-4] f = os.path.join(static_files, f) with open(f, 'r') as handle: md5 = handle.read().strip('\n') self.static_md5[key] = md5 #set up the logging directory log_path = self.log_path process_iden = global_conf.get('scgi_port', 'default') if log_path: if not os.path.exists(log_path): os.makedirs(log_path) for fname in os.listdir(log_path): if fname.startswith(process_iden): full_name = os.path.join(log_path, fname) os.remove(full_name) #setup the logger self.log = logging.getLogger('reddit') self.log.addHandler(logging.StreamHandler()) if self.debug: self.log.setLevel(logging.DEBUG) #read in our CSS so that it can become a default for subreddit #stylesheets stylesheet_path = os.path.join(paths.get('static_files'), self.static_path.lstrip('/'), self.stylesheet) with open(stylesheet_path) as s: self.default_stylesheet = s.read() self.reddit_host = socket.gethostname() self.reddit_pid = os.getpid()