def setUp(self):
credentials = {'username': '******', 'password': '******'}
self.client = connect_thread_local('Keyspace1', credentials=credentials)
self.cf = ColumnFamily(self.client, 'Standard2')
self.cf_long = ColumnFamily(self.client, 'StdLong')
self.cf_int = ColumnFamily(self.client, 'StdInteger')
self.cf_time = ColumnFamily(self.client, 'StdTimeUUID')
self.cf_lex = ColumnFamily(self.client, 'StdLexicalUUID')
self.cf_ascii = ColumnFamily(self.client, 'StdAscii')
self.cf_utf8 = ColumnFamily(self.client, 'StdUTF8')
self.cf_bytes = ColumnFamily(self.client, 'StdBytes')
self.cf_suplong = ColumnFamily(self.client, 'SuperLong', super=True)
self.cf_supint = ColumnFamily(self.client, 'SuperInt', super=True)
self.cf_suptime = ColumnFamily(self.client, 'SuperTime', super=True)
self.cf_suplex = ColumnFamily(self.client, 'SuperLex', super=True)
self.cf_supascii = ColumnFamily(self.client, 'SuperAscii', super=True)
self.cf_suputf8 = ColumnFamily(self.client, 'SuperUTF8', super=True)
self.cf_supbytes = ColumnFamily(self.client, 'SuperBytes', super=True)
self.cf_suplong_sublong = ColumnFamily(self.client, 'SuperLongSubLong', super=True)
self.cf_suplong_subint = ColumnFamily(self.client, 'SuperLongSubInt', super=True)
self.cf_suplong_subtime = ColumnFamily(self.client, 'SuperLongSubTime', super=True)
self.cf_suplong_sublex = ColumnFamily(self.client, 'SuperLongSubLex', super=True)
self.cf_suplong_subascii = ColumnFamily(self.client, 'SuperLongSubAscii', super=True)
self.cf_suplong_subutf8 = ColumnFamily(self.client, 'SuperLongSubUTF8', super=True)
self.cf_suplong_subbytes = ColumnFamily(self.client, 'SuperLongSubBytes', super=True)
self.cf_valid_long = ColumnFamily(self.client, 'ValidatorLong')
self.cf_valid_int = ColumnFamily(self.client, 'ValidatorInt')
self.cf_valid_time = ColumnFamily(self.client, 'ValidatorTime')
self.cf_valid_lex = ColumnFamily(self.client, 'ValidatorLex')
self.cf_valid_ascii = ColumnFamily(self.client, 'ValidatorAscii')
self.cf_valid_utf8 = ColumnFamily(self.client, 'ValidatorUTF8')
self.cf_valid_bytes = ColumnFamily(self.client, 'ValidatorBytes')
self.cfs = [self.cf_long, self.cf_int, self.cf_time, self.cf_lex,
self.cf_ascii, self.cf_utf8, self.cf_bytes,
self.cf_suplong, self.cf_supint, self.cf_suptime,
self.cf_suplex, self.cf_supascii, self.cf_suputf8,
self.cf_supbytes,
self.cf_suplong_subint, self.cf_suplong_subint,
self.cf_suplong_subtime, self.cf_suplong_sublex,
self.cf_suplong_subascii, self.cf_suplong_subutf8,
self.cf_suplong_subbytes,
self.cf_valid_long, self.cf_valid_int, self.cf_valid_time,
self.cf_valid_lex, self.cf_valid_ascii, self.cf_valid_utf8,
self.cf_valid_bytes]
try:
self.timestamp_n = int(self.cf.get('meta')['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def test_connections():
version = connect("Keyspace1").describe_version()
thread_local = connect_thread_local("Keyspace1")
threads = []
for i in xrange(10):
threads.append(threading.Thread(target=version_check, args=(thread_local, version)))
threads[-1].start()
for thread in threads:
thread.join()
def test_connections():
version = connect().get_string_property('version')
thread_local = connect_thread_local()
threads = []
for i in xrange(10):
threads.append(threading.Thread(target=version_check,
args=(thread_local, version)))
threads[-1].start()
for thread in threads:
thread.join()
def __init__(self, keyspace, column_family, seeds,
read_consistency_level = CL_ONE,
write_consistency_level = CL_ONE):
self.keyspace = keyspace
self.column_family = column_family
self.seeds = seeds
self.client = pycassa.connect_thread_local(seeds)
self.cf = pycassa.ColumnFamily(self.client, self.keyspace,
self.column_family,
read_consistency_level = read_consistency_level,
write_consistency_level = write_consistency_level)
def __init__(self, keyspace, column_family, seeds,
read_consistency_level = CL_QUORUM,
write_consistency_level = CL_QUORUM):
self.keyspace = keyspace
self.column_family = column_family
self.seeds = seeds
self.client = pycassa.connect_thread_local(seeds)
self.cf = pycassa.ColumnFamily(self.client, self.keyspace,
self.column_family,
read_consistency_level = read_consistency_level,
write_consistency_level = write_consistency_level)
def test_connections():
version = connect().get_string_property('version')
thread_local = connect_thread_local()
threads = []
for i in xrange(10):
threads.append(
threading.Thread(target=version_check,
args=(thread_local, version)))
threads[-1].start()
for thread in threads:
thread.join()
def setUp(self):
credentials = {'username': '******', 'password': '******'}
self.client = connect_thread_local('Keyspace1', credentials=credentials)
self.cf = ColumnFamily(self.client, 'Super2',
write_consistency_level=ConsistencyLevel.ONE,
buffer_size=2, timestamp=self.timestamp,
super=True)
try:
self.timestamp_n = int(self.cf.get('meta')['meta']['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def setUp(self):
self.client = connect_thread_local()
self.client.login('Keyspace1', {'username': '******', 'password': '******'})
self.cf = ColumnFamily(self.client, 'Keyspace1', 'Super2',
write_consistency_level=ConsistencyLevel.ONE,
buffer_size=2, timestamp=self.timestamp,
super=True)
try:
self.timestamp_n = int(self.cf.get('meta')['meta']['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def setUp(self):
credentials = {'username': '******', 'password': '******'}
self.client = connect_thread_local('Keyspace1', credentials=credentials)
self.cf = ColumnFamily(self.client, 'Super2',
write_consistency_level=ConsistencyLevel.ONE,
timestamp=self.timestamp,
super=True)
self.map = ColumnFamilyMap(TestUTF8, self.cf)
try:
self.timestamp_n = int(self.cf.get('meta')['meta']['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def setUp(self):
self.client = connect_thread_local()
self.client.login('Keyspace1', {'username': '******', 'password': '******'})
self.cf = ColumnFamily(self.client, 'Keyspace1', 'Super2',
write_consistency_level=ConsistencyLevel.ONE,
timestamp=self.timestamp,
super=True)
self.map = ColumnFamilyMap(TestUTF8, self.cf)
try:
self.timestamp_n = int(self.cf.get('meta')['meta']['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def test_connections(self):
def version_check(connection, version):
assert connection.describe_version() == version
version = connect('Keyspace1').describe_version()
thread_local = connect_thread_local('Keyspace1')
threads = []
for i in xrange(10):
threads.append(threading.Thread(target=version_check,
args=(thread_local, version)))
threads[-1].start()
for thread in threads:
thread.join()
def __init__(self, init_args):
import pycassa
import cassandra.ttypes
seed, keyspace, column_family,seen_cf = init_args.split(',')
self.seeds = [seed]
self.keyspace = keyspace
self.column_family = column_family
self.client = pycassa.connect_thread_local(self.seeds)
self.cf = pycassa.ColumnFamily(self.client, self.keyspace,
self.column_family)
self.seen_cf = pycassa.ColumnFamily(self.client, self.keyspace,
seen_cf)
def test_connections(self):
def version_check(connection, version):
assert connection.describe_version() == version
version = connect('Keyspace1').describe_version()
thread_local = connect_thread_local('Keyspace1')
threads = []
for i in xrange(10):
threads.append(
threading.Thread(target=version_check,
args=(thread_local, version)))
threads[-1].start()
for thread in threads:
thread.join()
def get_client(keyspace):
config = settings.get_config()
logger.info("get_client: servers: %s, timeout: %s, user: %s, pass: %s" % (
config.servers,
config.connection_timeout,
config.username,
config.password))
client = pycassa.connect_thread_local(
config.servers,
timeout=config.connection_timeout
)
cred = dict(username=config.username, password=config.password)
client.login(keyspace, cred)
logger.info("get_client => %s" % repr(client))
return client
def setUp(self):
credentials = {'username': '******', 'password': '******'}
self.client = connect_thread_local('Keyspace1',
credentials=credentials)
self.cf = ColumnFamily(self.client, 'Standard2')
self.cf_long = ColumnFamily(self.client, 'StdLong')
self.cf_int = ColumnFamily(self.client, 'StdInteger')
self.cf_time = ColumnFamily(self.client, 'StdTimeUUID')
self.cf_lex = ColumnFamily(self.client, 'StdLexicalUUID')
self.cf_ascii = ColumnFamily(self.client, 'StdAscii')
self.cf_utf8 = ColumnFamily(self.client, 'StdUTF8')
self.cf_bytes = ColumnFamily(self.client, 'StdBytes')
self.cf_suplong = ColumnFamily(self.client, 'SuperLong', super=True)
self.cf_supint = ColumnFamily(self.client, 'SuperInt', super=True)
self.cf_suptime = ColumnFamily(self.client, 'SuperTime', super=True)
self.cf_suplex = ColumnFamily(self.client, 'SuperLex', super=True)
self.cf_supascii = ColumnFamily(self.client, 'SuperAscii', super=True)
self.cf_suputf8 = ColumnFamily(self.client, 'SuperUTF8', super=True)
self.cf_supbytes = ColumnFamily(self.client, 'SuperBytes', super=True)
self.cf_suplong_sublong = ColumnFamily(self.client,
'SuperLongSubLong',
super=True)
self.cf_suplong_subint = ColumnFamily(self.client,
'SuperLongSubInt',
super=True)
self.cf_suplong_subtime = ColumnFamily(self.client,
'SuperLongSubTime',
super=True)
self.cf_suplong_sublex = ColumnFamily(self.client,
'SuperLongSubLex',
super=True)
self.cf_suplong_subascii = ColumnFamily(self.client,
'SuperLongSubAscii',
super=True)
self.cf_suplong_subutf8 = ColumnFamily(self.client,
'SuperLongSubUTF8',
super=True)
self.cf_suplong_subbytes = ColumnFamily(self.client,
'SuperLongSubBytes',
super=True)
self.cf_valid_long = ColumnFamily(self.client, 'ValidatorLong')
self.cf_valid_int = ColumnFamily(self.client, 'ValidatorInt')
self.cf_valid_time = ColumnFamily(self.client, 'ValidatorTime')
self.cf_valid_lex = ColumnFamily(self.client, 'ValidatorLex')
self.cf_valid_ascii = ColumnFamily(self.client, 'ValidatorAscii')
self.cf_valid_utf8 = ColumnFamily(self.client, 'ValidatorUTF8')
self.cf_valid_bytes = ColumnFamily(self.client, 'ValidatorBytes')
self.cf_def_valid = ColumnFamily(self.client, 'DefaultValidator')
self.cfs = [
self.cf_long,
self.cf_int,
self.cf_time,
self.cf_lex,
self.cf_ascii,
self.cf_utf8,
self.cf_bytes,
#
self.cf_suplong,
self.cf_supint,
self.cf_suptime,
self.cf_suplex,
self.cf_supascii,
self.cf_suputf8,
self.cf_supbytes,
#
self.cf_suplong_subint,
self.cf_suplong_subint,
self.cf_suplong_subtime,
self.cf_suplong_sublex,
self.cf_suplong_subascii,
self.cf_suplong_subutf8,
self.cf_suplong_subbytes,
#
self.cf_valid_long,
self.cf_valid_int,
self.cf_valid_time,
self.cf_valid_lex,
self.cf_valid_ascii,
self.cf_valid_utf8,
self.cf_valid_bytes,
#
self.cf_def_valid,
]
try:
self.timestamp_n = int(self.cf.get('meta')['timestamp'])
except NotFoundException:
self.timestamp_n = 0
self.clear()
def __init__(self, global_conf, app_conf, paths, **extra):
"""
Globals acts as a container for objects available throughout
the life of the application.
One instance of Globals is created by Pylons during
application initialization and is available during requests
via the 'g' variable.
``global_conf``
The same variable used throughout ``config/middleware.py``
namely, the variables from the ``[DEFAULT]`` section of the
configuration file.
``app_conf``
The same ``kw`` dictionary used throughout
``config/middleware.py`` namely, the variables from the
section in the config file for your application.
``extra``
The configuration returned from ``load_config`` in
``config/middleware.py`` which may be of use in the setup of
your global variables.
"""
# slop over all variables to start with
for k, v in global_conf.iteritems():
if not k.startswith("_") and not hasattr(self, k):
if k in self.int_props:
v = int(v)
elif k in self.float_props:
v = float(v)
elif k in self.bool_props:
v = self.to_bool(v)
elif k in self.tuple_props:
v = tuple(self.to_iter(v))
elif k in self.choice_props:
if v not in self.choice_props[k]:
raise ValueError("Unknown option for %r: %r not in %r"
% (k, v, self.choice_props[k]))
v = self.choice_props[k][v]
setattr(self, k, v)
self.running_as_script = global_conf.get('running_as_script', False)
# initialize caches. Any cache-chains built here must be added
# to cache_chains (closed around by reset_caches) so that they
# can properly reset their local components
localcache_cls = (SelfEmptyingCache if self.running_as_script
else LocalCache)
num_mc_clients = self.num_mc_clients
self.cache_chains = []
self.memcache = CMemcache(self.memcaches, num_clients = num_mc_clients)
self.make_lock = make_lock_factory(self.memcache)
if not self.cassandra_seeds:
raise ValueError("cassandra_seeds not set in the .ini")
if not self.url_seeds:
raise ValueError("url_seeds not set in the .ini")
self.cassandra_seeds = list(self.cassandra_seeds)
random.shuffle(self.cassandra_seeds)
self.cassandra = pycassa.connect_thread_local(self.cassandra_seeds)
perma_memcache = (CMemcache(self.permacache_memcaches, num_clients = num_mc_clients)
if self.permacache_memcaches
else None)
self.permacache = self.init_cass_cache('permacache', 'permacache',
self.cassandra,
self.make_lock,
memcache = perma_memcache,
read_consistency_level = self.cassandra_rcl,
write_consistency_level = self.cassandra_wcl,
localcache_cls = localcache_cls)
self.cache_chains.append(self.permacache)
self.url_seeds = list(self.url_seeds)
random.shuffle(self.url_seeds)
self.url_cassandra = pycassa.connect_thread_local(self.url_seeds)
self.urlcache = self.init_cass_cache('urls', 'urls',
self.url_cassandra,
self.make_lock,
# until we've merged this
# with the regular
# cluster, this will
# always be CL_ONE
read_consistency_level = CL_ONE,
write_consistency_level = CL_ONE,
localcache_cls = localcache_cls)
self.cache_chains.append(self.urlcache)
# hardcache is done after the db info is loaded, and then the
# chains are reset to use the appropriate initial entries
self.cache = MemcacheChain((localcache_cls(), self.memcache))
self.cache_chains.append(self.cache)
self.rendercache = MemcacheChain((localcache_cls(),
CMemcache(self.rendercaches,
noreply=True, no_block=True,
num_clients = num_mc_clients)))
self.cache_chains.append(self.rendercache)
self.servicecache = MemcacheChain((localcache_cls(),
CMemcache(self.servicecaches,
num_clients = num_mc_clients)))
self.cache_chains.append(self.servicecache)
self.thing_cache = CacheChain((localcache_cls(),))
self.cache_chains.append(self.thing_cache)
# set default time zone if one is not set
tz = global_conf.get('timezone')
dtz = global_conf.get('display_timezone', tz)
self.tz = pytz.timezone(tz)
self.display_tz = pytz.timezone(dtz)
#load the database info
self.dbm = self.load_db_params(global_conf)
# can't do this until load_db_params() has been called
self.hardcache = HardcacheChain((localcache_cls(),
self.memcache,
HardCache(self)),
cache_negative_results = True)
self.cache_chains.append(self.hardcache)
# I know this sucks, but we need non-request-threads to be
# able to reset the caches, so we need them be able to close
# around 'cache_chains' without being able to call getattr on
# 'g'
cache_chains = self.cache_chains[::]
def reset_caches():
for chain in cache_chains:
chain.reset()
self.reset_caches = reset_caches
self.reset_caches()
#make a query cache
self.stats_collector = QueryStats()
# set the modwindow
self.MODWINDOW = timedelta(self.MODWINDOW)
self.REDDIT_MAIN = bool(os.environ.get('REDDIT_MAIN'))
# turn on for language support
self.languages, self.lang_name = \
get_active_langs(default_lang= self.lang)
all_languages = self.lang_name.keys()
all_languages.sort()
self.all_languages = all_languages
self.paths = paths
# load the md5 hashes of files under static
static_files = os.path.join(paths.get('static_files'), 'static')
self.static_md5 = {}
if os.path.exists(static_files):
for f in os.listdir(static_files):
if f.endswith('.md5'):
key = f[0:-4]
f = os.path.join(static_files, f)
with open(f, 'r') as handle:
md5 = handle.read().strip('\n')
self.static_md5[key] = md5
#set up the logging directory
log_path = self.log_path
process_iden = global_conf.get('scgi_port', 'default')
self.reddit_port = process_iden
if log_path:
if not os.path.exists(log_path):
os.makedirs(log_path)
for fname in os.listdir(log_path):
if fname.startswith(process_iden):
full_name = os.path.join(log_path, fname)
os.remove(full_name)
#setup the logger
self.log = logging.getLogger('reddit')
self.log.addHandler(logging.StreamHandler())
if self.debug:
self.log.setLevel(logging.DEBUG)
else:
self.log.setLevel(logging.INFO)
# set log level for pycountry which is chatty
logging.getLogger('pycountry.db').setLevel(logging.CRITICAL)
if not self.media_domain:
self.media_domain = self.domain
#if self.media_domain == self.domain:
#print ("Warning: g.media_domain == g.domain. " +
# "This may give untrusted content access to user cookies")
#read in our CSS so that it can become a default for subreddit
#stylesheets
stylesheet_path = os.path.join(paths.get('static_files'),
self.static_path.lstrip('/'),
self.stylesheet)
with open(stylesheet_path) as s:
self.default_stylesheet = s.read()
self.profanities = None
if self.profanity_wordlist and os.path.exists(self.profanity_wordlist):
with open(self.profanity_wordlist, 'r') as handle:
words = []
for line in handle:
words.append(line.strip(' \n\r'))
if words:
self.profanities = re.compile(r"\b(%s)\b" % '|'.join(words),
re.I | re.U)
self.reddit_host = socket.gethostname()
self.reddit_pid = os.getpid()
#the shutdown toggle
self.shutdown = False
#if we're going to use the query_queue, we need amqp
if self.write_query_queue and not self.amqp_host:
raise Exception("amqp_host must be defined to use the query queue")
# This requirement doesn't *have* to be a requirement, but there are
# bugs at the moment that will pop up if you violate it
if self.write_query_queue and not self.use_query_cache:
raise Exception("write_query_queue requires use_query_cache")
# try to set the source control revision number
try:
popen = subprocess.Popen(["git", "log", "--date=short",
"--pretty=format:%H %h", '-n1'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
resp, stderrdata = popen.communicate()
resp = resp.strip().split(' ')
self.version, self.short_version = resp
except object, e:
self.log.info("Couldn't read source revision (%r)" % e)
self.version = self.short_version = '(unknown)'
import struct
import time
from odict import OrderedDict
import pycassa
from cassandra.ttypes import NotFoundException
__all__ = ['get_user_by_id', 'get_user_by_username', 'get_friend_ids',
'get_follower_ids', 'get_users_for_user_ids', 'get_friends',
'get_followers', 'get_timeline', 'get_userline', 'get_tweet', 'save_user',
'save_tweet', 'add_friends', 'remove_friends', 'DatabaseError',
'NotFound', 'InvalidDictionary', 'PUBLIC_USERLINE_KEY']
CLIENT = pycassa.connect_thread_local(framed_transport=True)
USER = pycassa.ColumnFamily(CLIENT, 'Twissandra', 'User',
dict_class=OrderedDict)
USERNAME = pycassa.ColumnFamily(CLIENT, 'Twissandra', 'Username',
dict_class=OrderedDict)
FRIENDS = pycassa.ColumnFamily(CLIENT, 'Twissandra', 'Friends',
dict_class=OrderedDict)
FOLLOWERS = pycassa.ColumnFamily(CLIENT, 'Twissandra', 'Followers',
dict_class=OrderedDict)
TWEET = pycassa.ColumnFamily(CLIENT, 'Twissandra', 'Tweet',
dict_class=OrderedDict)
TIMELINE = pycassa.ColumnFamily(CLIENT, 'Twissandra', 'Timeline',
dict_class=OrderedDict)
USERLINE = pycassa.ColumnFamily(CLIENT, 'Twissandra', 'Userline',
dict_class=OrderedDict)
import time
from ordereddict import OrderedDict
import pycassa
from pycassa.cassandra.ttypes import NotFoundException
__all__ = ['get_user_by_username', 'get_friend_usernames',
'get_follower_usernames', 'get_users_for_usernames', 'get_friends',
'get_followers', 'get_timeline', 'get_userline', 'get_tweet', 'save_user',
'save_tweet', 'add_friends', 'remove_friends', 'DatabaseError',
'NotFound', 'InvalidDictionary', 'PUBLIC_USERLINE_KEY']
CLIENT = pycassa.connect_thread_local('Twissandra')
USER = pycassa.ColumnFamily(CLIENT, 'User', dict_class=OrderedDict)
FRIENDS = pycassa.ColumnFamily(CLIENT, 'Friends', dict_class=OrderedDict)
FOLLOWERS = pycassa.ColumnFamily(CLIENT, 'Followers', dict_class=OrderedDict)
TWEET = pycassa.ColumnFamily(CLIENT, 'Tweet', dict_class=OrderedDict)
TIMELINE = pycassa.ColumnFamily(CLIENT, 'Timeline', dict_class=OrderedDict)
USERLINE = pycassa.ColumnFamily(CLIENT, 'Userline', dict_class=OrderedDict)
# NOTE: Having a single userline key to store all of the public tweets is not
# scalable. Currently, Cassandra requires that an entire row (meaning
# every column under a given key) to be able to fit in memory. You can
# imagine that after a while, the entire public timeline would exceed
# available memory.
#
# The fix for this is to partition the timeline by time, so we could use
# a key like !PUBLIC!2010-04-01 to partition it per day. We could drill
def __init__(self, global_conf, app_conf, paths, **extra):
"""
Globals acts as a container for objects available throughout
the life of the application.
One instance of Globals is created by Pylons during
application initialization and is available during requests
via the 'g' variable.
``global_conf``
The same variable used throughout ``config/middleware.py``
namely, the variables from the ``[DEFAULT]`` section of the
configuration file.
``app_conf``
The same ``kw`` dictionary used throughout
``config/middleware.py`` namely, the variables from the
section in the config file for your application.
``extra``
The configuration returned from ``load_config`` in
``config/middleware.py`` which may be of use in the setup of
your global variables.
"""
# slop over all variables to start with
for k, v in global_conf.iteritems():
if not k.startswith("_") and not hasattr(self, k):
if k in self.int_props:
v = int(v)
elif k in self.float_props:
v = float(v)
elif k in self.bool_props:
v = self.to_bool(v)
elif k in self.tuple_props:
v = tuple(self.to_iter(v))
elif k in self.choice_props:
if v not in self.choice_props[k]:
raise ValueError(
"Unknown option for %r: %r not in %r" %
(k, v, self.choice_props[k]))
v = self.choice_props[k][v]
setattr(self, k, v)
self.running_as_script = global_conf.get('running_as_script', False)
# initialize caches. Any cache-chains built here must be added
# to cache_chains (closed around by reset_caches) so that they
# can properly reset their local components
localcache_cls = (SelfEmptyingCache
if self.running_as_script else LocalCache)
num_mc_clients = self.num_mc_clients
self.cache_chains = []
self.memcache = CMemcache(self.memcaches, num_clients=num_mc_clients)
self.make_lock = make_lock_factory(self.memcache)
if not self.cassandra_seeds:
raise ValueError("cassandra_seeds not set in the .ini")
if not self.url_seeds:
raise ValueError("url_seeds not set in the .ini")
self.cassandra_seeds = list(self.cassandra_seeds)
random.shuffle(self.cassandra_seeds)
self.cassandra = pycassa.connect_thread_local(self.cassandra_seeds)
perma_memcache = (CMemcache(self.permacache_memcaches,
num_clients=num_mc_clients)
if self.permacache_memcaches else None)
self.permacache = self.init_cass_cache(
'permacache',
'permacache',
self.cassandra,
self.make_lock,
memcache=perma_memcache,
read_consistency_level=self.cassandra_rcl,
write_consistency_level=self.cassandra_wcl,
localcache_cls=localcache_cls)
self.cache_chains.append(self.permacache)
self.url_seeds = list(self.url_seeds)
random.shuffle(self.url_seeds)
self.url_cassandra = pycassa.connect_thread_local(self.url_seeds)
self.urlcache = self.init_cass_cache(
'urls',
'urls',
self.url_cassandra,
self.make_lock,
# until we've merged this
# with the regular
# cluster, this will
# always be CL_ONE
read_consistency_level=CL_ONE,
write_consistency_level=CL_ONE,
localcache_cls=localcache_cls)
self.cache_chains.append(self.urlcache)
# hardcache is done after the db info is loaded, and then the
# chains are reset to use the appropriate initial entries
self.cache = MemcacheChain((localcache_cls(), self.memcache))
self.cache_chains.append(self.cache)
self.rendercache = MemcacheChain(
(localcache_cls(),
CMemcache(self.rendercaches,
noreply=True,
no_block=True,
num_clients=num_mc_clients)))
self.cache_chains.append(self.rendercache)
self.servicecache = MemcacheChain(
(localcache_cls(),
CMemcache(self.servicecaches, num_clients=num_mc_clients)))
self.cache_chains.append(self.servicecache)
self.thing_cache = CacheChain((localcache_cls(), ))
self.cache_chains.append(self.thing_cache)
# set default time zone if one is not set
tz = global_conf.get('timezone')
dtz = global_conf.get('display_timezone', tz)
self.tz = pytz.timezone(tz)
self.display_tz = pytz.timezone(dtz)
#load the database info
self.dbm = self.load_db_params(global_conf)
# can't do this until load_db_params() has been called
self.hardcache = HardcacheChain(
(localcache_cls(), self.memcache, HardCache(self)),
cache_negative_results=True)
self.cache_chains.append(self.hardcache)
# I know this sucks, but we need non-request-threads to be
# able to reset the caches, so we need them be able to close
# around 'cache_chains' without being able to call getattr on
# 'g'
cache_chains = self.cache_chains[::]
def reset_caches():
for chain in cache_chains:
chain.reset()
self.reset_caches = reset_caches
self.reset_caches()
#make a query cache
self.stats_collector = QueryStats()
# set the modwindow
self.MODWINDOW = timedelta(self.MODWINDOW)
self.REDDIT_MAIN = bool(os.environ.get('REDDIT_MAIN'))
# turn on for language support
self.languages, self.lang_name = \
get_active_langs(default_lang= self.lang)
all_languages = self.lang_name.keys()
all_languages.sort()
self.all_languages = all_languages
self.paths = paths
# load the md5 hashes of files under static
static_files = os.path.join(paths.get('static_files'), 'static')
self.static_md5 = {}
if os.path.exists(static_files):
for f in os.listdir(static_files):
if f.endswith('.md5'):
key = f[0:-4]
f = os.path.join(static_files, f)
with open(f, 'r') as handle:
md5 = handle.read().strip('\n')
self.static_md5[key] = md5
#set up the logging directory
log_path = self.log_path
process_iden = global_conf.get('scgi_port', 'default')
self.reddit_port = process_iden
if log_path:
if not os.path.exists(log_path):
os.makedirs(log_path)
for fname in os.listdir(log_path):
if fname.startswith(process_iden):
full_name = os.path.join(log_path, fname)
os.remove(full_name)
#setup the logger
self.log = logging.getLogger('reddit')
self.log.addHandler(logging.StreamHandler())
if self.debug:
self.log.setLevel(logging.DEBUG)
else:
self.log.setLevel(logging.INFO)
# set log level for pycountry which is chatty
logging.getLogger('pycountry.db').setLevel(logging.CRITICAL)
if not self.media_domain:
self.media_domain = self.domain
if self.media_domain == self.domain:
print("Warning: g.media_domain == g.domain. " +
"This may give untrusted content access to user cookies")
#read in our CSS so that it can become a default for subreddit
#stylesheets
stylesheet_path = os.path.join(paths.get('static_files'),
self.static_path.lstrip('/'),
self.stylesheet)
with open(stylesheet_path) as s:
self.default_stylesheet = s.read()
self.profanities = None
if self.profanity_wordlist and os.path.exists(self.profanity_wordlist):
with open(self.profanity_wordlist, 'r') as handle:
words = []
for line in handle:
words.append(line.strip(' \n\r'))
if words:
self.profanities = re.compile(
r"\b(%s)\b" % '|'.join(words), re.I | re.U)
self.reddit_host = socket.gethostname()
self.reddit_pid = os.getpid()
#the shutdown toggle
self.shutdown = False
#if we're going to use the query_queue, we need amqp
if self.write_query_queue and not self.amqp_host:
raise Exception("amqp_host must be defined to use the query queue")
# This requirement doesn't *have* to be a requirement, but there are
# bugs at the moment that will pop up if you violate it
if self.write_query_queue and not self.use_query_cache:
raise Exception("write_query_queue requires use_query_cache")
# try to set the source control revision number
try:
popen = subprocess.Popen(
["git", "log", "--date=short", "--pretty=format:%H %h", '-n1'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
resp, stderrdata = popen.communicate()
resp = resp.strip().split(' ')
self.version, self.short_version = resp
except object, e:
self.log.info("Couldn't read source revision (%r)" % e)
self.version = self.short_version = '(unknown)'
