def load_environment(global_conf={}, app_conf={}, setup_globals=True):
# Setup our paths
root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = {'root': root_path,
'controllers': os.path.join(root_path, 'controllers'),
'templates': [os.path.join(root_path, 'templates')],
}
if ConfigValue.bool(global_conf.get('uncompressedJS')):
paths['static_files'] = os.path.join(root_path, 'public')
else:
paths['static_files'] = os.path.join(os.path.dirname(root_path), 'build/public')
config.init_app(global_conf, app_conf, package='r2',
template_engine='mako', paths=paths)
g = config['pylons.g'] = Globals(global_conf, app_conf, paths)
if setup_globals:
g.setup()
g.plugins.declare_queues(g.queues)
r2.config.cache = g.cache
g.plugins.load_plugins()
config['r2.plugins'] = g.plugins
g.startup_timer.intermediate("plugins")
config['pylons.h'] = r2.lib.helpers
config['routes.map'] = routing.make_map()
#override the default response options
config['pylons.response_options']['headers'] = {}
# The following template options are passed to your template engines
tmpl_options = config['buffet.template_options']
tmpl_options['mako.filesystem_checks'] = getattr(g, 'reload_templates', False)
tmpl_options['mako.default_filters'] = ["mako_websafe"]
tmpl_options['mako.imports'] = \
["from r2.lib.filters import websafe, unsafe, mako_websafe",
"from pylons import c, g, request",
"from pylons.i18n import _, ungettext"]
# when mako loads a previously compiled template file from its cache, it
# doesn't check that the original template path matches the current path.
# in the event that a new plugin defines a template overriding a reddit
# template, unless the mtime newer, mako doesn't update the compiled
# template. as a workaround, this makes mako store compiled templates with
# the original path in the filename, forcing it to update with the path.
def mako_module_path(filename, uri):
module_directory = tmpl_options['mako.module_directory']
filename = filename.lstrip('/').replace('/', '-')
path = os.path.join(module_directory, filename + ".py")
return os.path.abspath(path)
tmpl_options['mako.modulename_callable'] = mako_module_path
if setup_globals:
g.setup_complete()
def load_environment(global_conf={}, app_conf={}, setup_globals=True):
# Setup our paths
root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = {
'root': root_path,
'controllers': os.path.join(root_path, 'controllers'),
'templates': tmpl_dirs,
}
if ConfigValue.bool(global_conf.get('uncompressedJS')):
paths['static_files'] = os.path.join(root_path, 'public')
else:
paths['static_files'] = os.path.join(os.path.dirname(root_path),
'build/public')
config.init_app(global_conf,
app_conf,
package='r2',
template_engine='mako',
paths=paths)
g = config['pylons.g'] = Globals(global_conf, app_conf, paths)
if setup_globals:
g.setup()
r2.config.cache = g.cache
config['pylons.h'] = r2.lib.helpers
g.plugins = config['r2.plugins'] = PluginLoader().load_plugins(
g.config.get('plugins', []))
config['routes.map'] = routing.make_map()
#override the default response options
config['pylons.response_options']['headers'] = {}
# The following template options are passed to your template engines
#tmpl_options = {}
#tmpl_options['myghty.log_errors'] = True
#tmpl_options['myghty.escapes'] = dict(l=webhelpers.auto_link, s=webhelpers.simple_format)
tmpl_options = config['buffet.template_options']
tmpl_options['mako.filesystem_checks'] = getattr(g, 'reload_templates',
False)
tmpl_options['mako.default_filters'] = ["mako_websafe"]
tmpl_options['mako.imports'] = \
["from r2.lib.filters import websafe, unsafe, mako_websafe",
"from pylons import c, g, request",
"from pylons.i18n import _, ungettext"]
def test_dict(self):
self.assertEquals({}, ConfigValue.dict(str, str)(''))
self.assertEquals({'a': ''}, ConfigValue.dict(str, str)('a'))
self.assertEquals({'a': 3}, ConfigValue.dict(str, int)('a: 3'))
self.assertEquals({'a': 3, 'b': 4},
ConfigValue.dict(str, int)('a: 3, b: 4'))
self.assertEquals({'a': (3, 5), 'b': (4, 6)},
ConfigValue.dict(
str, ConfigValue.tuple_of(int), delim=';')
('a: 3, 5; b: 4, 6'))
class Adzerk(Plugin):
needs_static_build = True
config = {
ConfigValue.int: [
'az_selfserve_site_id',
'az_selfserve_advertiser_id',
'az_selfserve_channel_id',
'az_selfserve_publisher_id',
'az_selfserve_network_id',
'az_selfserve_ad_type',
'az_selfserve_num_request',
],
ConfigValue.dict(ConfigValue.str, ConfigValue.int): [
'az_selfserve_priorities',
],
}
js = {
'reddit-init': Module(
'reddit-init.js',
'adzerk/adzerk.js',
)
}
def add_routes(self, mc):
mc('/api/request_promo/',
controller='adzerkapi',
action='request_promo')
def declare_queues(self, queues):
from r2.config.queues import MessageQueue
queues.declare({
"adzerk_q": MessageQueue(bind_to_self=True),
})
def load_controllers(self):
# replace the standard Ads view with an Adzerk specific one.
import r2.lib.pages.pages
from adzerkads import Ads as AdzerkAds
r2.lib.pages.pages.Ads = AdzerkAds
# replace standard adserver with Adzerk.
from adzerkpromote import AdzerkApiController
from adzerkpromote import hooks as adzerkpromote_hooks
adzerkpromote_hooks.register_all()
def load_environment(global_conf={}, app_conf={}, setup_globals=True):
# Setup our paths
root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = {
"root": root_path,
"controllers": os.path.join(root_path, "controllers"),
"templates": [os.path.join(root_path, "templates")],
}
if ConfigValue.bool(global_conf.get("uncompressedJS")):
paths["static_files"] = os.path.join(root_path, "public")
else:
paths["static_files"] = os.path.join(os.path.dirname(root_path), "build/public")
config.init_app(global_conf, app_conf, package="r2", template_engine="mako", paths=paths)
g = config["pylons.g"] = Globals(global_conf, app_conf, paths)
if setup_globals:
g.setup()
r2.config.cache = g.cache
g.plugins.load_plugins()
config["r2.plugins"] = g.plugins
config["pylons.h"] = r2.lib.helpers
config["routes.map"] = routing.make_map()
# override the default response options
config["pylons.response_options"]["headers"] = {}
# The following template options are passed to your template engines
# tmpl_options = {}
# tmpl_options['myghty.log_errors'] = True
# tmpl_options['myghty.escapes'] = dict(l=webhelpers.auto_link, s=webhelpers.simple_format)
tmpl_options = config["buffet.template_options"]
tmpl_options["mako.filesystem_checks"] = getattr(g, "reload_templates", False)
tmpl_options["mako.default_filters"] = ["mako_websafe"]
tmpl_options["mako.imports"] = [
"from r2.lib.filters import websafe, unsafe, mako_websafe",
"from pylons import c, g, request",
"from pylons.i18n import _, ungettext",
]
def load_environment(global_conf={}, app_conf={}, setup_globals=True):
# Setup our paths
root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = {'root': root_path,
'controllers': os.path.join(root_path, 'controllers'),
'templates': tmpl_dirs,
}
if ConfigValue.bool(global_conf.get('uncompressedJS')):
paths['static_files'] = os.path.join(root_path, 'public')
else:
paths['static_files'] = os.path.join(os.path.dirname(root_path), 'build/public')
config.init_app(global_conf, app_conf, package='r2',
template_engine='mako', paths=paths)
g = config['pylons.g'] = Globals(global_conf, app_conf, paths)
if setup_globals:
g.setup()
reddit_config.cache = g.cache
config['pylons.h'] = r2.lib.helpers
g.plugins = config['r2.plugins'] = PluginLoader().load_plugins(g.config.get('plugins', []))
config['routes.map'] = routing.make_map()
#override the default response options
config['pylons.response_options']['headers'] = {}
# The following template options are passed to your template engines
#tmpl_options = {}
#tmpl_options['myghty.log_errors'] = True
#tmpl_options['myghty.escapes'] = dict(l=webhelpers.auto_link, s=webhelpers.simple_format)
tmpl_options = config['buffet.template_options']
tmpl_options['mako.filesystem_checks'] = getattr(g, 'reload_templates', False)
tmpl_options['mako.default_filters'] = ["mako_websafe"]
tmpl_options['mako.imports'] = \
["from r2.lib.filters import websafe, unsafe, mako_websafe",
"from pylons import c, g, request",
"from pylons.i18n import _, ungettext"]
def test_dict(self):
self.assertEquals({}, ConfigValue.dict(str, str)(''))
self.assertEquals({'a': ''}, ConfigValue.dict(str, str)('a'))
self.assertEquals({'a': 3}, ConfigValue.dict(str, int)('a: 3'))
self.assertEquals({
'a': 3,
'b': 4
},
ConfigValue.dict(str, int)('a: 3, b: 4'))
self.assertEquals({
'a': (3, 5),
'b': (4, 6)
},
ConfigValue.dict(str,
ConfigValue.tuple_of(int),
delim=';')('a: 3, 5; b: 4, 6'))
class LiveUpdate(Plugin):
needs_static_build = True
errors = {
"LIVEUPDATE_NOT_CONTRIBUTOR":
N_("that user is not a contributor"),
"LIVEUPDATE_NO_INVITE_FOUND":
N_("there is no pending invite for that thread"),
"LIVEUPDATE_TOO_MANY_INVITES":
N_("there are too many pending invites outstanding"),
"LIVEUPDATE_ALREADY_CONTRIBUTOR":
N_("that user is already a contributor"),
"LIVEUPDATE_LINK_IS_NOT_DISCUSSION":
N_("the specified link is not a discussion about this live thread"),
}
config = {
ConfigValue.int: [
"liveupdate_invite_quota",
"liveupdate_min_score_for_discussions",
],
ConfigValue.str: [
"liveupdate_pixel_domain",
],
ConfigValue.baseplate(Date): [
"liveupdate_min_date_viewcounts",
],
}
js = {
"liveupdate":
LocalizedModule(
"liveupdate.js",
"lib/page-visibility.js",
"lib/tinycon.js",
"lib/moment.js",
"websocket.js",
"liveupdate/init.js",
"liveupdate/activity.js",
"liveupdate/embeds.js",
"liveupdate/event.js",
"liveupdate/favicon.js",
"liveupdate/listings.js",
"liveupdate/notifications.js",
"liveupdate/statusBar.js",
"liveupdate/report.js",
TemplateFileSource("liveupdate/update.html"),
TemplateFileSource("liveupdate/separator.html"),
TemplateFileSource("liveupdate/edit-button.html"),
TemplateFileSource("liveupdate/reported.html"),
PermissionsDataSource({
"liveupdate_contributor":
ContributorPermissionSet,
"liveupdate_contributor_invite":
ContributorPermissionSet,
}),
localized_appendices=[
MomentTranslations(),
],
),
}
def add_routes(self, mc):
mc(
"/live",
controller="liveupdateevents",
action="home",
conditions={"function": not_in_sr},
)
mc(
"/live/create",
controller="liveupdateevents",
action="create",
conditions={"function": not_in_sr},
)
mc(
"/api/live/by_id/:names",
action="listing",
controller="liveupdatebyid",
)
mc(
"/live/:filter",
action="listing",
controller="liveupdateevents",
conditions={"function": not_in_sr},
requirements={
"filter": "open|closed|reported|active|happening_now|mine"
},
)
mc(
"/api/live/:action",
controller="liveupdateevents",
conditions={"function": not_in_sr},
requirements={"action": "create|happening_now"},
)
mc("/live/:event",
controller="liveupdate",
action="listing",
conditions={"function": not_in_sr},
is_embed=False)
mc("/live/:event/embed",
controller="liveupdate",
action="listing",
conditions={"function": not_in_sr},
is_embed=True)
mc(
"/live/:event/updates/:target",
controller="liveupdate",
action="focus",
conditions={"function": not_in_sr},
)
mc("/live/:event/pixel",
controller="liveupdatepixel",
action="pixel",
conditions={"function": not_in_sr})
mc("/live/:event/:action",
controller="liveupdate",
conditions={"function": not_in_sr})
mc("/api/live/:event/:action",
controller="liveupdate",
conditions={"function": not_in_sr})
mc('/mediaembed/liveupdate/:event/:liveupdate/:embed_index',
controller="liveupdateembed",
action="mediaembed")
mc('/admin/happening-now',
controller='liveupdateadmin',
action='happening_now')
def load_controllers(self):
from r2.controllers.api_docs import api_section, section_info
api_section["live"] = "live"
section_info["live"] = {
"title": "live threads",
"description": sys.modules[__name__].__doc__,
}
from r2.models.token import OAuth2Scope
OAuth2Scope.scope_info["livemanage"] = {
"id":
"livemanage",
"name":
N_("Manage live threads"),
"description":
N_("Manage settings and contributors of live threads "
"I contribute to."),
}
from reddit_liveupdate.controllers import (
controller_hooks,
LiveUpdateByIDController,
LiveUpdateController,
LiveUpdateEventsController,
LiveUpdatePixelController,
)
from r2.config.templates import api
from r2.lib.jsontemplates import ListingJsonTemplate
from reddit_liveupdate import pages
api('liveupdateeventapp', pages.LiveUpdateEventAppJsonTemplate)
api('liveupdatefocusapp', pages.LiveUpdateEventAppJsonTemplate)
api('liveupdateevent', pages.LiveUpdateEventJsonTemplate)
api('liveupdatereportedeventrow', pages.LiveUpdateEventJsonTemplate)
api('liveupdatefeaturedevent',
pages.LiveUpdateFeaturedEventJsonTemplate)
api('liveupdate', pages.LiveUpdateJsonTemplate)
api('liveupdatecontributortableitem',
pages.ContributorTableItemJsonTemplate)
api('liveupdatediscussionslisting', ListingJsonTemplate)
controller_hooks.register_all()
from reddit_liveupdate import scraper
scraper.hooks.register_all()
def declare_queues(self, queues):
from r2.config.queues import MessageQueue
queues.declare({
"liveupdate_scraper_q": MessageQueue(bind_to_self=True),
})
queues.liveupdate_scraper_q << ("new_liveupdate_update", )
source_root_url = "https://github.com/reddit/reddit-plugin-liveupdate/blob/master/reddit_liveupdate/"
def get_documented_controllers(self):
from reddit_liveupdate.controllers import (
LiveUpdateController,
LiveUpdateEventsController,
LiveUpdateByIDController,
)
yield LiveUpdateController, "/api/live/{thread}"
yield LiveUpdateEventsController, ""
yield LiveUpdateByIDController, ""
def load_environment(global_conf={}, app_conf={}, setup_globals=True):
r2_path = get_r2_path()
root_path = os.path.join(r2_path, "r2")
paths = {
"root": root_path,
"controllers": os.path.join(root_path, "controllers"),
"templates": [os.path.join(root_path, "templates")],
}
if ConfigValue.bool(global_conf.get("uncompressedJS")):
paths["static_files"] = get_raw_statics_path()
else:
paths["static_files"] = get_built_statics_path()
config = PylonsConfig()
config.init_app(global_conf, app_conf, package="r2", paths=paths)
# don't put action arguments onto c automatically
config["pylons.c_attach_args"] = False
# when accessing non-existent attributes on c, return "" instead of dying
config["pylons.strict_tmpl_context"] = False
g = Globals(config, global_conf, app_conf, paths)
config["pylons.app_globals"] = g
if setup_globals:
config["r2.import_private"] = ConfigValue.bool(global_conf["import_private"])
g.setup()
g.plugins.declare_queues(g.queues)
g.plugins.load_plugins(config)
config["r2.plugins"] = g.plugins
g.startup_timer.intermediate("plugins")
config["pylons.h"] = r2.lib.helpers
config["routes.map"] = make_map(config)
# override the default response options
config["pylons.response_options"]["headers"] = {}
# when mako loads a previously compiled template file from its cache, it
# doesn't check that the original template path matches the current path.
# in the event that a new plugin defines a template overriding a reddit
# template, unless the mtime newer, mako doesn't update the compiled
# template. as a workaround, this makes mako store compiled templates with
# the original path in the filename, forcing it to update with the path.
if "cache_dir" in app_conf:
module_directory = os.path.join(app_conf["cache_dir"], "templates")
def mako_module_path(filename, uri):
filename = filename.lstrip("/").replace("/", "-")
path = os.path.join(module_directory, filename + ".py")
return os.path.abspath(path)
else:
# disable caching templates since we don't know where they should go.
module_directory = mako_module_path = None
# set up the templating system
config["pylons.app_globals"].mako_lookup = TemplateLookup(
directories=paths["templates"],
error_handler=handle_mako_error,
module_directory=module_directory,
input_encoding="utf-8",
default_filters=["conditional_websafe"],
filesystem_checks=getattr(g, "reload_templates", False),
imports=[
"from r2.lib.filters import websafe, unsafe, conditional_websafe",
"from pylons import request",
"from pylons import tmpl_context as c",
"from pylons import app_globals as g",
"from pylons.i18n import _, ungettext",
],
modulename_callable=mako_module_path,
)
if setup_globals:
g.setup_complete()
return config
def test_set(self):
self.assertEquals(set([]), ConfigValue.set(''))
self.assertEquals(set(['a', 'b']), ConfigValue.set('a, b'))
def test_tuple_of(self):
self.assertEquals((), ConfigValue.tuple_of(str)(''))
self.assertEquals(('a', 'b'), ConfigValue.tuple_of(str)('a, b'))
self.assertEquals(('a', 'b'),
ConfigValue.tuple_of(str, delim=':')('a : b'))
def test_int(self):
self.assertEquals(3, ConfigValue.int('3'))
self.assertEquals(-3, ConfigValue.int('-3'))
with self.assertRaises(ValueError):
ConfigValue.int('asdf')
def test_bool(self):
self.assertEquals(True, ConfigValue.bool('TrUe'))
self.assertEquals(False, ConfigValue.bool('fAlSe'))
with self.assertRaises(ValueError):
ConfigValue.bool('asdf')
def test_tuple(self):
self.assertEquals((), ConfigValue.tuple(''))
self.assertEquals(('a', 'b'), ConfigValue.tuple('a, b'))
class Globals(object):
spec = {
ConfigValue.int: [
'db_pool_size',
'db_pool_overflow_size',
'page_cache_time',
'commentpane_cache_time',
'num_mc_clients',
'MAX_CAMPAIGNS_PER_LINK',
'MIN_DOWN_LINK',
'MIN_UP_KARMA',
'MIN_DOWN_KARMA',
'MIN_RATE_LIMIT_KARMA',
'MIN_RATE_LIMIT_COMMENT_KARMA',
'VOTE_AGE_LIMIT',
'REPLY_AGE_LIMIT',
'REPORT_AGE_LIMIT',
'HOT_PAGE_AGE',
'RATELIMIT',
'QUOTA_THRESHOLD',
'ADMIN_COOKIE_TTL',
'ADMIN_COOKIE_MAX_IDLE',
'OTP_COOKIE_TTL',
'num_comments',
'max_comments',
'max_comments_gold',
'num_default_reddits',
'max_sr_images',
'num_serendipity',
'sr_dropdown_threshold',
'comment_visits_period',
'min_membership_create_community',
'bcrypt_work_factor',
'cassandra_pool_size',
'sr_banned_quota',
'sr_wikibanned_quota',
'sr_wikicontributor_quota',
'sr_moderator_invite_quota',
'sr_contributor_quota',
'sr_quota_time',
'wiki_keep_recent_days',
'wiki_max_page_length_bytes',
'wiki_max_page_name_length',
'wiki_max_page_separators',
],
ConfigValue.float: [
'min_promote_bid',
'max_promote_bid',
'statsd_sample_rate',
'querycache_prune_chance',
],
ConfigValue.bool: [
'debug',
'log_start',
'sqlprinting',
'template_debug',
'reload_templates',
'uncompressedJS',
'css_killswitch',
'db_create_tables',
'disallow_db_writes',
'disable_ratelimit',
'amqp_logging',
'read_only_mode',
'disable_wiki',
'heavy_load_mode',
's3_media_direct',
'disable_captcha',
'disable_ads',
'disable_require_admin_otp',
'static_pre_gzipped',
'static_secure_pre_gzipped',
'trust_local_proxies',
'shard_link_vote_queues',
],
ConfigValue.tuple: [
'plugins',
'stalecaches',
'memcaches',
'lockcaches',
'permacache_memcaches',
'rendercaches',
'pagecaches',
'cassandra_seeds',
'admins',
'sponsors',
'automatic_reddits',
'agents',
'allowed_css_linked_domains',
'authorized_cnames',
'hardcache_categories',
's3_media_buckets',
'allowed_pay_countries',
'case_sensitive_domains',
'reserved_subdomains',
'TRAFFIC_LOG_HOSTS',
'exempt_login_user_agents',
'timed_templates',
],
ConfigValue.str: [
'wiki_page_registration_info',
'wiki_page_privacy_policy',
'wiki_page_user_agreement',
],
ConfigValue.choice: {
'cassandra_rcl': {
'ONE': CL_ONE,
'QUORUM': CL_QUORUM
},
'cassandra_wcl': {
'ONE': CL_ONE,
'QUORUM': CL_QUORUM
},
},
config_gold_price: [
'gold_month_price',
'gold_year_price',
],
}
live_config_spec = {
ConfigValue.bool: [
'frontpage_dart',
],
ConfigValue.float: [
'spotlight_interest_sub_p',
'spotlight_interest_nosub_p',
],
ConfigValue.tuple: [
'sr_discovery_links',
'fastlane_links',
],
ConfigValue.dict(ConfigValue.int, ConfigValue.float): [
'comment_tree_version_weights',
],
ConfigValue.messages: [
'goldvertisement_blurbs',
'goldvertisement_has_gold_blurbs',
'welcomebar_messages',
'sidebar_message',
'gold_sidebar_message',
],
}
def __init__(self, global_conf, app_conf, paths, **extra):
"""
Globals acts as a container for objects available throughout
the life of the application.
One instance of Globals is created by Pylons during
application initialization and is available during requests
via the 'g' variable.
``global_conf``
The same variable used throughout ``config/middleware.py``
namely, the variables from the ``[DEFAULT]`` section of the
configuration file.
``app_conf``
The same ``kw`` dictionary used throughout
``config/middleware.py`` namely, the variables from the
section in the config file for your application.
``extra``
The configuration returned from ``load_config`` in
``config/middleware.py`` which may be of use in the setup of
your global variables.
"""
global_conf.setdefault("debug", False)
self.config = ConfigValueParser(global_conf)
self.config.add_spec(self.spec)
self.plugins = PluginLoader(self.config.get("plugins", []))
self.stats = Stats(self.config.get('statsd_addr'),
self.config.get('statsd_sample_rate'))
self.startup_timer = self.stats.get_timer("app_startup")
self.startup_timer.start()
self.paths = paths
self.running_as_script = global_conf.get('running_as_script', False)
# turn on for language support
self.lang = getattr(self, 'site_lang', 'en')
self.languages, self.lang_name = \
get_active_langs(default_lang=self.lang)
all_languages = self.lang_name.keys()
all_languages.sort()
self.all_languages = all_languages
# set default time zone if one is not set
tz = global_conf.get('timezone', 'UTC')
self.tz = pytz.timezone(tz)
dtz = global_conf.get('display_timezone', tz)
self.display_tz = pytz.timezone(dtz)
self.startup_timer.intermediate("init")
def __getattr__(self, name):
if not name.startswith('_') and name in self.config:
return self.config[name]
else:
raise AttributeError
def setup(self):
self.queues = queues.declare_queues(self)
################# CONFIGURATION
# AMQP is required
if not self.amqp_host:
raise ValueError("amqp_host not set in the .ini")
if not self.cassandra_seeds:
raise ValueError("cassandra_seeds not set in the .ini")
# heavy load mode is read only mode with a different infobar
if self.heavy_load_mode:
self.read_only_mode = True
origin_prefix = self.domain_prefix + "." if self.domain_prefix else ""
self.origin = "http://" + origin_prefix + self.domain
self.secure_domains = set([urlparse(self.payment_domain).netloc])
self.trusted_domains = set([self.domain])
self.trusted_domains.update(self.authorized_cnames)
if self.https_endpoint:
https_url = urlparse(self.https_endpoint)
self.secure_domains.add(https_url.netloc)
self.trusted_domains.add(https_url.hostname)
if getattr(self, 'oauth_domain', None):
self.secure_domains.add(self.oauth_domain)
# load the unique hashed names of files under static
static_files = os.path.join(self.paths.get('static_files'), 'static')
names_file_path = os.path.join(static_files, 'names.json')
if os.path.exists(names_file_path):
with open(names_file_path) as handle:
self.static_names = json.load(handle)
else:
self.static_names = {}
# make python warnings go through the logging system
logging.captureWarnings(capture=True)
log = logging.getLogger('reddit')
# when we're a script (paster run) just set up super simple logging
if self.running_as_script:
log.setLevel(logging.INFO)
log.addHandler(logging.StreamHandler())
# if in debug mode, override the logging level to DEBUG
if self.debug:
log.setLevel(logging.DEBUG)
# attempt to figure out which pool we're in and add that to the
# LogRecords.
try:
with open("/etc/ec2_asg", "r") as f:
pool = f.read().strip()
# clean up the pool name since we're putting stuff after "-"
pool = pool.partition("-")[0]
except IOError:
pool = "reddit-app"
self.log = logging.LoggerAdapter(log, {"pool": pool})
# make cssutils use the real logging system
csslog = logging.getLogger("cssutils")
cssutils.log.setLog(csslog)
# load the country list
countries_file_path = os.path.join(static_files, "countries.json")
try:
with open(countries_file_path) as handle:
self.countries = json.load(handle)
self.log.debug("Using countries.json.")
except IOError:
self.log.warning("Couldn't find countries.json. Using pycountry.")
self.countries = get_countries_and_codes()
if not self.media_domain:
self.media_domain = self.domain
if self.media_domain == self.domain:
print ("Warning: g.media_domain == g.domain. " +
"This may give untrusted content access to user cookies")
for arg in sys.argv:
tokens = arg.split("=")
if len(tokens) == 2:
k, v = tokens
self.log.debug("Overriding g.%s to %s" % (k, v))
setattr(self, k, v)
self.reddit_host = socket.gethostname()
self.reddit_pid = os.getpid()
if hasattr(signal, 'SIGUSR1'):
# not all platforms have user signals
signal.signal(signal.SIGUSR1, thread_dump)
self.startup_timer.intermediate("configuration")
################# ZOOKEEPER
# for now, zookeeper will be an optional part of the stack.
# if it's not configured, we will grab the expected config from the
# [live_config] section of the ini file
zk_hosts = self.config.get("zookeeper_connection_string")
if zk_hosts:
from r2.lib.zookeeper import (connect_to_zookeeper,
LiveConfig, LiveList)
zk_username = self.config["zookeeper_username"]
zk_password = self.config["zookeeper_password"]
self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username,
zk_password))
self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE)
self.throttles = LiveList(self.zookeeper, "/throttles",
map_fn=ipaddress.ip_network,
reduce_fn=ipaddress.collapse_addresses)
self.banned_domains = LiveDict(self.zookeeper,
"/banned-domains",
watch=True)
else:
self.zookeeper = None
parser = ConfigParser.RawConfigParser()
parser.read([self.config["__file__"]])
self.live_config = extract_live_config(parser, self.plugins)
self.throttles = tuple() # immutable since it's not real
self.banned_domains = dict()
self.startup_timer.intermediate("zookeeper")
################# MEMCACHE
num_mc_clients = self.num_mc_clients
# the main memcache pool. used for most everything.
self.memcache = CMemcache(
self.memcaches,
min_compress_len=50 * 1024,
num_clients=num_mc_clients,
)
# a smaller pool of caches used only for distributed locks.
# TODO: move this to ZooKeeper
self.lock_cache = CMemcache(self.lockcaches,
num_clients=num_mc_clients)
self.make_lock = make_lock_factory(self.lock_cache, self.stats)
# memcaches used in front of the permacache CF in cassandra.
# XXX: this is a legacy thing; permacache was made when C* didn't have
# a row cache.
if self.permacache_memcaches:
permacache_memcaches = CMemcache(self.permacache_memcaches,
min_compress_len=50 * 1024,
num_clients=num_mc_clients)
else:
permacache_memcaches = None
# the stalecache is a memcached local to the current app server used
# for data that's frequently fetched but doesn't need to be fresh.
if self.stalecaches:
stalecaches = CMemcache(self.stalecaches,
num_clients=num_mc_clients)
else:
stalecaches = None
# rendercache holds rendered partial templates.
rendercaches = CMemcache(
self.rendercaches,
noreply=True,
no_block=True,
num_clients=num_mc_clients,
min_compress_len=1400,
)
# pagecaches hold fully rendered pages
pagecaches = CMemcache(
self.pagecaches,
noreply=True,
no_block=True,
num_clients=num_mc_clients,
min_compress_len=1400,
)
self.startup_timer.intermediate("memcache")
################# CASSANDRA
keyspace = "reddit"
self.cassandra_pools = {
"main":
StatsCollectingConnectionPool(
keyspace,
stats=self.stats,
logging_name="main",
server_list=self.cassandra_seeds,
pool_size=self.cassandra_pool_size,
timeout=4,
max_retries=3,
prefill=False
),
}
permacache_cf = CassandraCache(
'permacache',
self.cassandra_pools[self.cassandra_default_pool],
read_consistency_level=self.cassandra_rcl,
write_consistency_level=self.cassandra_wcl
)
self.startup_timer.intermediate("cassandra")
################# POSTGRES
event.listens_for(engine.Engine, 'before_cursor_execute')(
self.stats.pg_before_cursor_execute)
event.listens_for(engine.Engine, 'after_cursor_execute')(
self.stats.pg_after_cursor_execute)
self.dbm = self.load_db_params()
self.startup_timer.intermediate("postgres")
################# CHAINS
# initialize caches. Any cache-chains built here must be added
# to cache_chains (closed around by reset_caches) so that they
# can properly reset their local components
cache_chains = {}
localcache_cls = (SelfEmptyingCache if self.running_as_script
else LocalCache)
if stalecaches:
self.cache = StaleCacheChain(
localcache_cls(),
stalecaches,
self.memcache,
)
else:
self.cache = MemcacheChain((localcache_cls(), self.memcache))
cache_chains.update(cache=self.cache)
self.rendercache = MemcacheChain((
localcache_cls(),
rendercaches,
))
cache_chains.update(rendercache=self.rendercache)
self.pagecache = MemcacheChain((
localcache_cls(),
pagecaches,
))
cache_chains.update(pagecache=self.pagecache)
# the thing_cache is used in tdb_cassandra.
self.thing_cache = CacheChain((localcache_cls(),))
cache_chains.update(thing_cache=self.thing_cache)
self.permacache = CassandraCacheChain(
localcache_cls(),
permacache_cf,
memcache=permacache_memcaches,
lock_factory=self.make_lock,
)
cache_chains.update(permacache=self.permacache)
# hardcache is used for various things that tend to expire
# TODO: replace hardcache w/ cassandra stuff
self.hardcache = HardcacheChain(
(localcache_cls(), self.memcache, HardCache(self)),
cache_negative_results=True,
)
cache_chains.update(hardcache=self.hardcache)
# I know this sucks, but we need non-request-threads to be
# able to reset the caches, so we need them be able to close
# around 'cache_chains' without being able to call getattr on
# 'g'
def reset_caches():
for name, chain in cache_chains.iteritems():
chain.reset()
chain.stats = CacheStats(self.stats, name)
self.cache_chains = cache_chains
self.reset_caches = reset_caches
self.reset_caches()
self.startup_timer.intermediate("cache_chains")
# try to set the source control revision numbers
self.versions = {}
r2_root = os.path.dirname(os.path.dirname(self.paths["root"]))
r2_gitdir = os.path.join(r2_root, ".git")
self.short_version = self.record_repo_version("r2", r2_gitdir)
if I18N_PATH:
i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git")
self.record_repo_version("i18n", i18n_git_path)
self.startup_timer.intermediate("revisions")
def setup_complete(self):
self.startup_timer.stop()
self.stats.flush()
if self.log_start:
self.log.error(
"%s:%s started %s at %s (took %.02fs)",
self.reddit_host,
self.reddit_pid,
self.short_version,
datetime.now().strftime("%H:%M:%S"),
self.startup_timer.elapsed_seconds()
)
def record_repo_version(self, repo_name, git_dir):
"""Get the currently checked out git revision for a given repository,
record it in g.versions, and return the short version of the hash."""
try:
subprocess.check_output
except AttributeError:
# python 2.6 compat
pass
else:
try:
revision = subprocess.check_output(["git",
"--git-dir", git_dir,
"rev-parse", "HEAD"])
except subprocess.CalledProcessError, e:
self.log.warning("Unable to fetch git revision: %r", e)
else:
def test_float(self):
self.assertEquals(3.0, ConfigValue.float('3'))
self.assertEquals(-3.0, ConfigValue.float('-3'))
with self.assertRaises(ValueError):
ConfigValue.float('asdf')
def test_bool(self):
self.assertEquals(True, ConfigValue.bool('TrUe'))
self.assertEquals(False, ConfigValue.bool('fAlSe'))
with self.assertRaises(ValueError):
ConfigValue.bool('asdf')
def test_int(self):
self.assertEquals(3, ConfigValue.int('3'))
self.assertEquals(-3, ConfigValue.int('-3'))
with self.assertRaises(ValueError):
ConfigValue.int('asdf')
def test_str(self):
self.assertEquals('x', ConfigValue.str('x'))
def load_environment(global_conf={}, app_conf={}, setup_globals=True):
# Setup our paths
root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = {
'root': root_path,
'controllers': os.path.join(root_path, 'controllers'),
'templates': [os.path.join(root_path, 'templates')],
}
if ConfigValue.bool(global_conf.get('uncompressedJS')):
paths['static_files'] = os.path.join(root_path, 'public')
else:
paths['static_files'] = os.path.join(os.path.dirname(root_path),
'build/public')
config.init_app(global_conf,
app_conf,
package='r2',
template_engine='mako',
paths=paths)
g = config['pylons.g'] = Globals(global_conf, app_conf, paths)
if setup_globals:
g.setup()
g.plugins.declare_queues(g.queues)
r2.config.cache = g.cache
g.plugins.load_plugins()
config['r2.plugins'] = g.plugins
g.startup_timer.intermediate("plugins")
config['pylons.h'] = r2.lib.helpers
config['routes.map'] = routing.make_map()
#override the default response options
config['pylons.response_options']['headers'] = {}
# The following template options are passed to your template engines
tmpl_options = config['buffet.template_options']
tmpl_options['mako.filesystem_checks'] = getattr(g, 'reload_templates',
False)
tmpl_options['mako.default_filters'] = ["mako_websafe"]
tmpl_options['mako.imports'] = \
["from r2.lib.filters import websafe, unsafe, mako_websafe",
"from pylons import c, g, request",
"from pylons.i18n import _, ungettext"]
# when mako loads a previously compiled template file from its cache, it
# doesn't check that the original template path matches the current path.
# in the event that a new plugin defines a template overriding a reddit
# template, unless the mtime newer, mako doesn't update the compiled
# template. as a workaround, this makes mako store compiled templates with
# the original path in the filename, forcing it to update with the path.
def mako_module_path(filename, uri):
module_directory = tmpl_options['mako.module_directory']
filename = filename.lstrip('/').replace('/', '-')
path = os.path.join(module_directory, filename + ".py")
return os.path.abspath(path)
tmpl_options['mako.modulename_callable'] = mako_module_path
if setup_globals:
g.setup_complete()
class Globals(object):
spec = {
ConfigValue.int: [
'db_pool_size',
'db_pool_overflow_size',
'commentpane_cache_time',
'num_mc_clients',
'MAX_CAMPAIGNS_PER_LINK',
'MIN_DOWN_LINK',
'MIN_UP_KARMA',
'MIN_DOWN_KARMA',
'MIN_RATE_LIMIT_KARMA',
'MIN_RATE_LIMIT_COMMENT_KARMA',
'HOT_PAGE_AGE',
'ADMIN_COOKIE_TTL',
'ADMIN_COOKIE_MAX_IDLE',
'OTP_COOKIE_TTL',
'hsts_max_age',
'num_comments',
'max_comments',
'max_comments_gold',
'max_comment_parent_walk',
'max_sr_images',
'num_serendipity',
'comment_visits_period',
'butler_max_mentions',
'min_membership_create_community',
'bcrypt_work_factor',
'cassandra_pool_size',
'sr_banned_quota',
'sr_muted_quota',
'sr_wikibanned_quota',
'sr_wikicontributor_quota',
'sr_moderator_invite_quota',
'sr_contributor_quota',
'sr_quota_time',
'sr_invite_limit',
'thumbnail_hidpi_scaling',
'wiki_keep_recent_days',
'wiki_max_page_length_bytes',
'wiki_max_config_stylesheet_length_bytes',
'wiki_max_page_name_length',
'wiki_max_page_separators',
'RL_RESET_MINUTES',
'RL_OAUTH_RESET_MINUTES',
'comment_karma_display_floor',
'link_karma_display_floor',
'mobile_auth_gild_time',
'default_total_budget_pennies',
'min_total_budget_pennies',
'max_total_budget_pennies',
'default_bid_pennies',
'min_bid_pennies',
'max_bid_pennies',
'frequency_cap_min',
'frequency_cap_default',
'eu_cookie_max_attempts',
'captcha_sol_length',
'captcha_font_size',
'banner_variants',
'precompute_limit',
'precompute_limit_hot',
'hot_max_links_per_subreddit',
'fetch_title_max_download_kb',
],
ConfigValue.float: [
'statsd_sample_rate',
'querycache_prune_chance',
'RL_AVG_REQ_PER_SEC',
'RL_OAUTH_AVG_REQ_PER_SEC',
'RL_LOGIN_AVG_PER_SEC',
'RL_LOGIN_IP_AVG_PER_SEC',
'RL_SHARE_AVG_PER_SEC',
'tracing_sample_rate',
'hot_period_seconds',
],
ConfigValue.bool: [
'debug',
'log_start',
'sqlprinting',
'template_debug',
'reload_templates',
'uncompressedJS',
'css_killswitch',
'db_create_tables',
'disallow_db_writes',
'disable_ratelimit',
'amqp_logging',
'read_only_mode',
'disable_wiki',
'heavy_load_mode',
'disable_captcha',
'disable_ads',
'disable_require_admin_otp',
'trust_local_proxies',
'shard_commentstree_queues',
'shard_author_query_queues',
'shard_subreddit_query_queues',
'shard_domain_query_queues',
'authnet_validate',
'ENFORCE_RATELIMIT',
'RL_SITEWIDE_ENABLED',
'RL_OAUTH_SITEWIDE_ENABLED',
'enable_loggedout_experiments',
'disable_geoip_service',
'disable_remote_fetch',
'disable_newsletter',
'remote_fetch_proxy_enabled',
'gold_gilding_enabled',
'sub_muting_enabled',
'allsr_prefilter_allow_top',
'site_index_user_configurable',
'allow_top_affects_new',
'allow_top_false_subreddits_tab',
'block_user_show_comments',
'block_user_show_links',
'chat_guest_chat_enabled',
'chat_all',
'chat_front',
],
ConfigValue.tuple: [
'plugins',
'stalecaches',
'lockcaches',
'permacache_memcaches',
'cassandra_seeds',
'automatic_reddits',
'hardcache_categories',
'case_sensitive_domains',
'known_image_domains',
'reserved_subdomains',
'offsite_subdomains',
'TRAFFIC_LOG_HOSTS',
'exempt_login_user_agents',
'autoexpand_media_types',
'media_preview_domain_whitelist',
'multi_icons',
'hide_subscribers_srs',
'mcrouter_addr',
'permacache_domain_priority',
],
ConfigValue.tuple_of(ConfigValue.int): [
'thumbnail_size',
'preview_image_max_size',
'preview_image_min_size',
'mobile_ad_image_size',
],
ConfigValue.tuple_of(ConfigValue.float): [
'ios_versions',
'android_versions',
],
ConfigValue.dict(ConfigValue.str, ConfigValue.int): [
'user_agent_ratelimit_regexes',
],
ConfigValue.str: [
'wiki_page_registration_info',
'wiki_page_privacy_policy',
'wiki_page_user_agreement',
'wiki_page_gold_bottlecaps',
'fraud_email',
'feedback_email',
'share_reply',
'community_email',
'smtp_server',
'events_collector_url',
'events_collector_test_url',
'search_provider',
'remote_fetch_proxy_url',
'brander_community',
'brander_community_plural',
'imgur_client_id',
],
ConfigValue.choice(ONE=CL_ONE, QUORUM=CL_QUORUM): [
'cassandra_rcl',
'cassandra_wcl',
],
ConfigValue.choice(zookeeper="zookeeper", config="config"): [
"liveconfig_source",
"secrets_source",
],
ConfigValue.timeinterval: [
'ARCHIVE_AGE',
"vote_queue_grace_period",
],
config_gold_price: [
'gold_month_price',
'gold_year_price',
'cpm_selfserve',
'cpm_selfserve_geotarget_metro',
'cpm_selfserve_geotarget_country',
'cpm_selfserve_collection',
],
ConfigValue.baseplate(
baseplate_config.Optional(baseplate_config.Endpoint)): [
"activity_endpoint",
"tracing_endpoint",
],
ConfigValue.dict(ConfigValue.str, ConfigValue.str): [
'emr_traffic_tags',
],
}
live_config_spec = {
ConfigValue.bool: [
'frontend_logging',
'mobile_gild_first_login',
'precomputed_comment_suggested_sort',
],
ConfigValue.int: [
'captcha_exempt_comment_karma',
'captcha_exempt_link_karma',
'create_sr_account_age_days',
'create_sr_comment_karma',
'create_sr_link_karma',
'cflag_min_votes',
'ads_popularity_threshold',
'precomputed_comment_sort_min_comments',
'comment_vote_update_threshold',
'comment_vote_update_period',
'create_sr_ratelimit_once_per_days',
],
ConfigValue.float: [
'cflag_lower_bound',
'cflag_upper_bound',
'spotlight_interest_sub_p',
'spotlight_interest_nosub_p',
'gold_revenue_goal',
'invalid_key_sample_rate',
'events_collector_vote_sample_rate',
'events_collector_poison_sample_rate',
'events_collector_mod_sample_rate',
'events_collector_quarantine_sample_rate',
'events_collector_modmail_sample_rate',
'events_collector_report_sample_rate',
'events_collector_submit_sample_rate',
'events_collector_comment_sample_rate',
'events_collector_use_gzip_chance',
'https_cert_testing_probability',
],
ConfigValue.tuple: [
'fastlane_links',
'listing_chooser_sample_multis',
'discovery_srs',
'proxy_gilding_accounts',
'mweb_blacklist_expressions',
'global_loid_experiments',
'precomputed_comment_sorts',
'mailgun_domains',
],
ConfigValue.str: [
'listing_chooser_gold_multi',
'listing_chooser_explore_sr',
],
ConfigValue.messages: [
'welcomebar_messages',
'sidebar_message',
'gold_sidebar_message',
],
ConfigValue.dict(ConfigValue.str, ConfigValue.int): [
'ticket_groups',
'ticket_user_fields',
],
ConfigValue.dict(ConfigValue.str, ConfigValue.float): [
'pennies_per_server_second',
],
ConfigValue.dict(ConfigValue.str, ConfigValue.str): [
'employee_approved_clients',
'mobile_auth_allowed_clients',
'modmail_forwarding_email',
'modmail_account_map',
],
ConfigValue.dict(ConfigValue.str, ConfigValue.choice(**PERMISSIONS)): [
'employees',
],
}
def __init__(self, config, global_conf, app_conf, paths, **extra):
"""
Globals acts as a container for objects available throughout
the life of the application.
One instance of Globals is created by Pylons during
application initialization and is available during requests
via the 'g' variable.
``config``
The PylonsConfig object passed in from ``config/environment.py``
``global_conf``
The same variable used throughout ``config/middleware.py``
namely, the variables from the ``[DEFAULT]`` section of the
configuration file.
``app_conf``
The same ``kw`` dictionary used throughout
``config/middleware.py`` namely, the variables from the
section in the config file for your application.
``extra``
The configuration returned from ``load_config`` in
``config/middleware.py`` which may be of use in the setup of
your global variables.
"""
global_conf.setdefault("debug", False)
# reloading site ensures that we have a fresh sys.path to build our
# working set off of. this means that forked worker processes won't get
# the sys.path that was current when the master process was spawned
# meaning that new plugins will be picked up on regular app reload
# rather than having to restart the master process as well.
reload(site)
self.pkg_resources_working_set = pkg_resources.WorkingSet()
self.config = ConfigValueParser(global_conf)
self.config.add_spec(self.spec)
self.plugins = PluginLoader(self.pkg_resources_working_set,
self.config.get("plugins", []))
self.stats = Stats(self.config.get('statsd_addr'),
self.config.get('statsd_sample_rate'))
self.startup_timer = self.stats.get_timer("app_startup")
self.startup_timer.start()
self.baseplate = Baseplate()
self.baseplate.configure_logging()
self.baseplate.register(R2BaseplateObserver())
self.baseplate.configure_tracing(
"r2",
tracing_endpoint=self.config.get("tracing_endpoint"),
sample_rate=self.config.get("tracing_sample_rate"),
)
self.paths = paths
self.running_as_script = global_conf.get('running_as_script', False)
# turn on for language support
self.lang = getattr(self, 'site_lang', 'en')
self.languages, self.lang_name = get_active_langs(
config, default_lang=self.lang)
all_languages = self.lang_name.keys()
all_languages.sort()
self.all_languages = all_languages
# set default time zone if one is not set
tz = global_conf.get('timezone', 'UTC')
self.tz = pytz.timezone(tz)
dtz = global_conf.get('display_timezone', tz)
self.display_tz = pytz.timezone(dtz)
self.startup_timer.intermediate("init")
def __getattr__(self, name):
if not name.startswith('_') and name in self.config:
return self.config[name]
else:
raise AttributeError("g has no attr %r" % name)
def setup(self):
self.env = ''
if (
# handle direct invocation of "nosetests"
"test" in sys.argv[0] or
# handle "setup.py test" and all permutations thereof.
"setup.py" in sys.argv[0] and "test" in sys.argv[1:]):
self.env = "unit_test"
self.queues = queues.declare_queues(self)
self.extension_subdomains = dict(
simple="mobile",
i="compact",
api="api",
rss="rss",
xml="xml",
json="json",
)
# SaidIt CUSTOM
self.extension_subdomains[
self.config['extension_subdomain_mobile_v2']] = self.config[
'extension_subdomain_mobile_v2_render_style']
################# PROVIDERS
self.auth_provider = select_provider(
self.config,
self.pkg_resources_working_set,
"r2.provider.auth",
self.authentication_provider,
)
self.media_provider = select_provider(
self.config,
self.pkg_resources_working_set,
"r2.provider.media",
self.media_provider,
)
self.cdn_provider = select_provider(
self.config,
self.pkg_resources_working_set,
"r2.provider.cdn",
self.cdn_provider,
)
self.ticket_provider = select_provider(
self.config,
self.pkg_resources_working_set,
"r2.provider.support",
# TODO: fix this later, it refuses to pick up
# g.config['ticket_provider'] value, so hardcoding for now.
# really, the next uncommented line should be:
#self.ticket_provider,
# instead of:
"zendesk",
)
self.image_resizing_provider = select_provider(
self.config,
self.pkg_resources_working_set,
"r2.provider.image_resizing",
self.image_resizing_provider,
)
self.email_provider = select_provider(
self.config,
self.pkg_resources_working_set,
"r2.provider.email",
self.email_provider,
)
self.startup_timer.intermediate("providers")
################# CONFIGURATION
# AMQP is required
if not self.amqp_host:
raise ValueError("amqp_host not set in the .ini")
if not self.cassandra_seeds:
raise ValueError("cassandra_seeds not set in the .ini")
# heavy load mode is read only mode with a different infobar
if self.heavy_load_mode:
self.read_only_mode = True
origin_prefix = self.domain_prefix + "." if self.domain_prefix else ""
self.origin = self.default_scheme + "://" + origin_prefix + self.domain
self.trusted_domains = set([self.domain])
if self.https_endpoint:
https_url = urlparse(self.https_endpoint)
self.trusted_domains.add(https_url.hostname)
# load the unique hashed names of files under static
static_files = os.path.join(self.paths.get('static_files'), 'static')
names_file_path = os.path.join(static_files, 'names.json')
if os.path.exists(names_file_path):
with open(names_file_path) as handle:
self.static_names = json.load(handle)
else:
self.static_names = {}
# make python warnings go through the logging system
logging.captureWarnings(capture=True)
log = logging.getLogger('reddit')
# when we're a script (paster run) just set up super simple logging
if self.running_as_script:
log.setLevel(logging.INFO)
log.addHandler(logging.StreamHandler())
# if in debug mode, override the logging level to DEBUG
if self.debug:
log.setLevel(logging.DEBUG)
# attempt to figure out which pool we're in and add that to the
# LogRecords.
try:
with open("/etc/ec2_asg", "r") as f:
pool = f.read().strip()
# clean up the pool name since we're putting stuff after "-"
pool = pool.partition("-")[0]
except IOError:
pool = "reddit-app"
self.log = logging.LoggerAdapter(log, {"pool": pool})
# set locations
locations = pkg_resources.resource_stream(__name__,
"../data/locations.json")
self.locations = json.loads(locations.read())
if not self.media_domain:
self.media_domain = self.domain
if self.media_domain == self.domain:
print >> sys.stderr, (
"Warning: g.media_domain == g.domain. " +
"This may give untrusted content access to user cookies")
if self.oauth_domain == self.domain:
print >> sys.stderr, ("Warning: g.oauth_domain == g.domain. "
"CORS requests to g.domain will be allowed")
for arg in sys.argv:
tokens = arg.split("=")
if len(tokens) == 2:
k, v = tokens
self.log.debug("Overriding g.%s to %s" % (k, v))
setattr(self, k, v)
self.reddit_host = socket.gethostname()
self.reddit_pid = os.getpid()
if hasattr(signal, 'SIGUSR1'):
# not all platforms have user signals
signal.signal(signal.SIGUSR1, thread_dump)
locale.setlocale(locale.LC_ALL, self.locale)
# Pre-calculate ratelimit values
self.RL_RESET_SECONDS = self.config["RL_RESET_MINUTES"] * 60
self.RL_MAX_REQS = int(self.config["RL_AVG_REQ_PER_SEC"] *
self.RL_RESET_SECONDS)
self.RL_OAUTH_RESET_SECONDS = self.config["RL_OAUTH_RESET_MINUTES"] * 60
self.RL_OAUTH_MAX_REQS = int(self.config["RL_OAUTH_AVG_REQ_PER_SEC"] *
self.RL_OAUTH_RESET_SECONDS)
self.RL_LOGIN_MAX_REQS = int(self.config["RL_LOGIN_AVG_PER_SEC"] *
self.RL_RESET_SECONDS)
self.RL_LOGIN_IP_MAX_REQS = int(
self.config["RL_LOGIN_IP_AVG_PER_SEC"] * self.RL_RESET_SECONDS)
self.RL_SHARE_MAX_REQS = int(self.config["RL_SHARE_AVG_PER_SEC"] *
self.RL_RESET_SECONDS)
# Compile ratelimit regexs
user_agent_ratelimit_regexes = {}
for agent_re, limit in self.user_agent_ratelimit_regexes.iteritems():
user_agent_ratelimit_regexes[re.compile(agent_re)] = limit
self.user_agent_ratelimit_regexes = user_agent_ratelimit_regexes
self.startup_timer.intermediate("configuration")
################# ZOOKEEPER
zk_hosts = self.config["zookeeper_connection_string"]
zk_username = self.config["zookeeper_username"]
zk_password = self.config["zookeeper_password"]
self.zookeeper = connect_to_zookeeper(zk_hosts,
(zk_username, zk_password))
self.throttles = IPNetworkLiveList(
self.zookeeper,
root="/throttles",
reduced_data_node="/throttles_reduced",
)
parser = ConfigParser.RawConfigParser()
parser.optionxform = str
parser.read([self.config["__file__"]])
if self.config["liveconfig_source"] == "zookeeper":
self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE)
else:
self.live_config = extract_live_config(parser, self.plugins)
if self.config["secrets_source"] == "zookeeper":
self.secrets = fetch_secrets(self.zookeeper)
else:
self.secrets = extract_secrets(parser)
################# PRIVILEGED USERS
self.admins = PermissionFilteredEmployeeList(self.live_config,
type="admin")
self.sponsors = PermissionFilteredEmployeeList(self.live_config,
type="sponsor")
self.employees = PermissionFilteredEmployeeList(self.live_config,
type="employee")
# Store which OAuth clients employees may use, the keys are just for
# readability.
self.employee_approved_clients = \
self.live_config["employee_approved_clients"].values()
self.mobile_auth_allowed_clients = self.live_config[
"mobile_auth_allowed_clients"].values()
self.startup_timer.intermediate("zookeeper")
################# MEMCACHE
num_mc_clients = self.num_mc_clients
# a smaller pool of caches used only for distributed locks.
self.lock_cache = CMemcache(
"lock",
self.lockcaches,
num_clients=num_mc_clients,
)
self.make_lock = make_lock_factory(self.lock_cache, self.stats)
# memcaches used in front of the permacache CF in cassandra.
# XXX: this is a legacy thing; permacache was made when C* didn't have
# a row cache.
permacache_memcaches = CMemcache(
"perma",
self.permacache_memcaches,
min_compress_len=1400,
num_clients=num_mc_clients,
)
# the stalecache is a memcached local to the current app server used
# for data that's frequently fetched but doesn't need to be fresh.
if self.stalecaches:
stalecaches = CMemcache(
"stale",
self.stalecaches,
num_clients=num_mc_clients,
)
else:
stalecaches = None
self.startup_timer.intermediate("memcache")
################# MCROUTER
self.mcrouter = Mcrouter(
"mcrouter",
self.mcrouter_addr,
min_compress_len=1400,
num_clients=num_mc_clients,
)
################# THRIFT-BASED SERVICES
activity_endpoint = self.config.get("activity_endpoint")
if activity_endpoint:
# make ActivityInfo objects rendercache-key friendly
# TODO: figure out a more general solution for this if
# we need to do this for other thrift-generated objects
ActivityInfo.cache_key = lambda self, style: repr(self)
activity_pool = ThriftConnectionPool(activity_endpoint,
timeout=0.1)
self.baseplate.add_to_context(
"activity_service",
ThriftContextFactory(activity_pool, ActivityService.Client))
self.startup_timer.intermediate("thrift")
################# CASSANDRA
keyspace = "reddit"
self.cassandra_pools = {
"main":
StatsCollectingConnectionPool(keyspace,
stats=self.stats,
logging_name="main",
server_list=self.cassandra_seeds,
pool_size=self.cassandra_pool_size,
timeout=4,
max_retries=3,
prefill=False),
}
permacache_cf = Permacache._setup_column_family(
'permacache',
self.cassandra_pools[self.cassandra_default_pool],
)
self.startup_timer.intermediate("cassandra")
################# POSTGRES
self.dbm = self.load_db_params()
self.startup_timer.intermediate("postgres")
################# CHAINS
# initialize caches. Any cache-chains built here must be added
# to cache_chains (closed around by reset_caches) so that they
# can properly reset their local components
cache_chains = {}
localcache_cls = (SelfEmptyingCache
if self.running_as_script else LocalCache)
if stalecaches:
self.gencache = StaleCacheChain(
localcache_cls(),
stalecaches,
self.mcrouter,
)
else:
self.gencache = CacheChain((localcache_cls(), self.mcrouter))
cache_chains.update(gencache=self.gencache)
if stalecaches:
self.thingcache = StaleCacheChain(
localcache_cls(),
stalecaches,
self.mcrouter,
)
else:
self.thingcache = CacheChain((localcache_cls(), self.mcrouter))
cache_chains.update(thingcache=self.thingcache)
if stalecaches:
self.memoizecache = StaleCacheChain(
localcache_cls(),
stalecaches,
self.mcrouter,
)
else:
self.memoizecache = MemcacheChain(
(localcache_cls(), self.mcrouter))
cache_chains.update(memoizecache=self.memoizecache)
if stalecaches:
self.srmembercache = StaleCacheChain(
localcache_cls(),
stalecaches,
self.mcrouter,
)
else:
self.srmembercache = MemcacheChain(
(localcache_cls(), self.mcrouter))
cache_chains.update(srmembercache=self.srmembercache)
if stalecaches:
self.relcache = StaleCacheChain(
localcache_cls(),
stalecaches,
self.mcrouter,
)
else:
self.relcache = MemcacheChain((localcache_cls(), self.mcrouter))
cache_chains.update(relcache=self.relcache)
self.ratelimitcache = MemcacheChain((localcache_cls(), self.mcrouter))
cache_chains.update(ratelimitcache=self.ratelimitcache)
# rendercache holds rendered partial templates.
self.rendercache = MemcacheChain((
localcache_cls(),
self.mcrouter,
))
cache_chains.update(rendercache=self.rendercache)
# commentpanecaches hold fully rendered comment panes
self.commentpanecache = MemcacheChain((
localcache_cls(),
self.mcrouter,
))
cache_chains.update(commentpanecache=self.commentpanecache)
# cassandra_local_cache is used for request-local caching in tdb_cassandra
self.cassandra_local_cache = localcache_cls()
cache_chains.update(cassandra_local_cache=self.cassandra_local_cache)
if stalecaches:
permacache_cache = StaleCacheChain(
localcache_cls(),
stalecaches,
permacache_memcaches,
)
else:
permacache_cache = CacheChain(
(localcache_cls(), permacache_memcaches), )
cache_chains.update(permacache=permacache_cache)
self.permacache = Permacache(
permacache_cache,
permacache_cf,
lock_factory=self.make_lock,
)
# hardcache is used for various things that tend to expire
# TODO: replace hardcache w/ cassandra stuff
self.hardcache = HardcacheChain(
(localcache_cls(), HardCache(self)),
cache_negative_results=True,
)
cache_chains.update(hardcache=self.hardcache)
# I know this sucks, but we need non-request-threads to be
# able to reset the caches, so we need them be able to close
# around 'cache_chains' without being able to call getattr on
# 'g'
def reset_caches():
for name, chain in cache_chains.iteritems():
if isinstance(chain, TransitionalCache):
chain = chain.read_chain
chain.reset()
if isinstance(chain, LocalCache):
continue
elif isinstance(chain, StaleCacheChain):
chain.stats = StaleCacheStats(self.stats, name)
else:
chain.stats = CacheStats(self.stats, name)
self.cache_chains = cache_chains
self.reset_caches = reset_caches
self.reset_caches()
self.startup_timer.intermediate("cache_chains")
# try to set the source control revision numbers
self.versions = {}
r2_root = os.path.dirname(os.path.dirname(self.paths["root"]))
r2_gitdir = os.path.join(r2_root, ".git")
self.short_version = self.record_repo_version("r2", r2_gitdir)
if I18N_PATH:
i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git")
self.record_repo_version("i18n", i18n_git_path)
# Initialize the amqp module globals, start the worker, etc.
r2.lib.amqp.initialize(self)
self.events = EventQueue()
self.startup_timer.intermediate("revisions")
def setup_complete(self):
self.startup_timer.stop()
self.stats.flush()
if self.log_start:
self.log.error("%s:%s started %s at %s (took %.02fs)",
self.reddit_host, self.reddit_pid,
self.short_version,
datetime.now().strftime("%H:%M:%S"),
self.startup_timer.elapsed_seconds())
if einhorn.is_worker():
einhorn.ack_startup()
def record_repo_version(self, repo_name, git_dir):
"""Get the currently checked out git revision for a given repository,
record it in g.versions, and return the short version of the hash."""
try:
subprocess.check_output
except AttributeError:
# python 2.6 compat
pass
else:
try:
revision = subprocess.check_output(
["git", "--git-dir", git_dir, "rev-parse", "HEAD"])
except subprocess.CalledProcessError, e:
self.log.warning("Unable to fetch git revision: %r", e)
else:
def test_set(self):
self.assertEquals(set([]), ConfigValue.set(''))
self.assertEquals(set(['a', 'b']), ConfigValue.set('a, b'))
def load_environment(global_conf={}, app_conf={}, setup_globals=True):
# Setup our paths
root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = {'root': root_path,
'controllers': os.path.join(root_path, 'controllers'),
'templates': [os.path.join(root_path, 'templates')],
}
if ConfigValue.bool(global_conf.get('uncompressedJS')):
paths['static_files'] = os.path.join(root_path, 'public')
else:
paths['static_files'] = os.path.join(os.path.dirname(root_path), 'build/public')
config.init_app(global_conf, app_conf, package='r2',
template_engine='mako', paths=paths)
# don't put action arguments onto c automatically
config['pylons.c_attach_args'] = False
# when accessing non-existent attributes on c, return "" instead of dying
config['pylons.strict_c'] = False
g = config['pylons.g'] = Globals(global_conf, app_conf, paths)
if setup_globals:
g.setup()
g.plugins.declare_queues(g.queues)
g.plugins.load_plugins()
config['r2.plugins'] = g.plugins
g.startup_timer.intermediate("plugins")
config['pylons.h'] = r2.lib.helpers
config['routes.map'] = routing.make_map()
#override the default response options
config['pylons.response_options']['headers'] = {}
# when mako loads a previously compiled template file from its cache, it
# doesn't check that the original template path matches the current path.
# in the event that a new plugin defines a template overriding a reddit
# template, unless the mtime newer, mako doesn't update the compiled
# template. as a workaround, this makes mako store compiled templates with
# the original path in the filename, forcing it to update with the path.
if "cache_dir" in app_conf:
module_directory = os.path.join(app_conf['cache_dir'], 'templates')
def mako_module_path(filename, uri):
filename = filename.lstrip('/').replace('/', '-')
path = os.path.join(module_directory, filename + ".py")
return os.path.abspath(path)
else:
# we're probably in "paster run standalone" mode. we'll just avoid
# caching templates since we don't know where they should go.
module_directory = mako_module_path = None
# set up the templating system
config["pylons.g"].mako_lookup = TemplateLookup(
directories=paths["templates"],
error_handler=handle_mako_error,
module_directory=module_directory,
input_encoding="utf-8",
default_filters=["mako_websafe"],
filesystem_checks=getattr(g, "reload_templates", False),
imports=[
"from r2.lib.filters import websafe, unsafe, mako_websafe",
"from pylons import c, g, request",
"from pylons.i18n import _, ungettext",
],
modulename_callable=mako_module_path,
)
if setup_globals:
g.setup_complete()
def test_set_of(self):
self.assertEquals(set([]), ConfigValue.set_of(str)(''))
self.assertEquals(set(['a', 'b']), ConfigValue.set_of(str)('a, b, b'))
self.assertEquals(set(['a', 'b']),
ConfigValue.set_of(str, delim=':')('b : a : b'))
def test_str(self):
self.assertEquals('x', ConfigValue.str('x'))
def test_choice(self):
self.assertEquals(1, ConfigValue.choice(alpha=1)('alpha'))
self.assertEquals(2, ConfigValue.choice(alpha=1, beta=2)('beta'))
with self.assertRaises(ValueError):
ConfigValue.choice(alpha=1)('asdf')
def test_float(self):
self.assertEquals(3.0, ConfigValue.float('3'))
self.assertEquals(-3.0, ConfigValue.float('-3'))
with self.assertRaises(ValueError):
ConfigValue.float('asdf')
def load_db_params(self):
self.databases = tuple(ConfigValue.to_iter(self.config.raw_data["databases"]))
self.db_params = {}
self.predefined_type_ids = {}
if not self.databases:
return
if self.env == "unit_test":
from mock import MagicMock
return MagicMock()
dbm = db_manager.db_manager()
db_param_names = ("name", "db_host", "db_user", "db_pass", "db_port", "pool_size", "max_overflow")
for db_name in self.databases:
conf_params = ConfigValue.to_iter(self.config.raw_data[db_name + "_db"])
params = dict(zip(db_param_names, conf_params))
if params["db_user"] == "*":
params["db_user"] = self.db_user
if params["db_pass"] == "*":
params["db_pass"] = self.db_pass
if params["db_port"] == "*":
params["db_port"] = self.db_port
if params["pool_size"] == "*":
params["pool_size"] = self.db_pool_size
if params["max_overflow"] == "*":
params["max_overflow"] = self.db_pool_overflow_size
dbm.setup_db(db_name, g_override=self, **params)
self.db_params[db_name] = params
dbm.type_db = dbm.get_engine(self.config.raw_data["type_db"])
dbm.relation_type_db = dbm.get_engine(self.config.raw_data["rel_type_db"])
def split_flags(raw_params):
params = []
flags = {}
for param in raw_params:
if not param.startswith("!"):
params.append(param)
else:
key, sep, value = param[1:].partition("=")
if sep:
flags[key] = value
else:
flags[key] = True
return params, flags
prefix = "db_table_"
for k, v in self.config.raw_data.iteritems():
if not k.startswith(prefix):
continue
params, table_flags = split_flags(ConfigValue.to_iter(v))
name = k[len(prefix) :]
kind = params[0]
server_list = self.config.raw_data["db_servers_" + name]
engines, flags = split_flags(ConfigValue.to_iter(server_list))
typeid = table_flags.get("typeid")
if typeid:
self.predefined_type_ids[name] = int(typeid)
if kind == "thing":
dbm.add_thing(name, dbm.get_engines(engines), **flags)
elif kind == "relation":
dbm.add_relation(name, params[1], params[2], dbm.get_engines(engines), **flags)
return dbm
def test_tuple(self):
self.assertEquals((), ConfigValue.tuple(''))
self.assertEquals(('a', 'b'), ConfigValue.tuple('a, b'))
class Globals(object):
spec = {
ConfigValue.int: [
'db_pool_size',
'db_pool_overflow_size',
'page_cache_time',
'commentpane_cache_time',
'num_mc_clients',
'MAX_CAMPAIGNS_PER_LINK',
'MIN_DOWN_LINK',
'MIN_UP_KARMA',
'MIN_DOWN_KARMA',
'MIN_RATE_LIMIT_KARMA',
'MIN_RATE_LIMIT_COMMENT_KARMA',
'HOT_PAGE_AGE',
'QUOTA_THRESHOLD',
'ADMIN_COOKIE_TTL',
'ADMIN_COOKIE_MAX_IDLE',
'OTP_COOKIE_TTL',
'hsts_max_age',
'num_comments',
'max_comments',
'max_comments_gold',
'max_comment_parent_walk',
'max_sr_images',
'num_serendipity',
'sr_dropdown_threshold',
'comment_visits_period',
'butler_max_mentions',
'min_membership_create_community',
'bcrypt_work_factor',
'cassandra_pool_size',
'sr_banned_quota',
'sr_wikibanned_quota',
'sr_wikicontributor_quota',
'sr_moderator_invite_quota',
'sr_contributor_quota',
'sr_quota_time',
'sr_invite_limit',
'thumbnail_hidpi_scaling',
'wiki_keep_recent_days',
'wiki_max_page_length_bytes',
'wiki_max_page_name_length',
'wiki_max_page_separators',
'RL_RESET_MINUTES',
'RL_OAUTH_RESET_MINUTES',
'comment_karma_display_floor',
'link_karma_display_floor',
],
ConfigValue.float: [
'default_promote_bid',
'min_promote_bid',
'max_promote_bid',
'statsd_sample_rate',
'querycache_prune_chance',
'RL_AVG_REQ_PER_SEC',
'RL_OAUTH_AVG_REQ_PER_SEC',
'RL_LOGIN_AVG_PER_SEC',
],
ConfigValue.bool: [
'debug',
'log_start',
'sqlprinting',
'template_debug',
'reload_templates',
'uncompressedJS',
'css_killswitch',
'db_create_tables',
'disallow_db_writes',
'disable_ratelimit',
'amqp_logging',
'read_only_mode',
'disable_wiki',
'heavy_load_mode',
'disable_captcha',
'disable_ads',
'disable_require_admin_otp',
'trust_local_proxies',
'shard_link_vote_queues',
'shard_commentstree_queues',
'ENFORCE_RATELIMIT',
'RL_SITEWIDE_ENABLED',
'RL_OAUTH_SITEWIDE_ENABLED',
],
ConfigValue.tuple: [
'plugins',
'stalecaches',
'memcaches',
'lockcaches',
'permacache_memcaches',
'rendercaches',
'pagecaches',
'memoizecaches',
'srmembercaches',
'relcaches',
'ratelimitcaches',
'cassandra_seeds',
'automatic_reddits',
'hardcache_categories',
'case_sensitive_domains',
'known_image_domains',
'reserved_subdomains',
'offsite_subdomains',
'TRAFFIC_LOG_HOSTS',
'exempt_login_user_agents',
'timed_templates',
'autoexpand_media_types',
'multi_icons',
'hide_subscribers_srs',
],
ConfigValue.tuple_of(ConfigValue.int): [
'thumbnail_size',
],
ConfigValue.dict(ConfigValue.str, ConfigValue.int): [
'agents',
],
ConfigValue.str: [
'wiki_page_registration_info',
'wiki_page_privacy_policy',
'wiki_page_user_agreement',
'wiki_page_gold_bottlecaps',
'fraud_email',
'feedback_email',
'share_reply',
'nerds_email',
'community_email',
'smtp_server',
],
ConfigValue.choice(ONE=CL_ONE, QUORUM=CL_QUORUM): [
'cassandra_rcl',
'cassandra_wcl',
],
ConfigValue.timeinterval: [
'ARCHIVE_AGE',
"vote_queue_grace_period",
],
config_gold_price: [
'gold_month_price',
'gold_year_price',
'cpm_selfserve',
'cpm_selfserve_geotarget_metro',
'cpm_selfserve_collection',
],
}
live_config_spec = {
ConfigValue.bool: [
'frontend_logging',
],
ConfigValue.int: [
'captcha_exempt_comment_karma',
'captcha_exempt_link_karma',
'create_sr_account_age_days',
'create_sr_comment_karma',
'create_sr_link_karma',
'cflag_min_votes',
],
ConfigValue.float: [
'cflag_lower_bound',
'cflag_upper_bound',
'spotlight_interest_sub_p',
'spotlight_interest_nosub_p',
'gold_revenue_goal',
'invalid_key_sample_rate',
],
ConfigValue.tuple: [
'fastlane_links',
'listing_chooser_sample_multis',
'discovery_srs',
'proxy_gilding_accounts',
],
ConfigValue.str: [
'listing_chooser_gold_multi',
'listing_chooser_explore_sr',
],
ConfigValue.dict(ConfigValue.int, ConfigValue.float): [
'comment_tree_version_weights',
],
ConfigValue.messages: [
'welcomebar_messages',
'sidebar_message',
'gold_sidebar_message',
],
ConfigValue.dict(ConfigValue.str, ConfigValue.float): [
'pennies_per_server_second',
],
ConfigValue.dict(ConfigValue.str, ConfigValue.choice(**PERMISSIONS)): [
'employees',
],
}
def __init__(self, global_conf, app_conf, paths, **extra):
"""
Globals acts as a container for objects available throughout
the life of the application.
One instance of Globals is created by Pylons during
application initialization and is available during requests
via the 'g' variable.
``global_conf``
The same variable used throughout ``config/middleware.py``
namely, the variables from the ``[DEFAULT]`` section of the
configuration file.
``app_conf``
The same ``kw`` dictionary used throughout
``config/middleware.py`` namely, the variables from the
section in the config file for your application.
``extra``
The configuration returned from ``load_config`` in
``config/middleware.py`` which may be of use in the setup of
your global variables.
"""
global_conf.setdefault("debug", False)
# reloading site ensures that we have a fresh sys.path to build our
# working set off of. this means that forked worker processes won't get
# the sys.path that was current when the master process was spawned
# meaning that new plugins will be picked up on regular app reload
# rather than having to restart the master process as well.
reload(site)
self.pkg_resources_working_set = pkg_resources.WorkingSet()
self.config = ConfigValueParser(global_conf)
self.config.add_spec(self.spec)
self.plugins = PluginLoader(self.pkg_resources_working_set,
self.config.get("plugins", []))
self.stats = Stats(self.config.get('statsd_addr'),
self.config.get('statsd_sample_rate'))
self.startup_timer = self.stats.get_timer("app_startup")
self.startup_timer.start()
self.paths = paths
self.running_as_script = global_conf.get('running_as_script', False)
# turn on for language support
self.lang = getattr(self, 'site_lang', 'en')
self.languages, self.lang_name = \
get_active_langs(default_lang=self.lang)
all_languages = self.lang_name.keys()
all_languages.sort()
self.all_languages = all_languages
# set default time zone if one is not set
tz = global_conf.get('timezone', 'UTC')
self.tz = pytz.timezone(tz)
dtz = global_conf.get('display_timezone', tz)
self.display_tz = pytz.timezone(dtz)
self.startup_timer.intermediate("init")
def __getattr__(self, name):
if not name.startswith('_') and name in self.config:
return self.config[name]
else:
raise AttributeError
def setup(self):
self.queues = queues.declare_queues(self)
self.extension_subdomains = dict(
m="mobile",
i="compact",
api="api",
rss="rss",
xml="xml",
json="json",
)
################# PROVIDERS
self.auth_provider = select_provider(
self.config,
self.pkg_resources_working_set,
"r2.provider.auth",
self.authentication_provider,
)
self.media_provider = select_provider(
self.config,
self.pkg_resources_working_set,
"r2.provider.media",
self.media_provider,
)
self.cdn_provider = select_provider(
self.config,
self.pkg_resources_working_set,
"r2.provider.cdn",
self.cdn_provider,
)
self.startup_timer.intermediate("providers")
################# CONFIGURATION
# AMQP is required
if not self.amqp_host:
raise ValueError("amqp_host not set in the .ini")
if not self.cassandra_seeds:
raise ValueError("cassandra_seeds not set in the .ini")
# heavy load mode is read only mode with a different infobar
if self.heavy_load_mode:
self.read_only_mode = True
origin_prefix = self.domain_prefix + "." if self.domain_prefix else ""
self.origin = "http://" + origin_prefix + self.domain
self.trusted_domains = set([self.domain])
if self.https_endpoint:
https_url = urlparse(self.https_endpoint)
self.trusted_domains.add(https_url.hostname)
# load the unique hashed names of files under static
static_files = os.path.join(self.paths.get('static_files'), 'static')
names_file_path = os.path.join(static_files, 'names.json')
if os.path.exists(names_file_path):
with open(names_file_path) as handle:
self.static_names = json.load(handle)
else:
self.static_names = {}
# make python warnings go through the logging system
logging.captureWarnings(capture=True)
log = logging.getLogger('reddit')
# when we're a script (paster run) just set up super simple logging
if self.running_as_script:
log.setLevel(logging.INFO)
log.addHandler(logging.StreamHandler())
# if in debug mode, override the logging level to DEBUG
if self.debug:
log.setLevel(logging.DEBUG)
# attempt to figure out which pool we're in and add that to the
# LogRecords.
try:
with open("/etc/ec2_asg", "r") as f:
pool = f.read().strip()
# clean up the pool name since we're putting stuff after "-"
pool = pool.partition("-")[0]
except IOError:
pool = "reddit-app"
self.log = logging.LoggerAdapter(log, {"pool": pool})
# set locations
locations = pkg_resources.resource_stream(__name__,
"../data/locations.json")
self.locations = json.loads(locations.read())
if not self.media_domain:
self.media_domain = self.domain
if self.media_domain == self.domain:
print >> sys.stderr, (
"Warning: g.media_domain == g.domain. " +
"This may give untrusted content access to user cookies")
if self.oauth_domain == self.domain:
print >> sys.stderr, ("Warning: g.oauth_domain == g.domain. "
"CORS requests to g.domain will be allowed")
for arg in sys.argv:
tokens = arg.split("=")
if len(tokens) == 2:
k, v = tokens
self.log.debug("Overriding g.%s to %s" % (k, v))
setattr(self, k, v)
self.reddit_host = socket.gethostname()
self.reddit_pid = os.getpid()
if hasattr(signal, 'SIGUSR1'):
# not all platforms have user signals
signal.signal(signal.SIGUSR1, thread_dump)
locale.setlocale(locale.LC_ALL, self.locale)
# Pre-calculate ratelimit values
self.RL_RESET_SECONDS = self.config["RL_RESET_MINUTES"] * 60
self.RL_MAX_REQS = int(self.config["RL_AVG_REQ_PER_SEC"] *
self.RL_RESET_SECONDS)
self.RL_OAUTH_RESET_SECONDS = self.config["RL_OAUTH_RESET_MINUTES"] * 60
self.RL_OAUTH_MAX_REQS = int(self.config["RL_OAUTH_AVG_REQ_PER_SEC"] *
self.RL_OAUTH_RESET_SECONDS)
self.RL_LOGIN_MAX_REQS = int(self.config["RL_LOGIN_AVG_PER_SEC"] *
self.RL_RESET_SECONDS)
self.startup_timer.intermediate("configuration")
################# ZOOKEEPER
# for now, zookeeper will be an optional part of the stack.
# if it's not configured, we will grab the expected config from the
# [live_config] section of the ini file
zk_hosts = self.config.get("zookeeper_connection_string")
if zk_hosts:
from r2.lib.zookeeper import (connect_to_zookeeper, LiveConfig,
LiveList)
zk_username = self.config["zookeeper_username"]
zk_password = self.config["zookeeper_password"]
self.zookeeper = connect_to_zookeeper(zk_hosts,
(zk_username, zk_password))
self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE)
self.secrets = fetch_secrets(self.zookeeper)
self.throttles = LiveList(self.zookeeper,
"/throttles",
map_fn=ipaddress.ip_network,
reduce_fn=ipaddress.collapse_addresses)
# close our zk connection when the app shuts down
SHUTDOWN_CALLBACKS.append(self.zookeeper.stop)
else:
self.zookeeper = None
parser = ConfigParser.RawConfigParser()
parser.optionxform = str
parser.read([self.config["__file__"]])
self.live_config = extract_live_config(parser, self.plugins)
self.secrets = extract_secrets(parser)
self.throttles = tuple() # immutable since it's not real
self.startup_timer.intermediate("zookeeper")
################# PRIVILEGED USERS
self.admins = PermissionFilteredEmployeeList(self.live_config,
type="admin")
self.sponsors = PermissionFilteredEmployeeList(self.live_config,
type="sponsor")
self.employees = PermissionFilteredEmployeeList(self.live_config,
type="employee")
################# MEMCACHE
num_mc_clients = self.num_mc_clients
# the main memcache pool. used for most everything.
memcache = CMemcache(
self.memcaches,
min_compress_len=1400,
num_clients=num_mc_clients,
binary=True,
)
# a pool just used for @memoize results
memoizecaches = CMemcache(
self.memoizecaches,
min_compress_len=50 * 1024,
num_clients=num_mc_clients,
binary=True,
)
# a pool just for srmember rels
srmembercaches = CMemcache(
self.srmembercaches,
min_compress_len=96,
num_clients=num_mc_clients,
binary=True,
)
# a pool just for rels
relcaches = CMemcache(
self.relcaches,
min_compress_len=96,
num_clients=num_mc_clients,
binary=True,
)
ratelimitcaches = CMemcache(
self.ratelimitcaches,
min_compress_len=96,
num_clients=num_mc_clients,
)
# a smaller pool of caches used only for distributed locks.
# TODO: move this to ZooKeeper
self.lock_cache = CMemcache(self.lockcaches,
binary=True,
num_clients=num_mc_clients)
self.make_lock = make_lock_factory(self.lock_cache, self.stats)
# memcaches used in front of the permacache CF in cassandra.
# XXX: this is a legacy thing; permacache was made when C* didn't have
# a row cache.
permacache_memcaches = CMemcache(self.permacache_memcaches,
min_compress_len=1400,
num_clients=num_mc_clients)
# the stalecache is a memcached local to the current app server used
# for data that's frequently fetched but doesn't need to be fresh.
if self.stalecaches:
stalecaches = CMemcache(self.stalecaches,
binary=True,
num_clients=num_mc_clients)
else:
stalecaches = None
# rendercache holds rendered partial templates.
rendercaches = CMemcache(
self.rendercaches,
noreply=True,
no_block=True,
num_clients=num_mc_clients,
min_compress_len=480,
)
# pagecaches hold fully rendered pages
pagecaches = CMemcache(
self.pagecaches,
noreply=True,
no_block=True,
num_clients=num_mc_clients,
min_compress_len=1400,
)
self.startup_timer.intermediate("memcache")
################# CASSANDRA
keyspace = "reddit"
self.cassandra_pools = {
"main":
StatsCollectingConnectionPool(keyspace,
stats=self.stats,
logging_name="main",
server_list=self.cassandra_seeds,
pool_size=self.cassandra_pool_size,
timeout=4,
max_retries=3,
prefill=False),
}
permacache_cf = Permacache._setup_column_family(
'permacache',
self.cassandra_pools[self.cassandra_default_pool],
)
self.startup_timer.intermediate("cassandra")
################# POSTGRES
self.dbm = self.load_db_params()
self.startup_timer.intermediate("postgres")
################# CHAINS
# initialize caches. Any cache-chains built here must be added
# to cache_chains (closed around by reset_caches) so that they
# can properly reset their local components
cache_chains = {}
localcache_cls = (SelfEmptyingCache
if self.running_as_script else LocalCache)
if stalecaches:
self.cache = StaleCacheChain(
localcache_cls(),
stalecaches,
memcache,
)
else:
self.cache = MemcacheChain((localcache_cls(), memcache))
cache_chains.update(cache=self.cache)
if stalecaches:
self.memoizecache = StaleCacheChain(
localcache_cls(),
stalecaches,
memoizecaches,
)
else:
self.memoizecache = MemcacheChain(
(localcache_cls(), memoizecaches))
cache_chains.update(memoizecache=self.memoizecache)
if stalecaches:
self.srmembercache = StaleCacheChain(
localcache_cls(),
stalecaches,
srmembercaches,
)
else:
self.srmembercache = MemcacheChain(
(localcache_cls(), srmembercaches))
cache_chains.update(srmembercache=self.srmembercache)
if stalecaches:
self.relcache = StaleCacheChain(
localcache_cls(),
stalecaches,
relcaches,
)
else:
self.relcache = MemcacheChain((localcache_cls(), relcaches))
cache_chains.update(relcache=self.relcache)
self.ratelimitcache = MemcacheChain(
(localcache_cls(), ratelimitcaches))
cache_chains.update(ratelimitcache=self.ratelimitcache)
self.rendercache = MemcacheChain((
localcache_cls(),
rendercaches,
))
cache_chains.update(rendercache=self.rendercache)
self.pagecache = MemcacheChain((
localcache_cls(),
pagecaches,
))
cache_chains.update(pagecache=self.pagecache)
# the thing_cache is used in tdb_cassandra.
self.thing_cache = CacheChain((localcache_cls(), ), check_keys=False)
cache_chains.update(thing_cache=self.thing_cache)
if stalecaches:
permacache_cache = StaleCacheChain(
localcache_cls(),
stalecaches,
permacache_memcaches,
check_keys=False,
)
else:
permacache_cache = CacheChain(
(localcache_cls(), permacache_memcaches),
check_keys=False,
)
cache_chains.update(permacache=permacache_cache)
self.permacache = Permacache(
permacache_cache,
permacache_cf,
lock_factory=self.make_lock,
)
# hardcache is used for various things that tend to expire
# TODO: replace hardcache w/ cassandra stuff
self.hardcache = HardcacheChain(
(localcache_cls(), memcache, HardCache(self)),
cache_negative_results=True,
)
cache_chains.update(hardcache=self.hardcache)
# I know this sucks, but we need non-request-threads to be
# able to reset the caches, so we need them be able to close
# around 'cache_chains' without being able to call getattr on
# 'g'
def reset_caches():
for name, chain in cache_chains.iteritems():
chain.reset()
if isinstance(chain, StaleCacheChain):
chain.stats = StaleCacheStats(self.stats, name)
else:
chain.stats = CacheStats(self.stats, name)
self.cache_chains = cache_chains
self.reset_caches = reset_caches
self.reset_caches()
self.startup_timer.intermediate("cache_chains")
# try to set the source control revision numbers
self.versions = {}
r2_root = os.path.dirname(os.path.dirname(self.paths["root"]))
r2_gitdir = os.path.join(r2_root, ".git")
self.short_version = self.record_repo_version("r2", r2_gitdir)
if I18N_PATH:
i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git")
self.record_repo_version("i18n", i18n_git_path)
self.startup_timer.intermediate("revisions")
def setup_complete(self):
self.startup_timer.stop()
self.stats.flush()
if self.log_start:
self.log.error("%s:%s started %s at %s (took %.02fs)",
self.reddit_host, self.reddit_pid,
self.short_version,
datetime.now().strftime("%H:%M:%S"),
self.startup_timer.elapsed_seconds())
def record_repo_version(self, repo_name, git_dir):
"""Get the currently checked out git revision for a given repository,
record it in g.versions, and return the short version of the hash."""
try:
subprocess.check_output
except AttributeError:
# python 2.6 compat
pass
else:
try:
revision = subprocess.check_output(
["git", "--git-dir", git_dir, "rev-parse", "HEAD"])
except subprocess.CalledProcessError, e:
self.log.warning("Unable to fetch git revision: %r", e)
else:
def test_set_of(self):
self.assertEquals(set([]), ConfigValue.set_of(str)(''))
self.assertEquals(set(['a', 'b']), ConfigValue.set_of(str)('a, b, b'))
self.assertEquals(set(['a', 'b']),
ConfigValue.set_of(str, delim=':')('b : a : b'))
class FreeToPlay(Plugin):
needs_static_build = True
config = {
ConfigValue.tuple: [
"f2pcaches",
],
ConfigValue.dict(str, str): [
"team_subreddits",
"steam_promo_items",
],
}
js = {
'reddit': Module('reddit.js',
'lib/iso8601.js',
'f2p/scrollupdater.js',
'f2p/f2p.js',
'f2p/utils.js',
'f2p/items.js',
TemplateFileSource('f2p/panel.html'),
TemplateFileSource('f2p/login-message.html'),
TemplateFileSource('f2p/item.html'),
TemplateFileSource('f2p/item-bubble.html'),
TemplateFileSource('f2p/scores.html'),
TemplateFileSource('f2p/target-overlay.html'),
)
}
live_config = {
ConfigValue.float: [
'drop_cooldown_mu',
'drop_cooldown_sigma',
],
ConfigValue.dict(str, int): [
'f2p_rarity_weights',
],
}
def declare_queues(self, queues):
# imported here so we don't depend on pyx files at import time
# which allows "make" to work in a clean clone of the repos
from r2.config.queues import MessageQueue
queues.declare({
"steam_q": MessageQueue(bind_to_self=True),
})
def on_load(self, g):
from r2.lib.cache import CMemcache, MemcacheChain, LocalCache
# TODO: use SelfEmptyingCache for localcache if we use this in jobs
f2p_memcaches = CMemcache(
'f2p',
g.f2pcaches,
num_clients=g.num_mc_clients,
)
g.f2pcache = MemcacheChain((
LocalCache(),
f2p_memcaches,
))
g.cache_chains.update(f2p=g.f2pcache)
compendium = pkg_resources.resource_stream(__name__,
"data/compendium.json")
g.f2pitems = json.load(compendium)
for kind, data in g.f2pitems.iteritems():
data["kind"] = kind
def add_routes(self, mc):
mc('/f2p/gamelog', controller='gamelog', action='listing')
mc('/api/f2p/:action', controller='freetoplayapi')
mc('/f2p/steam/:action', controller='steam', action='start')
def load_controllers(self):
from r2.lib.pages import Reddit
Reddit.extra_stylesheets.append('f2p.less')
from reddit_f2p import f2p
f2p.hooks.register_all()
f2p.monkeypatch()
from reddit_f2p.steam import SteamController
from reddit_f2p.gamelog import GameLogController
def test_tuple_of(self):
self.assertEquals((), ConfigValue.tuple_of(str)(''))
self.assertEquals(('a', 'b'), ConfigValue.tuple_of(str)('a, b'))
self.assertEquals(('a', 'b'),
ConfigValue.tuple_of(str, delim=':')('a : b'))
class Adzerk(Plugin):
needs_static_build = True
config = {
ConfigValue.str: [
'adzerk_engine_domain',
],
ConfigValue.int: [
'az_selfserve_salesperson_id',
'az_selfserve_network_id',
'az_reporting_timeout',
],
ConfigValue.float: [
'display_ad_skip_probability',
],
ConfigValue.tuple: [
'display_ad_skip_keywords',
],
ConfigValue.dict(ConfigValue.str, ConfigValue.int): [
'az_selfserve_priorities',
'az_selfserve_site_ids',
],
ConfigValue.tuple_of(ConfigValue.int): [
'adserver_campaign_ids',
],
}
js = {
'reddit-init': Module('reddit-init.js',
'adzerk/adzerk.js',
),
'display': Module('display.js',
'adzerk/display.js',
),
'companion': Module('companion.js',
'adzerk/companion.js',
),
'ad-dependencies': Module('ad-dependencies.js',
'adzerk/jquery.js',
),
}
def add_routes(self, mc):
mc('/api/request_promo/', controller='adzerkapi', action='request_promo')
mc('/ads/display/300x250/', controller='adserving', action='ad_300_250')
mc('/ads/display/300x250-companion/', controller='adserving', action='ad_300_250_companion')
def declare_queues(self, queues):
from r2.config.queues import MessageQueue
queues.declare({
"adzerk_q": MessageQueue(bind_to_self=True),
"adzerk_reporting_q": MessageQueue(bind_to_self=True),
})
def load_controllers(self):
# replace the standard Ads view with an Adzerk specific one.
import r2.lib.pages.pages
from adzerkads import Ads as AdzerkAds
r2.lib.pages.pages.Ads = AdzerkAds
# replace standard adserver with Adzerk.
from adzerkpromote import AdzerkApiController
from adzerkpromote import hooks as adzerkpromote_hooks
from adzerkads import AdServingController
adzerkpromote_hooks.register_all()
def test_timeinterval(self):
self.assertEquals(datetime.timedelta(0, 60),
ConfigValue.timeinterval('1 minute'))
with self.assertRaises(KeyError):
ConfigValue.timeinterval('asdf')
def test_choice(self):
self.assertEquals(1, ConfigValue.choice(alpha=1)('alpha'))
self.assertEquals(2, ConfigValue.choice(alpha=1, beta=2)('beta'))
with self.assertRaises(ValueError):
ConfigValue.choice(alpha=1)('asdf')
def load_db_params(self):
self.databases = tuple(ConfigValue.to_iter(self.config.raw_data['databases']))
self.db_params = {}
if not self.databases:
return
dbm = db_manager.db_manager()
db_param_names = ('name', 'db_host', 'db_user', 'db_pass', 'db_port',
'pool_size', 'max_overflow')
for db_name in self.databases:
conf_params = ConfigValue.to_iter(self.config.raw_data[db_name + '_db'])
params = dict(zip(db_param_names, conf_params))
if params['db_user'] == "*":
params['db_user'] = self.db_user
if params['db_pass'] == "*":
params['db_pass'] = self.db_pass
if params['db_port'] == "*":
params['db_port'] = self.db_port
if params['pool_size'] == "*":
params['pool_size'] = self.db_pool_size
if params['max_overflow'] == "*":
params['max_overflow'] = self.db_pool_overflow_size
dbm.setup_db(db_name, g_override=self, **params)
self.db_params[db_name] = params
dbm.type_db = dbm.get_engine(self.config.raw_data['type_db'])
dbm.relation_type_db = dbm.get_engine(self.config.raw_data['rel_type_db'])
def split_flags(raw_params):
params = []
flags = {}
for param in raw_params:
if not param.startswith("!"):
params.append(param)
else:
key, sep, value = param[1:].partition("=")
if sep:
flags[key] = value
else:
flags[key] = True
return params, flags
prefix = 'db_table_'
self.predefined_type_ids = {}
for k, v in self.config.raw_data.iteritems():
if not k.startswith(prefix):
continue
params, table_flags = split_flags(ConfigValue.to_iter(v))
name = k[len(prefix):]
kind = params[0]
server_list = self.config.raw_data["db_servers_" + name]
engines, flags = split_flags(ConfigValue.to_iter(server_list))
typeid = table_flags.get("typeid")
if typeid:
self.predefined_type_ids[name] = int(typeid)
if kind == 'thing':
dbm.add_thing(name, dbm.get_engines(engines),
**flags)
elif kind == 'relation':
dbm.add_relation(name, params[1], params[2],
dbm.get_engines(engines),
**flags)
return dbm
def test_timeinterval(self):
self.assertEquals(datetime.timedelta(0, 60),
ConfigValue.timeinterval('1 minute'))
with self.assertRaises(KeyError):
ConfigValue.timeinterval('asdf')
def load_db_params(self):
self.databases = tuple(
ConfigValue.to_iter(self.config.raw_data['databases']))
self.db_params = {}
if not self.databases:
return
dbm = db_manager.db_manager()
db_param_names = ('name', 'db_host', 'db_user', 'db_pass', 'db_port',
'pool_size', 'max_overflow')
for db_name in self.databases:
conf_params = ConfigValue.to_iter(self.config.raw_data[db_name +
'_db'])
params = dict(zip(db_param_names, conf_params))
if params['db_user'] == "*":
params['db_user'] = self.db_user
if params['db_pass'] == "*":
params['db_pass'] = self.db_pass
if params['db_port'] == "*":
params['db_port'] = self.db_port
if params['pool_size'] == "*":
params['pool_size'] = self.db_pool_size
if params['max_overflow'] == "*":
params['max_overflow'] = self.db_pool_overflow_size
dbm.setup_db(db_name, g_override=self, **params)
self.db_params[db_name] = params
dbm.type_db = dbm.get_engine(self.config.raw_data['type_db'])
dbm.relation_type_db = dbm.get_engine(
self.config.raw_data['rel_type_db'])
def split_flags(raw_params):
params = []
flags = {}
for param in raw_params:
if not param.startswith("!"):
params.append(param)
else:
key, sep, value = param[1:].partition("=")
if sep:
flags[key] = value
else:
flags[key] = True
return params, flags
prefix = 'db_table_'
self.predefined_type_ids = {}
for k, v in self.config.raw_data.iteritems():
if not k.startswith(prefix):
continue
params, table_flags = split_flags(ConfigValue.to_iter(v))
name = k[len(prefix):]
kind = params[0]
server_list = self.config.raw_data["db_servers_" + name]
engines, flags = split_flags(ConfigValue.to_iter(server_list))
typeid = table_flags.get("typeid")
if typeid:
self.predefined_type_ids[name] = int(typeid)
if kind == 'thing':
dbm.add_thing(name, dbm.get_engines(engines), **flags)
elif kind == 'relation':
dbm.add_relation(name, params[1], params[2],
dbm.get_engines(engines), **flags)
return dbm
class Robin(Plugin):
needs_static_build = True
js = {
"robin": LocalizedModule("robin.js",
"lib/page-visibility.js",
"lib/tinycon.js",
"websocket.js",
TemplateFileSource("robin/robinmessage.html"),
TemplateFileSource("robin/robinroomparticipant.html"),
"errors.js",
"models/validators.js",
"robin/models.js",
"robin/views.js",
"robin/notifications.js",
"robin/favicon.js",
"robin/init.js",
),
"robin-join": Module("robin-join.js",
"robin/join.js",
),
}
live_config = {
ConfigValue.int: [
"robin_ratelimit_window",
],
ConfigValue.dict(ConfigValue.int, ConfigValue.float): [
"robin_ratelimit_avg_per_sec",
],
}
def declare_queues(self, queues):
from r2.config.queues import MessageQueue
queues.declare({
"robin_presence_q": MessageQueue(),
"robin_waitinglist_q": MessageQueue(bind_to_self=True),
"robin_subreddit_maker_q": MessageQueue(bind_to_self=True),
})
queues.robin_presence_q << (
"websocket.connect",
"websocket.disconnect",
)
def add_routes(self, mc):
mc("/robin", controller="robin", action="chat",
conditions={"function": not_in_sr})
mc("/robin/all", controller="robin", action="all",
conditions={"function": not_in_sr})
mc("/robin/admin", controller="robin", action="admin",
conditions={"function": not_in_sr})
mc("/robin/join", controller="robin", action="join",
conditions={"function": not_in_sr})
mc("/robin/:room_id", controller="robin", action="force_room",
conditions={"function": not_in_sr})
mc("/robin/user/:user", controller="robin", action="user_room",
conditions={"function": not_in_sr})
mc("/api/robin/:room_id/:action", controller="robin",
conditions={"function": not_in_sr})
mc("/api/join_room", controller="robin", action="join_room",
conditions={"function": not_in_sr})
mc("/api/room_assignment", controller="robin", action="room_assignment",
conditions={"function": not_in_sr})
mc("/api/admin_prompt", controller="robin", action="admin_prompt",
conditions={"function": not_in_sr})
mc("/api/admin_reap", controller="robin", action="admin_reap",
conditions={"function": not_in_sr})
mc("/api/admin_broadcast", controller="robin", action="admin_broadcast",
conditions={"function": not_in_sr})
def load_controllers(self):
from r2.lib.pages import Reddit
from reddit_robin.controllers import (
RobinController,
)
Reddit.extra_stylesheets.append('robin_global.less')
from reddit_robin.hooks import hooks
hooks.register_all()
