def load_environment(global_conf={}, app_conf={}, setup_globals=True): # Setup our paths root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) paths = {'root': root_path, 'controllers': os.path.join(root_path, 'controllers'), 'templates': [os.path.join(root_path, 'templates')], } if ConfigValue.bool(global_conf.get('uncompressedJS')): paths['static_files'] = os.path.join(root_path, 'public') else: paths['static_files'] = os.path.join(os.path.dirname(root_path), 'build/public') config.init_app(global_conf, app_conf, package='r2', template_engine='mako', paths=paths) g = config['pylons.g'] = Globals(global_conf, app_conf, paths) if setup_globals: g.setup() g.plugins.declare_queues(g.queues) r2.config.cache = g.cache g.plugins.load_plugins() config['r2.plugins'] = g.plugins g.startup_timer.intermediate("plugins") config['pylons.h'] = r2.lib.helpers config['routes.map'] = routing.make_map() #override the default response options config['pylons.response_options']['headers'] = {} # The following template options are passed to your template engines tmpl_options = config['buffet.template_options'] tmpl_options['mako.filesystem_checks'] = getattr(g, 'reload_templates', False) tmpl_options['mako.default_filters'] = ["mako_websafe"] tmpl_options['mako.imports'] = \ ["from r2.lib.filters import websafe, unsafe, mako_websafe", "from pylons import c, g, request", "from pylons.i18n import _, ungettext"] # when mako loads a previously compiled template file from its cache, it # doesn't check that the original template path matches the current path. # in the event that a new plugin defines a template overriding a reddit # template, unless the mtime newer, mako doesn't update the compiled # template. as a workaround, this makes mako store compiled templates with # the original path in the filename, forcing it to update with the path. def mako_module_path(filename, uri): module_directory = tmpl_options['mako.module_directory'] filename = filename.lstrip('/').replace('/', '-') path = os.path.join(module_directory, filename + ".py") return os.path.abspath(path) tmpl_options['mako.modulename_callable'] = mako_module_path if setup_globals: g.setup_complete()
def load_environment(global_conf={}, app_conf={}, setup_globals=True): # Setup our paths root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) paths = { 'root': root_path, 'controllers': os.path.join(root_path, 'controllers'), 'templates': tmpl_dirs, } if ConfigValue.bool(global_conf.get('uncompressedJS')): paths['static_files'] = os.path.join(root_path, 'public') else: paths['static_files'] = os.path.join(os.path.dirname(root_path), 'build/public') config.init_app(global_conf, app_conf, package='r2', template_engine='mako', paths=paths) g = config['pylons.g'] = Globals(global_conf, app_conf, paths) if setup_globals: g.setup() r2.config.cache = g.cache config['pylons.h'] = r2.lib.helpers g.plugins = config['r2.plugins'] = PluginLoader().load_plugins( g.config.get('plugins', [])) config['routes.map'] = routing.make_map() #override the default response options config['pylons.response_options']['headers'] = {} # The following template options are passed to your template engines #tmpl_options = {} #tmpl_options['myghty.log_errors'] = True #tmpl_options['myghty.escapes'] = dict(l=webhelpers.auto_link, s=webhelpers.simple_format) tmpl_options = config['buffet.template_options'] tmpl_options['mako.filesystem_checks'] = getattr(g, 'reload_templates', False) tmpl_options['mako.default_filters'] = ["mako_websafe"] tmpl_options['mako.imports'] = \ ["from r2.lib.filters import websafe, unsafe, mako_websafe", "from pylons import c, g, request", "from pylons.i18n import _, ungettext"]
def test_dict(self): self.assertEquals({}, ConfigValue.dict(str, str)('')) self.assertEquals({'a': ''}, ConfigValue.dict(str, str)('a')) self.assertEquals({'a': 3}, ConfigValue.dict(str, int)('a: 3')) self.assertEquals({'a': 3, 'b': 4}, ConfigValue.dict(str, int)('a: 3, b: 4')) self.assertEquals({'a': (3, 5), 'b': (4, 6)}, ConfigValue.dict( str, ConfigValue.tuple_of(int), delim=';') ('a: 3, 5; b: 4, 6'))
class Adzerk(Plugin): needs_static_build = True config = { ConfigValue.int: [ 'az_selfserve_site_id', 'az_selfserve_advertiser_id', 'az_selfserve_channel_id', 'az_selfserve_publisher_id', 'az_selfserve_network_id', 'az_selfserve_ad_type', 'az_selfserve_num_request', ], ConfigValue.dict(ConfigValue.str, ConfigValue.int): [ 'az_selfserve_priorities', ], } js = { 'reddit-init': Module( 'reddit-init.js', 'adzerk/adzerk.js', ) } def add_routes(self, mc): mc('/api/request_promo/', controller='adzerkapi', action='request_promo') def declare_queues(self, queues): from r2.config.queues import MessageQueue queues.declare({ "adzerk_q": MessageQueue(bind_to_self=True), }) def load_controllers(self): # replace the standard Ads view with an Adzerk specific one. import r2.lib.pages.pages from adzerkads import Ads as AdzerkAds r2.lib.pages.pages.Ads = AdzerkAds # replace standard adserver with Adzerk. from adzerkpromote import AdzerkApiController from adzerkpromote import hooks as adzerkpromote_hooks adzerkpromote_hooks.register_all()
def load_environment(global_conf={}, app_conf={}, setup_globals=True): # Setup our paths root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) paths = { "root": root_path, "controllers": os.path.join(root_path, "controllers"), "templates": [os.path.join(root_path, "templates")], } if ConfigValue.bool(global_conf.get("uncompressedJS")): paths["static_files"] = os.path.join(root_path, "public") else: paths["static_files"] = os.path.join(os.path.dirname(root_path), "build/public") config.init_app(global_conf, app_conf, package="r2", template_engine="mako", paths=paths) g = config["pylons.g"] = Globals(global_conf, app_conf, paths) if setup_globals: g.setup() r2.config.cache = g.cache g.plugins.load_plugins() config["r2.plugins"] = g.plugins config["pylons.h"] = r2.lib.helpers config["routes.map"] = routing.make_map() # override the default response options config["pylons.response_options"]["headers"] = {} # The following template options are passed to your template engines # tmpl_options = {} # tmpl_options['myghty.log_errors'] = True # tmpl_options['myghty.escapes'] = dict(l=webhelpers.auto_link, s=webhelpers.simple_format) tmpl_options = config["buffet.template_options"] tmpl_options["mako.filesystem_checks"] = getattr(g, "reload_templates", False) tmpl_options["mako.default_filters"] = ["mako_websafe"] tmpl_options["mako.imports"] = [ "from r2.lib.filters import websafe, unsafe, mako_websafe", "from pylons import c, g, request", "from pylons.i18n import _, ungettext", ]
def load_environment(global_conf={}, app_conf={}, setup_globals=True): # Setup our paths root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) paths = {'root': root_path, 'controllers': os.path.join(root_path, 'controllers'), 'templates': tmpl_dirs, } if ConfigValue.bool(global_conf.get('uncompressedJS')): paths['static_files'] = os.path.join(root_path, 'public') else: paths['static_files'] = os.path.join(os.path.dirname(root_path), 'build/public') config.init_app(global_conf, app_conf, package='r2', template_engine='mako', paths=paths) g = config['pylons.g'] = Globals(global_conf, app_conf, paths) if setup_globals: g.setup() reddit_config.cache = g.cache config['pylons.h'] = r2.lib.helpers g.plugins = config['r2.plugins'] = PluginLoader().load_plugins(g.config.get('plugins', [])) config['routes.map'] = routing.make_map() #override the default response options config['pylons.response_options']['headers'] = {} # The following template options are passed to your template engines #tmpl_options = {} #tmpl_options['myghty.log_errors'] = True #tmpl_options['myghty.escapes'] = dict(l=webhelpers.auto_link, s=webhelpers.simple_format) tmpl_options = config['buffet.template_options'] tmpl_options['mako.filesystem_checks'] = getattr(g, 'reload_templates', False) tmpl_options['mako.default_filters'] = ["mako_websafe"] tmpl_options['mako.imports'] = \ ["from r2.lib.filters import websafe, unsafe, mako_websafe", "from pylons import c, g, request", "from pylons.i18n import _, ungettext"]
def test_dict(self): self.assertEquals({}, ConfigValue.dict(str, str)('')) self.assertEquals({'a': ''}, ConfigValue.dict(str, str)('a')) self.assertEquals({'a': 3}, ConfigValue.dict(str, int)('a: 3')) self.assertEquals({ 'a': 3, 'b': 4 }, ConfigValue.dict(str, int)('a: 3, b: 4')) self.assertEquals({ 'a': (3, 5), 'b': (4, 6) }, ConfigValue.dict(str, ConfigValue.tuple_of(int), delim=';')('a: 3, 5; b: 4, 6'))
class LiveUpdate(Plugin): needs_static_build = True errors = { "LIVEUPDATE_NOT_CONTRIBUTOR": N_("that user is not a contributor"), "LIVEUPDATE_NO_INVITE_FOUND": N_("there is no pending invite for that thread"), "LIVEUPDATE_TOO_MANY_INVITES": N_("there are too many pending invites outstanding"), "LIVEUPDATE_ALREADY_CONTRIBUTOR": N_("that user is already a contributor"), "LIVEUPDATE_LINK_IS_NOT_DISCUSSION": N_("the specified link is not a discussion about this live thread"), } config = { ConfigValue.int: [ "liveupdate_invite_quota", "liveupdate_min_score_for_discussions", ], ConfigValue.str: [ "liveupdate_pixel_domain", ], ConfigValue.baseplate(Date): [ "liveupdate_min_date_viewcounts", ], } js = { "liveupdate": LocalizedModule( "liveupdate.js", "lib/page-visibility.js", "lib/tinycon.js", "lib/moment.js", "websocket.js", "liveupdate/init.js", "liveupdate/activity.js", "liveupdate/embeds.js", "liveupdate/event.js", "liveupdate/favicon.js", "liveupdate/listings.js", "liveupdate/notifications.js", "liveupdate/statusBar.js", "liveupdate/report.js", TemplateFileSource("liveupdate/update.html"), TemplateFileSource("liveupdate/separator.html"), TemplateFileSource("liveupdate/edit-button.html"), TemplateFileSource("liveupdate/reported.html"), PermissionsDataSource({ "liveupdate_contributor": ContributorPermissionSet, "liveupdate_contributor_invite": ContributorPermissionSet, }), localized_appendices=[ MomentTranslations(), ], ), } def add_routes(self, mc): mc( "/live", controller="liveupdateevents", action="home", conditions={"function": not_in_sr}, ) mc( "/live/create", controller="liveupdateevents", action="create", conditions={"function": not_in_sr}, ) mc( "/api/live/by_id/:names", action="listing", controller="liveupdatebyid", ) mc( "/live/:filter", action="listing", controller="liveupdateevents", conditions={"function": not_in_sr}, requirements={ "filter": "open|closed|reported|active|happening_now|mine" }, ) mc( "/api/live/:action", controller="liveupdateevents", conditions={"function": not_in_sr}, requirements={"action": "create|happening_now"}, ) mc("/live/:event", controller="liveupdate", action="listing", conditions={"function": not_in_sr}, is_embed=False) mc("/live/:event/embed", controller="liveupdate", action="listing", conditions={"function": not_in_sr}, is_embed=True) mc( "/live/:event/updates/:target", controller="liveupdate", action="focus", conditions={"function": not_in_sr}, ) mc("/live/:event/pixel", controller="liveupdatepixel", action="pixel", conditions={"function": not_in_sr}) mc("/live/:event/:action", controller="liveupdate", conditions={"function": not_in_sr}) mc("/api/live/:event/:action", controller="liveupdate", conditions={"function": not_in_sr}) mc('/mediaembed/liveupdate/:event/:liveupdate/:embed_index', controller="liveupdateembed", action="mediaembed") mc('/admin/happening-now', controller='liveupdateadmin', action='happening_now') def load_controllers(self): from r2.controllers.api_docs import api_section, section_info api_section["live"] = "live" section_info["live"] = { "title": "live threads", "description": sys.modules[__name__].__doc__, } from r2.models.token import OAuth2Scope OAuth2Scope.scope_info["livemanage"] = { "id": "livemanage", "name": N_("Manage live threads"), "description": N_("Manage settings and contributors of live threads " "I contribute to."), } from reddit_liveupdate.controllers import ( controller_hooks, LiveUpdateByIDController, LiveUpdateController, LiveUpdateEventsController, LiveUpdatePixelController, ) from r2.config.templates import api from r2.lib.jsontemplates import ListingJsonTemplate from reddit_liveupdate import pages api('liveupdateeventapp', pages.LiveUpdateEventAppJsonTemplate) api('liveupdatefocusapp', pages.LiveUpdateEventAppJsonTemplate) api('liveupdateevent', pages.LiveUpdateEventJsonTemplate) api('liveupdatereportedeventrow', pages.LiveUpdateEventJsonTemplate) api('liveupdatefeaturedevent', pages.LiveUpdateFeaturedEventJsonTemplate) api('liveupdate', pages.LiveUpdateJsonTemplate) api('liveupdatecontributortableitem', pages.ContributorTableItemJsonTemplate) api('liveupdatediscussionslisting', ListingJsonTemplate) controller_hooks.register_all() from reddit_liveupdate import scraper scraper.hooks.register_all() def declare_queues(self, queues): from r2.config.queues import MessageQueue queues.declare({ "liveupdate_scraper_q": MessageQueue(bind_to_self=True), }) queues.liveupdate_scraper_q << ("new_liveupdate_update", ) source_root_url = "https://github.com/reddit/reddit-plugin-liveupdate/blob/master/reddit_liveupdate/" def get_documented_controllers(self): from reddit_liveupdate.controllers import ( LiveUpdateController, LiveUpdateEventsController, LiveUpdateByIDController, ) yield LiveUpdateController, "/api/live/{thread}" yield LiveUpdateEventsController, "" yield LiveUpdateByIDController, ""
def load_environment(global_conf={}, app_conf={}, setup_globals=True): r2_path = get_r2_path() root_path = os.path.join(r2_path, "r2") paths = { "root": root_path, "controllers": os.path.join(root_path, "controllers"), "templates": [os.path.join(root_path, "templates")], } if ConfigValue.bool(global_conf.get("uncompressedJS")): paths["static_files"] = get_raw_statics_path() else: paths["static_files"] = get_built_statics_path() config = PylonsConfig() config.init_app(global_conf, app_conf, package="r2", paths=paths) # don't put action arguments onto c automatically config["pylons.c_attach_args"] = False # when accessing non-existent attributes on c, return "" instead of dying config["pylons.strict_tmpl_context"] = False g = Globals(config, global_conf, app_conf, paths) config["pylons.app_globals"] = g if setup_globals: config["r2.import_private"] = ConfigValue.bool(global_conf["import_private"]) g.setup() g.plugins.declare_queues(g.queues) g.plugins.load_plugins(config) config["r2.plugins"] = g.plugins g.startup_timer.intermediate("plugins") config["pylons.h"] = r2.lib.helpers config["routes.map"] = make_map(config) # override the default response options config["pylons.response_options"]["headers"] = {} # when mako loads a previously compiled template file from its cache, it # doesn't check that the original template path matches the current path. # in the event that a new plugin defines a template overriding a reddit # template, unless the mtime newer, mako doesn't update the compiled # template. as a workaround, this makes mako store compiled templates with # the original path in the filename, forcing it to update with the path. if "cache_dir" in app_conf: module_directory = os.path.join(app_conf["cache_dir"], "templates") def mako_module_path(filename, uri): filename = filename.lstrip("/").replace("/", "-") path = os.path.join(module_directory, filename + ".py") return os.path.abspath(path) else: # disable caching templates since we don't know where they should go. module_directory = mako_module_path = None # set up the templating system config["pylons.app_globals"].mako_lookup = TemplateLookup( directories=paths["templates"], error_handler=handle_mako_error, module_directory=module_directory, input_encoding="utf-8", default_filters=["conditional_websafe"], filesystem_checks=getattr(g, "reload_templates", False), imports=[ "from r2.lib.filters import websafe, unsafe, conditional_websafe", "from pylons import request", "from pylons import tmpl_context as c", "from pylons import app_globals as g", "from pylons.i18n import _, ungettext", ], modulename_callable=mako_module_path, ) if setup_globals: g.setup_complete() return config
def test_set(self): self.assertEquals(set([]), ConfigValue.set('')) self.assertEquals(set(['a', 'b']), ConfigValue.set('a, b'))
def test_tuple_of(self): self.assertEquals((), ConfigValue.tuple_of(str)('')) self.assertEquals(('a', 'b'), ConfigValue.tuple_of(str)('a, b')) self.assertEquals(('a', 'b'), ConfigValue.tuple_of(str, delim=':')('a : b'))
def test_int(self): self.assertEquals(3, ConfigValue.int('3')) self.assertEquals(-3, ConfigValue.int('-3')) with self.assertRaises(ValueError): ConfigValue.int('asdf')
def test_bool(self): self.assertEquals(True, ConfigValue.bool('TrUe')) self.assertEquals(False, ConfigValue.bool('fAlSe')) with self.assertRaises(ValueError): ConfigValue.bool('asdf')
def test_tuple(self): self.assertEquals((), ConfigValue.tuple('')) self.assertEquals(('a', 'b'), ConfigValue.tuple('a, b'))
class Globals(object): spec = { ConfigValue.int: [ 'db_pool_size', 'db_pool_overflow_size', 'page_cache_time', 'commentpane_cache_time', 'num_mc_clients', 'MAX_CAMPAIGNS_PER_LINK', 'MIN_DOWN_LINK', 'MIN_UP_KARMA', 'MIN_DOWN_KARMA', 'MIN_RATE_LIMIT_KARMA', 'MIN_RATE_LIMIT_COMMENT_KARMA', 'VOTE_AGE_LIMIT', 'REPLY_AGE_LIMIT', 'REPORT_AGE_LIMIT', 'HOT_PAGE_AGE', 'RATELIMIT', 'QUOTA_THRESHOLD', 'ADMIN_COOKIE_TTL', 'ADMIN_COOKIE_MAX_IDLE', 'OTP_COOKIE_TTL', 'num_comments', 'max_comments', 'max_comments_gold', 'num_default_reddits', 'max_sr_images', 'num_serendipity', 'sr_dropdown_threshold', 'comment_visits_period', 'min_membership_create_community', 'bcrypt_work_factor', 'cassandra_pool_size', 'sr_banned_quota', 'sr_wikibanned_quota', 'sr_wikicontributor_quota', 'sr_moderator_invite_quota', 'sr_contributor_quota', 'sr_quota_time', 'wiki_keep_recent_days', 'wiki_max_page_length_bytes', 'wiki_max_page_name_length', 'wiki_max_page_separators', ], ConfigValue.float: [ 'min_promote_bid', 'max_promote_bid', 'statsd_sample_rate', 'querycache_prune_chance', ], ConfigValue.bool: [ 'debug', 'log_start', 'sqlprinting', 'template_debug', 'reload_templates', 'uncompressedJS', 'css_killswitch', 'db_create_tables', 'disallow_db_writes', 'disable_ratelimit', 'amqp_logging', 'read_only_mode', 'disable_wiki', 'heavy_load_mode', 's3_media_direct', 'disable_captcha', 'disable_ads', 'disable_require_admin_otp', 'static_pre_gzipped', 'static_secure_pre_gzipped', 'trust_local_proxies', 'shard_link_vote_queues', ], ConfigValue.tuple: [ 'plugins', 'stalecaches', 'memcaches', 'lockcaches', 'permacache_memcaches', 'rendercaches', 'pagecaches', 'cassandra_seeds', 'admins', 'sponsors', 'automatic_reddits', 'agents', 'allowed_css_linked_domains', 'authorized_cnames', 'hardcache_categories', 's3_media_buckets', 'allowed_pay_countries', 'case_sensitive_domains', 'reserved_subdomains', 'TRAFFIC_LOG_HOSTS', 'exempt_login_user_agents', 'timed_templates', ], ConfigValue.str: [ 'wiki_page_registration_info', 'wiki_page_privacy_policy', 'wiki_page_user_agreement', ], ConfigValue.choice: { 'cassandra_rcl': { 'ONE': CL_ONE, 'QUORUM': CL_QUORUM }, 'cassandra_wcl': { 'ONE': CL_ONE, 'QUORUM': CL_QUORUM }, }, config_gold_price: [ 'gold_month_price', 'gold_year_price', ], } live_config_spec = { ConfigValue.bool: [ 'frontpage_dart', ], ConfigValue.float: [ 'spotlight_interest_sub_p', 'spotlight_interest_nosub_p', ], ConfigValue.tuple: [ 'sr_discovery_links', 'fastlane_links', ], ConfigValue.dict(ConfigValue.int, ConfigValue.float): [ 'comment_tree_version_weights', ], ConfigValue.messages: [ 'goldvertisement_blurbs', 'goldvertisement_has_gold_blurbs', 'welcomebar_messages', 'sidebar_message', 'gold_sidebar_message', ], } def __init__(self, global_conf, app_conf, paths, **extra): """ Globals acts as a container for objects available throughout the life of the application. One instance of Globals is created by Pylons during application initialization and is available during requests via the 'g' variable. ``global_conf`` The same variable used throughout ``config/middleware.py`` namely, the variables from the ``[DEFAULT]`` section of the configuration file. ``app_conf`` The same ``kw`` dictionary used throughout ``config/middleware.py`` namely, the variables from the section in the config file for your application. ``extra`` The configuration returned from ``load_config`` in ``config/middleware.py`` which may be of use in the setup of your global variables. """ global_conf.setdefault("debug", False) self.config = ConfigValueParser(global_conf) self.config.add_spec(self.spec) self.plugins = PluginLoader(self.config.get("plugins", [])) self.stats = Stats(self.config.get('statsd_addr'), self.config.get('statsd_sample_rate')) self.startup_timer = self.stats.get_timer("app_startup") self.startup_timer.start() self.paths = paths self.running_as_script = global_conf.get('running_as_script', False) # turn on for language support self.lang = getattr(self, 'site_lang', 'en') self.languages, self.lang_name = \ get_active_langs(default_lang=self.lang) all_languages = self.lang_name.keys() all_languages.sort() self.all_languages = all_languages # set default time zone if one is not set tz = global_conf.get('timezone', 'UTC') self.tz = pytz.timezone(tz) dtz = global_conf.get('display_timezone', tz) self.display_tz = pytz.timezone(dtz) self.startup_timer.intermediate("init") def __getattr__(self, name): if not name.startswith('_') and name in self.config: return self.config[name] else: raise AttributeError def setup(self): self.queues = queues.declare_queues(self) ################# CONFIGURATION # AMQP is required if not self.amqp_host: raise ValueError("amqp_host not set in the .ini") if not self.cassandra_seeds: raise ValueError("cassandra_seeds not set in the .ini") # heavy load mode is read only mode with a different infobar if self.heavy_load_mode: self.read_only_mode = True origin_prefix = self.domain_prefix + "." if self.domain_prefix else "" self.origin = "http://" + origin_prefix + self.domain self.secure_domains = set([urlparse(self.payment_domain).netloc]) self.trusted_domains = set([self.domain]) self.trusted_domains.update(self.authorized_cnames) if self.https_endpoint: https_url = urlparse(self.https_endpoint) self.secure_domains.add(https_url.netloc) self.trusted_domains.add(https_url.hostname) if getattr(self, 'oauth_domain', None): self.secure_domains.add(self.oauth_domain) # load the unique hashed names of files under static static_files = os.path.join(self.paths.get('static_files'), 'static') names_file_path = os.path.join(static_files, 'names.json') if os.path.exists(names_file_path): with open(names_file_path) as handle: self.static_names = json.load(handle) else: self.static_names = {} # make python warnings go through the logging system logging.captureWarnings(capture=True) log = logging.getLogger('reddit') # when we're a script (paster run) just set up super simple logging if self.running_as_script: log.setLevel(logging.INFO) log.addHandler(logging.StreamHandler()) # if in debug mode, override the logging level to DEBUG if self.debug: log.setLevel(logging.DEBUG) # attempt to figure out which pool we're in and add that to the # LogRecords. try: with open("/etc/ec2_asg", "r") as f: pool = f.read().strip() # clean up the pool name since we're putting stuff after "-" pool = pool.partition("-")[0] except IOError: pool = "reddit-app" self.log = logging.LoggerAdapter(log, {"pool": pool}) # make cssutils use the real logging system csslog = logging.getLogger("cssutils") cssutils.log.setLog(csslog) # load the country list countries_file_path = os.path.join(static_files, "countries.json") try: with open(countries_file_path) as handle: self.countries = json.load(handle) self.log.debug("Using countries.json.") except IOError: self.log.warning("Couldn't find countries.json. Using pycountry.") self.countries = get_countries_and_codes() if not self.media_domain: self.media_domain = self.domain if self.media_domain == self.domain: print ("Warning: g.media_domain == g.domain. " + "This may give untrusted content access to user cookies") for arg in sys.argv: tokens = arg.split("=") if len(tokens) == 2: k, v = tokens self.log.debug("Overriding g.%s to %s" % (k, v)) setattr(self, k, v) self.reddit_host = socket.gethostname() self.reddit_pid = os.getpid() if hasattr(signal, 'SIGUSR1'): # not all platforms have user signals signal.signal(signal.SIGUSR1, thread_dump) self.startup_timer.intermediate("configuration") ################# ZOOKEEPER # for now, zookeeper will be an optional part of the stack. # if it's not configured, we will grab the expected config from the # [live_config] section of the ini file zk_hosts = self.config.get("zookeeper_connection_string") if zk_hosts: from r2.lib.zookeeper import (connect_to_zookeeper, LiveConfig, LiveList) zk_username = self.config["zookeeper_username"] zk_password = self.config["zookeeper_password"] self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username, zk_password)) self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE) self.throttles = LiveList(self.zookeeper, "/throttles", map_fn=ipaddress.ip_network, reduce_fn=ipaddress.collapse_addresses) self.banned_domains = LiveDict(self.zookeeper, "/banned-domains", watch=True) else: self.zookeeper = None parser = ConfigParser.RawConfigParser() parser.read([self.config["__file__"]]) self.live_config = extract_live_config(parser, self.plugins) self.throttles = tuple() # immutable since it's not real self.banned_domains = dict() self.startup_timer.intermediate("zookeeper") ################# MEMCACHE num_mc_clients = self.num_mc_clients # the main memcache pool. used for most everything. self.memcache = CMemcache( self.memcaches, min_compress_len=50 * 1024, num_clients=num_mc_clients, ) # a smaller pool of caches used only for distributed locks. # TODO: move this to ZooKeeper self.lock_cache = CMemcache(self.lockcaches, num_clients=num_mc_clients) self.make_lock = make_lock_factory(self.lock_cache, self.stats) # memcaches used in front of the permacache CF in cassandra. # XXX: this is a legacy thing; permacache was made when C* didn't have # a row cache. if self.permacache_memcaches: permacache_memcaches = CMemcache(self.permacache_memcaches, min_compress_len=50 * 1024, num_clients=num_mc_clients) else: permacache_memcaches = None # the stalecache is a memcached local to the current app server used # for data that's frequently fetched but doesn't need to be fresh. if self.stalecaches: stalecaches = CMemcache(self.stalecaches, num_clients=num_mc_clients) else: stalecaches = None # rendercache holds rendered partial templates. rendercaches = CMemcache( self.rendercaches, noreply=True, no_block=True, num_clients=num_mc_clients, min_compress_len=1400, ) # pagecaches hold fully rendered pages pagecaches = CMemcache( self.pagecaches, noreply=True, no_block=True, num_clients=num_mc_clients, min_compress_len=1400, ) self.startup_timer.intermediate("memcache") ################# CASSANDRA keyspace = "reddit" self.cassandra_pools = { "main": StatsCollectingConnectionPool( keyspace, stats=self.stats, logging_name="main", server_list=self.cassandra_seeds, pool_size=self.cassandra_pool_size, timeout=4, max_retries=3, prefill=False ), } permacache_cf = CassandraCache( 'permacache', self.cassandra_pools[self.cassandra_default_pool], read_consistency_level=self.cassandra_rcl, write_consistency_level=self.cassandra_wcl ) self.startup_timer.intermediate("cassandra") ################# POSTGRES event.listens_for(engine.Engine, 'before_cursor_execute')( self.stats.pg_before_cursor_execute) event.listens_for(engine.Engine, 'after_cursor_execute')( self.stats.pg_after_cursor_execute) self.dbm = self.load_db_params() self.startup_timer.intermediate("postgres") ################# CHAINS # initialize caches. Any cache-chains built here must be added # to cache_chains (closed around by reset_caches) so that they # can properly reset their local components cache_chains = {} localcache_cls = (SelfEmptyingCache if self.running_as_script else LocalCache) if stalecaches: self.cache = StaleCacheChain( localcache_cls(), stalecaches, self.memcache, ) else: self.cache = MemcacheChain((localcache_cls(), self.memcache)) cache_chains.update(cache=self.cache) self.rendercache = MemcacheChain(( localcache_cls(), rendercaches, )) cache_chains.update(rendercache=self.rendercache) self.pagecache = MemcacheChain(( localcache_cls(), pagecaches, )) cache_chains.update(pagecache=self.pagecache) # the thing_cache is used in tdb_cassandra. self.thing_cache = CacheChain((localcache_cls(),)) cache_chains.update(thing_cache=self.thing_cache) self.permacache = CassandraCacheChain( localcache_cls(), permacache_cf, memcache=permacache_memcaches, lock_factory=self.make_lock, ) cache_chains.update(permacache=self.permacache) # hardcache is used for various things that tend to expire # TODO: replace hardcache w/ cassandra stuff self.hardcache = HardcacheChain( (localcache_cls(), self.memcache, HardCache(self)), cache_negative_results=True, ) cache_chains.update(hardcache=self.hardcache) # I know this sucks, but we need non-request-threads to be # able to reset the caches, so we need them be able to close # around 'cache_chains' without being able to call getattr on # 'g' def reset_caches(): for name, chain in cache_chains.iteritems(): chain.reset() chain.stats = CacheStats(self.stats, name) self.cache_chains = cache_chains self.reset_caches = reset_caches self.reset_caches() self.startup_timer.intermediate("cache_chains") # try to set the source control revision numbers self.versions = {} r2_root = os.path.dirname(os.path.dirname(self.paths["root"])) r2_gitdir = os.path.join(r2_root, ".git") self.short_version = self.record_repo_version("r2", r2_gitdir) if I18N_PATH: i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git") self.record_repo_version("i18n", i18n_git_path) self.startup_timer.intermediate("revisions") def setup_complete(self): self.startup_timer.stop() self.stats.flush() if self.log_start: self.log.error( "%s:%s started %s at %s (took %.02fs)", self.reddit_host, self.reddit_pid, self.short_version, datetime.now().strftime("%H:%M:%S"), self.startup_timer.elapsed_seconds() ) def record_repo_version(self, repo_name, git_dir): """Get the currently checked out git revision for a given repository, record it in g.versions, and return the short version of the hash.""" try: subprocess.check_output except AttributeError: # python 2.6 compat pass else: try: revision = subprocess.check_output(["git", "--git-dir", git_dir, "rev-parse", "HEAD"]) except subprocess.CalledProcessError, e: self.log.warning("Unable to fetch git revision: %r", e) else:
def test_float(self): self.assertEquals(3.0, ConfigValue.float('3')) self.assertEquals(-3.0, ConfigValue.float('-3')) with self.assertRaises(ValueError): ConfigValue.float('asdf')
def test_str(self): self.assertEquals('x', ConfigValue.str('x'))
def load_environment(global_conf={}, app_conf={}, setup_globals=True): # Setup our paths root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) paths = { 'root': root_path, 'controllers': os.path.join(root_path, 'controllers'), 'templates': [os.path.join(root_path, 'templates')], } if ConfigValue.bool(global_conf.get('uncompressedJS')): paths['static_files'] = os.path.join(root_path, 'public') else: paths['static_files'] = os.path.join(os.path.dirname(root_path), 'build/public') config.init_app(global_conf, app_conf, package='r2', template_engine='mako', paths=paths) g = config['pylons.g'] = Globals(global_conf, app_conf, paths) if setup_globals: g.setup() g.plugins.declare_queues(g.queues) r2.config.cache = g.cache g.plugins.load_plugins() config['r2.plugins'] = g.plugins g.startup_timer.intermediate("plugins") config['pylons.h'] = r2.lib.helpers config['routes.map'] = routing.make_map() #override the default response options config['pylons.response_options']['headers'] = {} # The following template options are passed to your template engines tmpl_options = config['buffet.template_options'] tmpl_options['mako.filesystem_checks'] = getattr(g, 'reload_templates', False) tmpl_options['mako.default_filters'] = ["mako_websafe"] tmpl_options['mako.imports'] = \ ["from r2.lib.filters import websafe, unsafe, mako_websafe", "from pylons import c, g, request", "from pylons.i18n import _, ungettext"] # when mako loads a previously compiled template file from its cache, it # doesn't check that the original template path matches the current path. # in the event that a new plugin defines a template overriding a reddit # template, unless the mtime newer, mako doesn't update the compiled # template. as a workaround, this makes mako store compiled templates with # the original path in the filename, forcing it to update with the path. def mako_module_path(filename, uri): module_directory = tmpl_options['mako.module_directory'] filename = filename.lstrip('/').replace('/', '-') path = os.path.join(module_directory, filename + ".py") return os.path.abspath(path) tmpl_options['mako.modulename_callable'] = mako_module_path if setup_globals: g.setup_complete()
class Globals(object): spec = { ConfigValue.int: [ 'db_pool_size', 'db_pool_overflow_size', 'commentpane_cache_time', 'num_mc_clients', 'MAX_CAMPAIGNS_PER_LINK', 'MIN_DOWN_LINK', 'MIN_UP_KARMA', 'MIN_DOWN_KARMA', 'MIN_RATE_LIMIT_KARMA', 'MIN_RATE_LIMIT_COMMENT_KARMA', 'HOT_PAGE_AGE', 'ADMIN_COOKIE_TTL', 'ADMIN_COOKIE_MAX_IDLE', 'OTP_COOKIE_TTL', 'hsts_max_age', 'num_comments', 'max_comments', 'max_comments_gold', 'max_comment_parent_walk', 'max_sr_images', 'num_serendipity', 'comment_visits_period', 'butler_max_mentions', 'min_membership_create_community', 'bcrypt_work_factor', 'cassandra_pool_size', 'sr_banned_quota', 'sr_muted_quota', 'sr_wikibanned_quota', 'sr_wikicontributor_quota', 'sr_moderator_invite_quota', 'sr_contributor_quota', 'sr_quota_time', 'sr_invite_limit', 'thumbnail_hidpi_scaling', 'wiki_keep_recent_days', 'wiki_max_page_length_bytes', 'wiki_max_config_stylesheet_length_bytes', 'wiki_max_page_name_length', 'wiki_max_page_separators', 'RL_RESET_MINUTES', 'RL_OAUTH_RESET_MINUTES', 'comment_karma_display_floor', 'link_karma_display_floor', 'mobile_auth_gild_time', 'default_total_budget_pennies', 'min_total_budget_pennies', 'max_total_budget_pennies', 'default_bid_pennies', 'min_bid_pennies', 'max_bid_pennies', 'frequency_cap_min', 'frequency_cap_default', 'eu_cookie_max_attempts', 'captcha_sol_length', 'captcha_font_size', 'banner_variants', 'precompute_limit', 'precompute_limit_hot', 'hot_max_links_per_subreddit', 'fetch_title_max_download_kb', ], ConfigValue.float: [ 'statsd_sample_rate', 'querycache_prune_chance', 'RL_AVG_REQ_PER_SEC', 'RL_OAUTH_AVG_REQ_PER_SEC', 'RL_LOGIN_AVG_PER_SEC', 'RL_LOGIN_IP_AVG_PER_SEC', 'RL_SHARE_AVG_PER_SEC', 'tracing_sample_rate', 'hot_period_seconds', ], ConfigValue.bool: [ 'debug', 'log_start', 'sqlprinting', 'template_debug', 'reload_templates', 'uncompressedJS', 'css_killswitch', 'db_create_tables', 'disallow_db_writes', 'disable_ratelimit', 'amqp_logging', 'read_only_mode', 'disable_wiki', 'heavy_load_mode', 'disable_captcha', 'disable_ads', 'disable_require_admin_otp', 'trust_local_proxies', 'shard_commentstree_queues', 'shard_author_query_queues', 'shard_subreddit_query_queues', 'shard_domain_query_queues', 'authnet_validate', 'ENFORCE_RATELIMIT', 'RL_SITEWIDE_ENABLED', 'RL_OAUTH_SITEWIDE_ENABLED', 'enable_loggedout_experiments', 'disable_geoip_service', 'disable_remote_fetch', 'disable_newsletter', 'remote_fetch_proxy_enabled', 'gold_gilding_enabled', 'sub_muting_enabled', 'allsr_prefilter_allow_top', 'site_index_user_configurable', 'allow_top_affects_new', 'allow_top_false_subreddits_tab', 'block_user_show_comments', 'block_user_show_links', 'chat_guest_chat_enabled', 'chat_all', 'chat_front', ], ConfigValue.tuple: [ 'plugins', 'stalecaches', 'lockcaches', 'permacache_memcaches', 'cassandra_seeds', 'automatic_reddits', 'hardcache_categories', 'case_sensitive_domains', 'known_image_domains', 'reserved_subdomains', 'offsite_subdomains', 'TRAFFIC_LOG_HOSTS', 'exempt_login_user_agents', 'autoexpand_media_types', 'media_preview_domain_whitelist', 'multi_icons', 'hide_subscribers_srs', 'mcrouter_addr', 'permacache_domain_priority', ], ConfigValue.tuple_of(ConfigValue.int): [ 'thumbnail_size', 'preview_image_max_size', 'preview_image_min_size', 'mobile_ad_image_size', ], ConfigValue.tuple_of(ConfigValue.float): [ 'ios_versions', 'android_versions', ], ConfigValue.dict(ConfigValue.str, ConfigValue.int): [ 'user_agent_ratelimit_regexes', ], ConfigValue.str: [ 'wiki_page_registration_info', 'wiki_page_privacy_policy', 'wiki_page_user_agreement', 'wiki_page_gold_bottlecaps', 'fraud_email', 'feedback_email', 'share_reply', 'community_email', 'smtp_server', 'events_collector_url', 'events_collector_test_url', 'search_provider', 'remote_fetch_proxy_url', 'brander_community', 'brander_community_plural', 'imgur_client_id', ], ConfigValue.choice(ONE=CL_ONE, QUORUM=CL_QUORUM): [ 'cassandra_rcl', 'cassandra_wcl', ], ConfigValue.choice(zookeeper="zookeeper", config="config"): [ "liveconfig_source", "secrets_source", ], ConfigValue.timeinterval: [ 'ARCHIVE_AGE', "vote_queue_grace_period", ], config_gold_price: [ 'gold_month_price', 'gold_year_price', 'cpm_selfserve', 'cpm_selfserve_geotarget_metro', 'cpm_selfserve_geotarget_country', 'cpm_selfserve_collection', ], ConfigValue.baseplate( baseplate_config.Optional(baseplate_config.Endpoint)): [ "activity_endpoint", "tracing_endpoint", ], ConfigValue.dict(ConfigValue.str, ConfigValue.str): [ 'emr_traffic_tags', ], } live_config_spec = { ConfigValue.bool: [ 'frontend_logging', 'mobile_gild_first_login', 'precomputed_comment_suggested_sort', ], ConfigValue.int: [ 'captcha_exempt_comment_karma', 'captcha_exempt_link_karma', 'create_sr_account_age_days', 'create_sr_comment_karma', 'create_sr_link_karma', 'cflag_min_votes', 'ads_popularity_threshold', 'precomputed_comment_sort_min_comments', 'comment_vote_update_threshold', 'comment_vote_update_period', 'create_sr_ratelimit_once_per_days', ], ConfigValue.float: [ 'cflag_lower_bound', 'cflag_upper_bound', 'spotlight_interest_sub_p', 'spotlight_interest_nosub_p', 'gold_revenue_goal', 'invalid_key_sample_rate', 'events_collector_vote_sample_rate', 'events_collector_poison_sample_rate', 'events_collector_mod_sample_rate', 'events_collector_quarantine_sample_rate', 'events_collector_modmail_sample_rate', 'events_collector_report_sample_rate', 'events_collector_submit_sample_rate', 'events_collector_comment_sample_rate', 'events_collector_use_gzip_chance', 'https_cert_testing_probability', ], ConfigValue.tuple: [ 'fastlane_links', 'listing_chooser_sample_multis', 'discovery_srs', 'proxy_gilding_accounts', 'mweb_blacklist_expressions', 'global_loid_experiments', 'precomputed_comment_sorts', 'mailgun_domains', ], ConfigValue.str: [ 'listing_chooser_gold_multi', 'listing_chooser_explore_sr', ], ConfigValue.messages: [ 'welcomebar_messages', 'sidebar_message', 'gold_sidebar_message', ], ConfigValue.dict(ConfigValue.str, ConfigValue.int): [ 'ticket_groups', 'ticket_user_fields', ], ConfigValue.dict(ConfigValue.str, ConfigValue.float): [ 'pennies_per_server_second', ], ConfigValue.dict(ConfigValue.str, ConfigValue.str): [ 'employee_approved_clients', 'mobile_auth_allowed_clients', 'modmail_forwarding_email', 'modmail_account_map', ], ConfigValue.dict(ConfigValue.str, ConfigValue.choice(**PERMISSIONS)): [ 'employees', ], } def __init__(self, config, global_conf, app_conf, paths, **extra): """ Globals acts as a container for objects available throughout the life of the application. One instance of Globals is created by Pylons during application initialization and is available during requests via the 'g' variable. ``config`` The PylonsConfig object passed in from ``config/environment.py`` ``global_conf`` The same variable used throughout ``config/middleware.py`` namely, the variables from the ``[DEFAULT]`` section of the configuration file. ``app_conf`` The same ``kw`` dictionary used throughout ``config/middleware.py`` namely, the variables from the section in the config file for your application. ``extra`` The configuration returned from ``load_config`` in ``config/middleware.py`` which may be of use in the setup of your global variables. """ global_conf.setdefault("debug", False) # reloading site ensures that we have a fresh sys.path to build our # working set off of. this means that forked worker processes won't get # the sys.path that was current when the master process was spawned # meaning that new plugins will be picked up on regular app reload # rather than having to restart the master process as well. reload(site) self.pkg_resources_working_set = pkg_resources.WorkingSet() self.config = ConfigValueParser(global_conf) self.config.add_spec(self.spec) self.plugins = PluginLoader(self.pkg_resources_working_set, self.config.get("plugins", [])) self.stats = Stats(self.config.get('statsd_addr'), self.config.get('statsd_sample_rate')) self.startup_timer = self.stats.get_timer("app_startup") self.startup_timer.start() self.baseplate = Baseplate() self.baseplate.configure_logging() self.baseplate.register(R2BaseplateObserver()) self.baseplate.configure_tracing( "r2", tracing_endpoint=self.config.get("tracing_endpoint"), sample_rate=self.config.get("tracing_sample_rate"), ) self.paths = paths self.running_as_script = global_conf.get('running_as_script', False) # turn on for language support self.lang = getattr(self, 'site_lang', 'en') self.languages, self.lang_name = get_active_langs( config, default_lang=self.lang) all_languages = self.lang_name.keys() all_languages.sort() self.all_languages = all_languages # set default time zone if one is not set tz = global_conf.get('timezone', 'UTC') self.tz = pytz.timezone(tz) dtz = global_conf.get('display_timezone', tz) self.display_tz = pytz.timezone(dtz) self.startup_timer.intermediate("init") def __getattr__(self, name): if not name.startswith('_') and name in self.config: return self.config[name] else: raise AttributeError("g has no attr %r" % name) def setup(self): self.env = '' if ( # handle direct invocation of "nosetests" "test" in sys.argv[0] or # handle "setup.py test" and all permutations thereof. "setup.py" in sys.argv[0] and "test" in sys.argv[1:]): self.env = "unit_test" self.queues = queues.declare_queues(self) self.extension_subdomains = dict( simple="mobile", i="compact", api="api", rss="rss", xml="xml", json="json", ) # SaidIt CUSTOM self.extension_subdomains[ self.config['extension_subdomain_mobile_v2']] = self.config[ 'extension_subdomain_mobile_v2_render_style'] ################# PROVIDERS self.auth_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.auth", self.authentication_provider, ) self.media_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.media", self.media_provider, ) self.cdn_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.cdn", self.cdn_provider, ) self.ticket_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.support", # TODO: fix this later, it refuses to pick up # g.config['ticket_provider'] value, so hardcoding for now. # really, the next uncommented line should be: #self.ticket_provider, # instead of: "zendesk", ) self.image_resizing_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.image_resizing", self.image_resizing_provider, ) self.email_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.email", self.email_provider, ) self.startup_timer.intermediate("providers") ################# CONFIGURATION # AMQP is required if not self.amqp_host: raise ValueError("amqp_host not set in the .ini") if not self.cassandra_seeds: raise ValueError("cassandra_seeds not set in the .ini") # heavy load mode is read only mode with a different infobar if self.heavy_load_mode: self.read_only_mode = True origin_prefix = self.domain_prefix + "." if self.domain_prefix else "" self.origin = self.default_scheme + "://" + origin_prefix + self.domain self.trusted_domains = set([self.domain]) if self.https_endpoint: https_url = urlparse(self.https_endpoint) self.trusted_domains.add(https_url.hostname) # load the unique hashed names of files under static static_files = os.path.join(self.paths.get('static_files'), 'static') names_file_path = os.path.join(static_files, 'names.json') if os.path.exists(names_file_path): with open(names_file_path) as handle: self.static_names = json.load(handle) else: self.static_names = {} # make python warnings go through the logging system logging.captureWarnings(capture=True) log = logging.getLogger('reddit') # when we're a script (paster run) just set up super simple logging if self.running_as_script: log.setLevel(logging.INFO) log.addHandler(logging.StreamHandler()) # if in debug mode, override the logging level to DEBUG if self.debug: log.setLevel(logging.DEBUG) # attempt to figure out which pool we're in and add that to the # LogRecords. try: with open("/etc/ec2_asg", "r") as f: pool = f.read().strip() # clean up the pool name since we're putting stuff after "-" pool = pool.partition("-")[0] except IOError: pool = "reddit-app" self.log = logging.LoggerAdapter(log, {"pool": pool}) # set locations locations = pkg_resources.resource_stream(__name__, "../data/locations.json") self.locations = json.loads(locations.read()) if not self.media_domain: self.media_domain = self.domain if self.media_domain == self.domain: print >> sys.stderr, ( "Warning: g.media_domain == g.domain. " + "This may give untrusted content access to user cookies") if self.oauth_domain == self.domain: print >> sys.stderr, ("Warning: g.oauth_domain == g.domain. " "CORS requests to g.domain will be allowed") for arg in sys.argv: tokens = arg.split("=") if len(tokens) == 2: k, v = tokens self.log.debug("Overriding g.%s to %s" % (k, v)) setattr(self, k, v) self.reddit_host = socket.gethostname() self.reddit_pid = os.getpid() if hasattr(signal, 'SIGUSR1'): # not all platforms have user signals signal.signal(signal.SIGUSR1, thread_dump) locale.setlocale(locale.LC_ALL, self.locale) # Pre-calculate ratelimit values self.RL_RESET_SECONDS = self.config["RL_RESET_MINUTES"] * 60 self.RL_MAX_REQS = int(self.config["RL_AVG_REQ_PER_SEC"] * self.RL_RESET_SECONDS) self.RL_OAUTH_RESET_SECONDS = self.config["RL_OAUTH_RESET_MINUTES"] * 60 self.RL_OAUTH_MAX_REQS = int(self.config["RL_OAUTH_AVG_REQ_PER_SEC"] * self.RL_OAUTH_RESET_SECONDS) self.RL_LOGIN_MAX_REQS = int(self.config["RL_LOGIN_AVG_PER_SEC"] * self.RL_RESET_SECONDS) self.RL_LOGIN_IP_MAX_REQS = int( self.config["RL_LOGIN_IP_AVG_PER_SEC"] * self.RL_RESET_SECONDS) self.RL_SHARE_MAX_REQS = int(self.config["RL_SHARE_AVG_PER_SEC"] * self.RL_RESET_SECONDS) # Compile ratelimit regexs user_agent_ratelimit_regexes = {} for agent_re, limit in self.user_agent_ratelimit_regexes.iteritems(): user_agent_ratelimit_regexes[re.compile(agent_re)] = limit self.user_agent_ratelimit_regexes = user_agent_ratelimit_regexes self.startup_timer.intermediate("configuration") ################# ZOOKEEPER zk_hosts = self.config["zookeeper_connection_string"] zk_username = self.config["zookeeper_username"] zk_password = self.config["zookeeper_password"] self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username, zk_password)) self.throttles = IPNetworkLiveList( self.zookeeper, root="/throttles", reduced_data_node="/throttles_reduced", ) parser = ConfigParser.RawConfigParser() parser.optionxform = str parser.read([self.config["__file__"]]) if self.config["liveconfig_source"] == "zookeeper": self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE) else: self.live_config = extract_live_config(parser, self.plugins) if self.config["secrets_source"] == "zookeeper": self.secrets = fetch_secrets(self.zookeeper) else: self.secrets = extract_secrets(parser) ################# PRIVILEGED USERS self.admins = PermissionFilteredEmployeeList(self.live_config, type="admin") self.sponsors = PermissionFilteredEmployeeList(self.live_config, type="sponsor") self.employees = PermissionFilteredEmployeeList(self.live_config, type="employee") # Store which OAuth clients employees may use, the keys are just for # readability. self.employee_approved_clients = \ self.live_config["employee_approved_clients"].values() self.mobile_auth_allowed_clients = self.live_config[ "mobile_auth_allowed_clients"].values() self.startup_timer.intermediate("zookeeper") ################# MEMCACHE num_mc_clients = self.num_mc_clients # a smaller pool of caches used only for distributed locks. self.lock_cache = CMemcache( "lock", self.lockcaches, num_clients=num_mc_clients, ) self.make_lock = make_lock_factory(self.lock_cache, self.stats) # memcaches used in front of the permacache CF in cassandra. # XXX: this is a legacy thing; permacache was made when C* didn't have # a row cache. permacache_memcaches = CMemcache( "perma", self.permacache_memcaches, min_compress_len=1400, num_clients=num_mc_clients, ) # the stalecache is a memcached local to the current app server used # for data that's frequently fetched but doesn't need to be fresh. if self.stalecaches: stalecaches = CMemcache( "stale", self.stalecaches, num_clients=num_mc_clients, ) else: stalecaches = None self.startup_timer.intermediate("memcache") ################# MCROUTER self.mcrouter = Mcrouter( "mcrouter", self.mcrouter_addr, min_compress_len=1400, num_clients=num_mc_clients, ) ################# THRIFT-BASED SERVICES activity_endpoint = self.config.get("activity_endpoint") if activity_endpoint: # make ActivityInfo objects rendercache-key friendly # TODO: figure out a more general solution for this if # we need to do this for other thrift-generated objects ActivityInfo.cache_key = lambda self, style: repr(self) activity_pool = ThriftConnectionPool(activity_endpoint, timeout=0.1) self.baseplate.add_to_context( "activity_service", ThriftContextFactory(activity_pool, ActivityService.Client)) self.startup_timer.intermediate("thrift") ################# CASSANDRA keyspace = "reddit" self.cassandra_pools = { "main": StatsCollectingConnectionPool(keyspace, stats=self.stats, logging_name="main", server_list=self.cassandra_seeds, pool_size=self.cassandra_pool_size, timeout=4, max_retries=3, prefill=False), } permacache_cf = Permacache._setup_column_family( 'permacache', self.cassandra_pools[self.cassandra_default_pool], ) self.startup_timer.intermediate("cassandra") ################# POSTGRES self.dbm = self.load_db_params() self.startup_timer.intermediate("postgres") ################# CHAINS # initialize caches. Any cache-chains built here must be added # to cache_chains (closed around by reset_caches) so that they # can properly reset their local components cache_chains = {} localcache_cls = (SelfEmptyingCache if self.running_as_script else LocalCache) if stalecaches: self.gencache = StaleCacheChain( localcache_cls(), stalecaches, self.mcrouter, ) else: self.gencache = CacheChain((localcache_cls(), self.mcrouter)) cache_chains.update(gencache=self.gencache) if stalecaches: self.thingcache = StaleCacheChain( localcache_cls(), stalecaches, self.mcrouter, ) else: self.thingcache = CacheChain((localcache_cls(), self.mcrouter)) cache_chains.update(thingcache=self.thingcache) if stalecaches: self.memoizecache = StaleCacheChain( localcache_cls(), stalecaches, self.mcrouter, ) else: self.memoizecache = MemcacheChain( (localcache_cls(), self.mcrouter)) cache_chains.update(memoizecache=self.memoizecache) if stalecaches: self.srmembercache = StaleCacheChain( localcache_cls(), stalecaches, self.mcrouter, ) else: self.srmembercache = MemcacheChain( (localcache_cls(), self.mcrouter)) cache_chains.update(srmembercache=self.srmembercache) if stalecaches: self.relcache = StaleCacheChain( localcache_cls(), stalecaches, self.mcrouter, ) else: self.relcache = MemcacheChain((localcache_cls(), self.mcrouter)) cache_chains.update(relcache=self.relcache) self.ratelimitcache = MemcacheChain((localcache_cls(), self.mcrouter)) cache_chains.update(ratelimitcache=self.ratelimitcache) # rendercache holds rendered partial templates. self.rendercache = MemcacheChain(( localcache_cls(), self.mcrouter, )) cache_chains.update(rendercache=self.rendercache) # commentpanecaches hold fully rendered comment panes self.commentpanecache = MemcacheChain(( localcache_cls(), self.mcrouter, )) cache_chains.update(commentpanecache=self.commentpanecache) # cassandra_local_cache is used for request-local caching in tdb_cassandra self.cassandra_local_cache = localcache_cls() cache_chains.update(cassandra_local_cache=self.cassandra_local_cache) if stalecaches: permacache_cache = StaleCacheChain( localcache_cls(), stalecaches, permacache_memcaches, ) else: permacache_cache = CacheChain( (localcache_cls(), permacache_memcaches), ) cache_chains.update(permacache=permacache_cache) self.permacache = Permacache( permacache_cache, permacache_cf, lock_factory=self.make_lock, ) # hardcache is used for various things that tend to expire # TODO: replace hardcache w/ cassandra stuff self.hardcache = HardcacheChain( (localcache_cls(), HardCache(self)), cache_negative_results=True, ) cache_chains.update(hardcache=self.hardcache) # I know this sucks, but we need non-request-threads to be # able to reset the caches, so we need them be able to close # around 'cache_chains' without being able to call getattr on # 'g' def reset_caches(): for name, chain in cache_chains.iteritems(): if isinstance(chain, TransitionalCache): chain = chain.read_chain chain.reset() if isinstance(chain, LocalCache): continue elif isinstance(chain, StaleCacheChain): chain.stats = StaleCacheStats(self.stats, name) else: chain.stats = CacheStats(self.stats, name) self.cache_chains = cache_chains self.reset_caches = reset_caches self.reset_caches() self.startup_timer.intermediate("cache_chains") # try to set the source control revision numbers self.versions = {} r2_root = os.path.dirname(os.path.dirname(self.paths["root"])) r2_gitdir = os.path.join(r2_root, ".git") self.short_version = self.record_repo_version("r2", r2_gitdir) if I18N_PATH: i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git") self.record_repo_version("i18n", i18n_git_path) # Initialize the amqp module globals, start the worker, etc. r2.lib.amqp.initialize(self) self.events = EventQueue() self.startup_timer.intermediate("revisions") def setup_complete(self): self.startup_timer.stop() self.stats.flush() if self.log_start: self.log.error("%s:%s started %s at %s (took %.02fs)", self.reddit_host, self.reddit_pid, self.short_version, datetime.now().strftime("%H:%M:%S"), self.startup_timer.elapsed_seconds()) if einhorn.is_worker(): einhorn.ack_startup() def record_repo_version(self, repo_name, git_dir): """Get the currently checked out git revision for a given repository, record it in g.versions, and return the short version of the hash.""" try: subprocess.check_output except AttributeError: # python 2.6 compat pass else: try: revision = subprocess.check_output( ["git", "--git-dir", git_dir, "rev-parse", "HEAD"]) except subprocess.CalledProcessError, e: self.log.warning("Unable to fetch git revision: %r", e) else:
def load_environment(global_conf={}, app_conf={}, setup_globals=True): # Setup our paths root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) paths = {'root': root_path, 'controllers': os.path.join(root_path, 'controllers'), 'templates': [os.path.join(root_path, 'templates')], } if ConfigValue.bool(global_conf.get('uncompressedJS')): paths['static_files'] = os.path.join(root_path, 'public') else: paths['static_files'] = os.path.join(os.path.dirname(root_path), 'build/public') config.init_app(global_conf, app_conf, package='r2', template_engine='mako', paths=paths) # don't put action arguments onto c automatically config['pylons.c_attach_args'] = False # when accessing non-existent attributes on c, return "" instead of dying config['pylons.strict_c'] = False g = config['pylons.g'] = Globals(global_conf, app_conf, paths) if setup_globals: g.setup() g.plugins.declare_queues(g.queues) g.plugins.load_plugins() config['r2.plugins'] = g.plugins g.startup_timer.intermediate("plugins") config['pylons.h'] = r2.lib.helpers config['routes.map'] = routing.make_map() #override the default response options config['pylons.response_options']['headers'] = {} # when mako loads a previously compiled template file from its cache, it # doesn't check that the original template path matches the current path. # in the event that a new plugin defines a template overriding a reddit # template, unless the mtime newer, mako doesn't update the compiled # template. as a workaround, this makes mako store compiled templates with # the original path in the filename, forcing it to update with the path. if "cache_dir" in app_conf: module_directory = os.path.join(app_conf['cache_dir'], 'templates') def mako_module_path(filename, uri): filename = filename.lstrip('/').replace('/', '-') path = os.path.join(module_directory, filename + ".py") return os.path.abspath(path) else: # we're probably in "paster run standalone" mode. we'll just avoid # caching templates since we don't know where they should go. module_directory = mako_module_path = None # set up the templating system config["pylons.g"].mako_lookup = TemplateLookup( directories=paths["templates"], error_handler=handle_mako_error, module_directory=module_directory, input_encoding="utf-8", default_filters=["mako_websafe"], filesystem_checks=getattr(g, "reload_templates", False), imports=[ "from r2.lib.filters import websafe, unsafe, mako_websafe", "from pylons import c, g, request", "from pylons.i18n import _, ungettext", ], modulename_callable=mako_module_path, ) if setup_globals: g.setup_complete()
def test_set_of(self): self.assertEquals(set([]), ConfigValue.set_of(str)('')) self.assertEquals(set(['a', 'b']), ConfigValue.set_of(str)('a, b, b')) self.assertEquals(set(['a', 'b']), ConfigValue.set_of(str, delim=':')('b : a : b'))
def test_choice(self): self.assertEquals(1, ConfigValue.choice(alpha=1)('alpha')) self.assertEquals(2, ConfigValue.choice(alpha=1, beta=2)('beta')) with self.assertRaises(ValueError): ConfigValue.choice(alpha=1)('asdf')
def load_db_params(self): self.databases = tuple(ConfigValue.to_iter(self.config.raw_data["databases"])) self.db_params = {} self.predefined_type_ids = {} if not self.databases: return if self.env == "unit_test": from mock import MagicMock return MagicMock() dbm = db_manager.db_manager() db_param_names = ("name", "db_host", "db_user", "db_pass", "db_port", "pool_size", "max_overflow") for db_name in self.databases: conf_params = ConfigValue.to_iter(self.config.raw_data[db_name + "_db"]) params = dict(zip(db_param_names, conf_params)) if params["db_user"] == "*": params["db_user"] = self.db_user if params["db_pass"] == "*": params["db_pass"] = self.db_pass if params["db_port"] == "*": params["db_port"] = self.db_port if params["pool_size"] == "*": params["pool_size"] = self.db_pool_size if params["max_overflow"] == "*": params["max_overflow"] = self.db_pool_overflow_size dbm.setup_db(db_name, g_override=self, **params) self.db_params[db_name] = params dbm.type_db = dbm.get_engine(self.config.raw_data["type_db"]) dbm.relation_type_db = dbm.get_engine(self.config.raw_data["rel_type_db"]) def split_flags(raw_params): params = [] flags = {} for param in raw_params: if not param.startswith("!"): params.append(param) else: key, sep, value = param[1:].partition("=") if sep: flags[key] = value else: flags[key] = True return params, flags prefix = "db_table_" for k, v in self.config.raw_data.iteritems(): if not k.startswith(prefix): continue params, table_flags = split_flags(ConfigValue.to_iter(v)) name = k[len(prefix) :] kind = params[0] server_list = self.config.raw_data["db_servers_" + name] engines, flags = split_flags(ConfigValue.to_iter(server_list)) typeid = table_flags.get("typeid") if typeid: self.predefined_type_ids[name] = int(typeid) if kind == "thing": dbm.add_thing(name, dbm.get_engines(engines), **flags) elif kind == "relation": dbm.add_relation(name, params[1], params[2], dbm.get_engines(engines), **flags) return dbm
class Globals(object): spec = { ConfigValue.int: [ 'db_pool_size', 'db_pool_overflow_size', 'page_cache_time', 'commentpane_cache_time', 'num_mc_clients', 'MAX_CAMPAIGNS_PER_LINK', 'MIN_DOWN_LINK', 'MIN_UP_KARMA', 'MIN_DOWN_KARMA', 'MIN_RATE_LIMIT_KARMA', 'MIN_RATE_LIMIT_COMMENT_KARMA', 'HOT_PAGE_AGE', 'QUOTA_THRESHOLD', 'ADMIN_COOKIE_TTL', 'ADMIN_COOKIE_MAX_IDLE', 'OTP_COOKIE_TTL', 'hsts_max_age', 'num_comments', 'max_comments', 'max_comments_gold', 'max_comment_parent_walk', 'max_sr_images', 'num_serendipity', 'sr_dropdown_threshold', 'comment_visits_period', 'butler_max_mentions', 'min_membership_create_community', 'bcrypt_work_factor', 'cassandra_pool_size', 'sr_banned_quota', 'sr_wikibanned_quota', 'sr_wikicontributor_quota', 'sr_moderator_invite_quota', 'sr_contributor_quota', 'sr_quota_time', 'sr_invite_limit', 'thumbnail_hidpi_scaling', 'wiki_keep_recent_days', 'wiki_max_page_length_bytes', 'wiki_max_page_name_length', 'wiki_max_page_separators', 'RL_RESET_MINUTES', 'RL_OAUTH_RESET_MINUTES', 'comment_karma_display_floor', 'link_karma_display_floor', ], ConfigValue.float: [ 'default_promote_bid', 'min_promote_bid', 'max_promote_bid', 'statsd_sample_rate', 'querycache_prune_chance', 'RL_AVG_REQ_PER_SEC', 'RL_OAUTH_AVG_REQ_PER_SEC', 'RL_LOGIN_AVG_PER_SEC', ], ConfigValue.bool: [ 'debug', 'log_start', 'sqlprinting', 'template_debug', 'reload_templates', 'uncompressedJS', 'css_killswitch', 'db_create_tables', 'disallow_db_writes', 'disable_ratelimit', 'amqp_logging', 'read_only_mode', 'disable_wiki', 'heavy_load_mode', 'disable_captcha', 'disable_ads', 'disable_require_admin_otp', 'trust_local_proxies', 'shard_link_vote_queues', 'shard_commentstree_queues', 'ENFORCE_RATELIMIT', 'RL_SITEWIDE_ENABLED', 'RL_OAUTH_SITEWIDE_ENABLED', ], ConfigValue.tuple: [ 'plugins', 'stalecaches', 'memcaches', 'lockcaches', 'permacache_memcaches', 'rendercaches', 'pagecaches', 'memoizecaches', 'srmembercaches', 'relcaches', 'ratelimitcaches', 'cassandra_seeds', 'automatic_reddits', 'hardcache_categories', 'case_sensitive_domains', 'known_image_domains', 'reserved_subdomains', 'offsite_subdomains', 'TRAFFIC_LOG_HOSTS', 'exempt_login_user_agents', 'timed_templates', 'autoexpand_media_types', 'multi_icons', 'hide_subscribers_srs', ], ConfigValue.tuple_of(ConfigValue.int): [ 'thumbnail_size', ], ConfigValue.dict(ConfigValue.str, ConfigValue.int): [ 'agents', ], ConfigValue.str: [ 'wiki_page_registration_info', 'wiki_page_privacy_policy', 'wiki_page_user_agreement', 'wiki_page_gold_bottlecaps', 'fraud_email', 'feedback_email', 'share_reply', 'nerds_email', 'community_email', 'smtp_server', ], ConfigValue.choice(ONE=CL_ONE, QUORUM=CL_QUORUM): [ 'cassandra_rcl', 'cassandra_wcl', ], ConfigValue.timeinterval: [ 'ARCHIVE_AGE', "vote_queue_grace_period", ], config_gold_price: [ 'gold_month_price', 'gold_year_price', 'cpm_selfserve', 'cpm_selfserve_geotarget_metro', 'cpm_selfserve_collection', ], } live_config_spec = { ConfigValue.bool: [ 'frontend_logging', ], ConfigValue.int: [ 'captcha_exempt_comment_karma', 'captcha_exempt_link_karma', 'create_sr_account_age_days', 'create_sr_comment_karma', 'create_sr_link_karma', 'cflag_min_votes', ], ConfigValue.float: [ 'cflag_lower_bound', 'cflag_upper_bound', 'spotlight_interest_sub_p', 'spotlight_interest_nosub_p', 'gold_revenue_goal', 'invalid_key_sample_rate', ], ConfigValue.tuple: [ 'fastlane_links', 'listing_chooser_sample_multis', 'discovery_srs', 'proxy_gilding_accounts', ], ConfigValue.str: [ 'listing_chooser_gold_multi', 'listing_chooser_explore_sr', ], ConfigValue.dict(ConfigValue.int, ConfigValue.float): [ 'comment_tree_version_weights', ], ConfigValue.messages: [ 'welcomebar_messages', 'sidebar_message', 'gold_sidebar_message', ], ConfigValue.dict(ConfigValue.str, ConfigValue.float): [ 'pennies_per_server_second', ], ConfigValue.dict(ConfigValue.str, ConfigValue.choice(**PERMISSIONS)): [ 'employees', ], } def __init__(self, global_conf, app_conf, paths, **extra): """ Globals acts as a container for objects available throughout the life of the application. One instance of Globals is created by Pylons during application initialization and is available during requests via the 'g' variable. ``global_conf`` The same variable used throughout ``config/middleware.py`` namely, the variables from the ``[DEFAULT]`` section of the configuration file. ``app_conf`` The same ``kw`` dictionary used throughout ``config/middleware.py`` namely, the variables from the section in the config file for your application. ``extra`` The configuration returned from ``load_config`` in ``config/middleware.py`` which may be of use in the setup of your global variables. """ global_conf.setdefault("debug", False) # reloading site ensures that we have a fresh sys.path to build our # working set off of. this means that forked worker processes won't get # the sys.path that was current when the master process was spawned # meaning that new plugins will be picked up on regular app reload # rather than having to restart the master process as well. reload(site) self.pkg_resources_working_set = pkg_resources.WorkingSet() self.config = ConfigValueParser(global_conf) self.config.add_spec(self.spec) self.plugins = PluginLoader(self.pkg_resources_working_set, self.config.get("plugins", [])) self.stats = Stats(self.config.get('statsd_addr'), self.config.get('statsd_sample_rate')) self.startup_timer = self.stats.get_timer("app_startup") self.startup_timer.start() self.paths = paths self.running_as_script = global_conf.get('running_as_script', False) # turn on for language support self.lang = getattr(self, 'site_lang', 'en') self.languages, self.lang_name = \ get_active_langs(default_lang=self.lang) all_languages = self.lang_name.keys() all_languages.sort() self.all_languages = all_languages # set default time zone if one is not set tz = global_conf.get('timezone', 'UTC') self.tz = pytz.timezone(tz) dtz = global_conf.get('display_timezone', tz) self.display_tz = pytz.timezone(dtz) self.startup_timer.intermediate("init") def __getattr__(self, name): if not name.startswith('_') and name in self.config: return self.config[name] else: raise AttributeError def setup(self): self.queues = queues.declare_queues(self) self.extension_subdomains = dict( m="mobile", i="compact", api="api", rss="rss", xml="xml", json="json", ) ################# PROVIDERS self.auth_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.auth", self.authentication_provider, ) self.media_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.media", self.media_provider, ) self.cdn_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.cdn", self.cdn_provider, ) self.startup_timer.intermediate("providers") ################# CONFIGURATION # AMQP is required if not self.amqp_host: raise ValueError("amqp_host not set in the .ini") if not self.cassandra_seeds: raise ValueError("cassandra_seeds not set in the .ini") # heavy load mode is read only mode with a different infobar if self.heavy_load_mode: self.read_only_mode = True origin_prefix = self.domain_prefix + "." if self.domain_prefix else "" self.origin = "http://" + origin_prefix + self.domain self.trusted_domains = set([self.domain]) if self.https_endpoint: https_url = urlparse(self.https_endpoint) self.trusted_domains.add(https_url.hostname) # load the unique hashed names of files under static static_files = os.path.join(self.paths.get('static_files'), 'static') names_file_path = os.path.join(static_files, 'names.json') if os.path.exists(names_file_path): with open(names_file_path) as handle: self.static_names = json.load(handle) else: self.static_names = {} # make python warnings go through the logging system logging.captureWarnings(capture=True) log = logging.getLogger('reddit') # when we're a script (paster run) just set up super simple logging if self.running_as_script: log.setLevel(logging.INFO) log.addHandler(logging.StreamHandler()) # if in debug mode, override the logging level to DEBUG if self.debug: log.setLevel(logging.DEBUG) # attempt to figure out which pool we're in and add that to the # LogRecords. try: with open("/etc/ec2_asg", "r") as f: pool = f.read().strip() # clean up the pool name since we're putting stuff after "-" pool = pool.partition("-")[0] except IOError: pool = "reddit-app" self.log = logging.LoggerAdapter(log, {"pool": pool}) # set locations locations = pkg_resources.resource_stream(__name__, "../data/locations.json") self.locations = json.loads(locations.read()) if not self.media_domain: self.media_domain = self.domain if self.media_domain == self.domain: print >> sys.stderr, ( "Warning: g.media_domain == g.domain. " + "This may give untrusted content access to user cookies") if self.oauth_domain == self.domain: print >> sys.stderr, ("Warning: g.oauth_domain == g.domain. " "CORS requests to g.domain will be allowed") for arg in sys.argv: tokens = arg.split("=") if len(tokens) == 2: k, v = tokens self.log.debug("Overriding g.%s to %s" % (k, v)) setattr(self, k, v) self.reddit_host = socket.gethostname() self.reddit_pid = os.getpid() if hasattr(signal, 'SIGUSR1'): # not all platforms have user signals signal.signal(signal.SIGUSR1, thread_dump) locale.setlocale(locale.LC_ALL, self.locale) # Pre-calculate ratelimit values self.RL_RESET_SECONDS = self.config["RL_RESET_MINUTES"] * 60 self.RL_MAX_REQS = int(self.config["RL_AVG_REQ_PER_SEC"] * self.RL_RESET_SECONDS) self.RL_OAUTH_RESET_SECONDS = self.config["RL_OAUTH_RESET_MINUTES"] * 60 self.RL_OAUTH_MAX_REQS = int(self.config["RL_OAUTH_AVG_REQ_PER_SEC"] * self.RL_OAUTH_RESET_SECONDS) self.RL_LOGIN_MAX_REQS = int(self.config["RL_LOGIN_AVG_PER_SEC"] * self.RL_RESET_SECONDS) self.startup_timer.intermediate("configuration") ################# ZOOKEEPER # for now, zookeeper will be an optional part of the stack. # if it's not configured, we will grab the expected config from the # [live_config] section of the ini file zk_hosts = self.config.get("zookeeper_connection_string") if zk_hosts: from r2.lib.zookeeper import (connect_to_zookeeper, LiveConfig, LiveList) zk_username = self.config["zookeeper_username"] zk_password = self.config["zookeeper_password"] self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username, zk_password)) self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE) self.secrets = fetch_secrets(self.zookeeper) self.throttles = LiveList(self.zookeeper, "/throttles", map_fn=ipaddress.ip_network, reduce_fn=ipaddress.collapse_addresses) # close our zk connection when the app shuts down SHUTDOWN_CALLBACKS.append(self.zookeeper.stop) else: self.zookeeper = None parser = ConfigParser.RawConfigParser() parser.optionxform = str parser.read([self.config["__file__"]]) self.live_config = extract_live_config(parser, self.plugins) self.secrets = extract_secrets(parser) self.throttles = tuple() # immutable since it's not real self.startup_timer.intermediate("zookeeper") ################# PRIVILEGED USERS self.admins = PermissionFilteredEmployeeList(self.live_config, type="admin") self.sponsors = PermissionFilteredEmployeeList(self.live_config, type="sponsor") self.employees = PermissionFilteredEmployeeList(self.live_config, type="employee") ################# MEMCACHE num_mc_clients = self.num_mc_clients # the main memcache pool. used for most everything. memcache = CMemcache( self.memcaches, min_compress_len=1400, num_clients=num_mc_clients, binary=True, ) # a pool just used for @memoize results memoizecaches = CMemcache( self.memoizecaches, min_compress_len=50 * 1024, num_clients=num_mc_clients, binary=True, ) # a pool just for srmember rels srmembercaches = CMemcache( self.srmembercaches, min_compress_len=96, num_clients=num_mc_clients, binary=True, ) # a pool just for rels relcaches = CMemcache( self.relcaches, min_compress_len=96, num_clients=num_mc_clients, binary=True, ) ratelimitcaches = CMemcache( self.ratelimitcaches, min_compress_len=96, num_clients=num_mc_clients, ) # a smaller pool of caches used only for distributed locks. # TODO: move this to ZooKeeper self.lock_cache = CMemcache(self.lockcaches, binary=True, num_clients=num_mc_clients) self.make_lock = make_lock_factory(self.lock_cache, self.stats) # memcaches used in front of the permacache CF in cassandra. # XXX: this is a legacy thing; permacache was made when C* didn't have # a row cache. permacache_memcaches = CMemcache(self.permacache_memcaches, min_compress_len=1400, num_clients=num_mc_clients) # the stalecache is a memcached local to the current app server used # for data that's frequently fetched but doesn't need to be fresh. if self.stalecaches: stalecaches = CMemcache(self.stalecaches, binary=True, num_clients=num_mc_clients) else: stalecaches = None # rendercache holds rendered partial templates. rendercaches = CMemcache( self.rendercaches, noreply=True, no_block=True, num_clients=num_mc_clients, min_compress_len=480, ) # pagecaches hold fully rendered pages pagecaches = CMemcache( self.pagecaches, noreply=True, no_block=True, num_clients=num_mc_clients, min_compress_len=1400, ) self.startup_timer.intermediate("memcache") ################# CASSANDRA keyspace = "reddit" self.cassandra_pools = { "main": StatsCollectingConnectionPool(keyspace, stats=self.stats, logging_name="main", server_list=self.cassandra_seeds, pool_size=self.cassandra_pool_size, timeout=4, max_retries=3, prefill=False), } permacache_cf = Permacache._setup_column_family( 'permacache', self.cassandra_pools[self.cassandra_default_pool], ) self.startup_timer.intermediate("cassandra") ################# POSTGRES self.dbm = self.load_db_params() self.startup_timer.intermediate("postgres") ################# CHAINS # initialize caches. Any cache-chains built here must be added # to cache_chains (closed around by reset_caches) so that they # can properly reset their local components cache_chains = {} localcache_cls = (SelfEmptyingCache if self.running_as_script else LocalCache) if stalecaches: self.cache = StaleCacheChain( localcache_cls(), stalecaches, memcache, ) else: self.cache = MemcacheChain((localcache_cls(), memcache)) cache_chains.update(cache=self.cache) if stalecaches: self.memoizecache = StaleCacheChain( localcache_cls(), stalecaches, memoizecaches, ) else: self.memoizecache = MemcacheChain( (localcache_cls(), memoizecaches)) cache_chains.update(memoizecache=self.memoizecache) if stalecaches: self.srmembercache = StaleCacheChain( localcache_cls(), stalecaches, srmembercaches, ) else: self.srmembercache = MemcacheChain( (localcache_cls(), srmembercaches)) cache_chains.update(srmembercache=self.srmembercache) if stalecaches: self.relcache = StaleCacheChain( localcache_cls(), stalecaches, relcaches, ) else: self.relcache = MemcacheChain((localcache_cls(), relcaches)) cache_chains.update(relcache=self.relcache) self.ratelimitcache = MemcacheChain( (localcache_cls(), ratelimitcaches)) cache_chains.update(ratelimitcache=self.ratelimitcache) self.rendercache = MemcacheChain(( localcache_cls(), rendercaches, )) cache_chains.update(rendercache=self.rendercache) self.pagecache = MemcacheChain(( localcache_cls(), pagecaches, )) cache_chains.update(pagecache=self.pagecache) # the thing_cache is used in tdb_cassandra. self.thing_cache = CacheChain((localcache_cls(), ), check_keys=False) cache_chains.update(thing_cache=self.thing_cache) if stalecaches: permacache_cache = StaleCacheChain( localcache_cls(), stalecaches, permacache_memcaches, check_keys=False, ) else: permacache_cache = CacheChain( (localcache_cls(), permacache_memcaches), check_keys=False, ) cache_chains.update(permacache=permacache_cache) self.permacache = Permacache( permacache_cache, permacache_cf, lock_factory=self.make_lock, ) # hardcache is used for various things that tend to expire # TODO: replace hardcache w/ cassandra stuff self.hardcache = HardcacheChain( (localcache_cls(), memcache, HardCache(self)), cache_negative_results=True, ) cache_chains.update(hardcache=self.hardcache) # I know this sucks, but we need non-request-threads to be # able to reset the caches, so we need them be able to close # around 'cache_chains' without being able to call getattr on # 'g' def reset_caches(): for name, chain in cache_chains.iteritems(): chain.reset() if isinstance(chain, StaleCacheChain): chain.stats = StaleCacheStats(self.stats, name) else: chain.stats = CacheStats(self.stats, name) self.cache_chains = cache_chains self.reset_caches = reset_caches self.reset_caches() self.startup_timer.intermediate("cache_chains") # try to set the source control revision numbers self.versions = {} r2_root = os.path.dirname(os.path.dirname(self.paths["root"])) r2_gitdir = os.path.join(r2_root, ".git") self.short_version = self.record_repo_version("r2", r2_gitdir) if I18N_PATH: i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git") self.record_repo_version("i18n", i18n_git_path) self.startup_timer.intermediate("revisions") def setup_complete(self): self.startup_timer.stop() self.stats.flush() if self.log_start: self.log.error("%s:%s started %s at %s (took %.02fs)", self.reddit_host, self.reddit_pid, self.short_version, datetime.now().strftime("%H:%M:%S"), self.startup_timer.elapsed_seconds()) def record_repo_version(self, repo_name, git_dir): """Get the currently checked out git revision for a given repository, record it in g.versions, and return the short version of the hash.""" try: subprocess.check_output except AttributeError: # python 2.6 compat pass else: try: revision = subprocess.check_output( ["git", "--git-dir", git_dir, "rev-parse", "HEAD"]) except subprocess.CalledProcessError, e: self.log.warning("Unable to fetch git revision: %r", e) else:
class FreeToPlay(Plugin): needs_static_build = True config = { ConfigValue.tuple: [ "f2pcaches", ], ConfigValue.dict(str, str): [ "team_subreddits", "steam_promo_items", ], } js = { 'reddit': Module('reddit.js', 'lib/iso8601.js', 'f2p/scrollupdater.js', 'f2p/f2p.js', 'f2p/utils.js', 'f2p/items.js', TemplateFileSource('f2p/panel.html'), TemplateFileSource('f2p/login-message.html'), TemplateFileSource('f2p/item.html'), TemplateFileSource('f2p/item-bubble.html'), TemplateFileSource('f2p/scores.html'), TemplateFileSource('f2p/target-overlay.html'), ) } live_config = { ConfigValue.float: [ 'drop_cooldown_mu', 'drop_cooldown_sigma', ], ConfigValue.dict(str, int): [ 'f2p_rarity_weights', ], } def declare_queues(self, queues): # imported here so we don't depend on pyx files at import time # which allows "make" to work in a clean clone of the repos from r2.config.queues import MessageQueue queues.declare({ "steam_q": MessageQueue(bind_to_self=True), }) def on_load(self, g): from r2.lib.cache import CMemcache, MemcacheChain, LocalCache # TODO: use SelfEmptyingCache for localcache if we use this in jobs f2p_memcaches = CMemcache( 'f2p', g.f2pcaches, num_clients=g.num_mc_clients, ) g.f2pcache = MemcacheChain(( LocalCache(), f2p_memcaches, )) g.cache_chains.update(f2p=g.f2pcache) compendium = pkg_resources.resource_stream(__name__, "data/compendium.json") g.f2pitems = json.load(compendium) for kind, data in g.f2pitems.iteritems(): data["kind"] = kind def add_routes(self, mc): mc('/f2p/gamelog', controller='gamelog', action='listing') mc('/api/f2p/:action', controller='freetoplayapi') mc('/f2p/steam/:action', controller='steam', action='start') def load_controllers(self): from r2.lib.pages import Reddit Reddit.extra_stylesheets.append('f2p.less') from reddit_f2p import f2p f2p.hooks.register_all() f2p.monkeypatch() from reddit_f2p.steam import SteamController from reddit_f2p.gamelog import GameLogController
class Adzerk(Plugin): needs_static_build = True config = { ConfigValue.str: [ 'adzerk_engine_domain', ], ConfigValue.int: [ 'az_selfserve_salesperson_id', 'az_selfserve_network_id', 'az_reporting_timeout', ], ConfigValue.float: [ 'display_ad_skip_probability', ], ConfigValue.tuple: [ 'display_ad_skip_keywords', ], ConfigValue.dict(ConfigValue.str, ConfigValue.int): [ 'az_selfserve_priorities', 'az_selfserve_site_ids', ], ConfigValue.tuple_of(ConfigValue.int): [ 'adserver_campaign_ids', ], } js = { 'reddit-init': Module('reddit-init.js', 'adzerk/adzerk.js', ), 'display': Module('display.js', 'adzerk/display.js', ), 'companion': Module('companion.js', 'adzerk/companion.js', ), 'ad-dependencies': Module('ad-dependencies.js', 'adzerk/jquery.js', ), } def add_routes(self, mc): mc('/api/request_promo/', controller='adzerkapi', action='request_promo') mc('/ads/display/300x250/', controller='adserving', action='ad_300_250') mc('/ads/display/300x250-companion/', controller='adserving', action='ad_300_250_companion') def declare_queues(self, queues): from r2.config.queues import MessageQueue queues.declare({ "adzerk_q": MessageQueue(bind_to_self=True), "adzerk_reporting_q": MessageQueue(bind_to_self=True), }) def load_controllers(self): # replace the standard Ads view with an Adzerk specific one. import r2.lib.pages.pages from adzerkads import Ads as AdzerkAds r2.lib.pages.pages.Ads = AdzerkAds # replace standard adserver with Adzerk. from adzerkpromote import AdzerkApiController from adzerkpromote import hooks as adzerkpromote_hooks from adzerkads import AdServingController adzerkpromote_hooks.register_all()
def test_timeinterval(self): self.assertEquals(datetime.timedelta(0, 60), ConfigValue.timeinterval('1 minute')) with self.assertRaises(KeyError): ConfigValue.timeinterval('asdf')
def load_db_params(self): self.databases = tuple(ConfigValue.to_iter(self.config.raw_data['databases'])) self.db_params = {} if not self.databases: return dbm = db_manager.db_manager() db_param_names = ('name', 'db_host', 'db_user', 'db_pass', 'db_port', 'pool_size', 'max_overflow') for db_name in self.databases: conf_params = ConfigValue.to_iter(self.config.raw_data[db_name + '_db']) params = dict(zip(db_param_names, conf_params)) if params['db_user'] == "*": params['db_user'] = self.db_user if params['db_pass'] == "*": params['db_pass'] = self.db_pass if params['db_port'] == "*": params['db_port'] = self.db_port if params['pool_size'] == "*": params['pool_size'] = self.db_pool_size if params['max_overflow'] == "*": params['max_overflow'] = self.db_pool_overflow_size dbm.setup_db(db_name, g_override=self, **params) self.db_params[db_name] = params dbm.type_db = dbm.get_engine(self.config.raw_data['type_db']) dbm.relation_type_db = dbm.get_engine(self.config.raw_data['rel_type_db']) def split_flags(raw_params): params = [] flags = {} for param in raw_params: if not param.startswith("!"): params.append(param) else: key, sep, value = param[1:].partition("=") if sep: flags[key] = value else: flags[key] = True return params, flags prefix = 'db_table_' self.predefined_type_ids = {} for k, v in self.config.raw_data.iteritems(): if not k.startswith(prefix): continue params, table_flags = split_flags(ConfigValue.to_iter(v)) name = k[len(prefix):] kind = params[0] server_list = self.config.raw_data["db_servers_" + name] engines, flags = split_flags(ConfigValue.to_iter(server_list)) typeid = table_flags.get("typeid") if typeid: self.predefined_type_ids[name] = int(typeid) if kind == 'thing': dbm.add_thing(name, dbm.get_engines(engines), **flags) elif kind == 'relation': dbm.add_relation(name, params[1], params[2], dbm.get_engines(engines), **flags) return dbm
def load_db_params(self): self.databases = tuple( ConfigValue.to_iter(self.config.raw_data['databases'])) self.db_params = {} if not self.databases: return dbm = db_manager.db_manager() db_param_names = ('name', 'db_host', 'db_user', 'db_pass', 'db_port', 'pool_size', 'max_overflow') for db_name in self.databases: conf_params = ConfigValue.to_iter(self.config.raw_data[db_name + '_db']) params = dict(zip(db_param_names, conf_params)) if params['db_user'] == "*": params['db_user'] = self.db_user if params['db_pass'] == "*": params['db_pass'] = self.db_pass if params['db_port'] == "*": params['db_port'] = self.db_port if params['pool_size'] == "*": params['pool_size'] = self.db_pool_size if params['max_overflow'] == "*": params['max_overflow'] = self.db_pool_overflow_size dbm.setup_db(db_name, g_override=self, **params) self.db_params[db_name] = params dbm.type_db = dbm.get_engine(self.config.raw_data['type_db']) dbm.relation_type_db = dbm.get_engine( self.config.raw_data['rel_type_db']) def split_flags(raw_params): params = [] flags = {} for param in raw_params: if not param.startswith("!"): params.append(param) else: key, sep, value = param[1:].partition("=") if sep: flags[key] = value else: flags[key] = True return params, flags prefix = 'db_table_' self.predefined_type_ids = {} for k, v in self.config.raw_data.iteritems(): if not k.startswith(prefix): continue params, table_flags = split_flags(ConfigValue.to_iter(v)) name = k[len(prefix):] kind = params[0] server_list = self.config.raw_data["db_servers_" + name] engines, flags = split_flags(ConfigValue.to_iter(server_list)) typeid = table_flags.get("typeid") if typeid: self.predefined_type_ids[name] = int(typeid) if kind == 'thing': dbm.add_thing(name, dbm.get_engines(engines), **flags) elif kind == 'relation': dbm.add_relation(name, params[1], params[2], dbm.get_engines(engines), **flags) return dbm
class Robin(Plugin): needs_static_build = True js = { "robin": LocalizedModule("robin.js", "lib/page-visibility.js", "lib/tinycon.js", "websocket.js", TemplateFileSource("robin/robinmessage.html"), TemplateFileSource("robin/robinroomparticipant.html"), "errors.js", "models/validators.js", "robin/models.js", "robin/views.js", "robin/notifications.js", "robin/favicon.js", "robin/init.js", ), "robin-join": Module("robin-join.js", "robin/join.js", ), } live_config = { ConfigValue.int: [ "robin_ratelimit_window", ], ConfigValue.dict(ConfigValue.int, ConfigValue.float): [ "robin_ratelimit_avg_per_sec", ], } def declare_queues(self, queues): from r2.config.queues import MessageQueue queues.declare({ "robin_presence_q": MessageQueue(), "robin_waitinglist_q": MessageQueue(bind_to_self=True), "robin_subreddit_maker_q": MessageQueue(bind_to_self=True), }) queues.robin_presence_q << ( "websocket.connect", "websocket.disconnect", ) def add_routes(self, mc): mc("/robin", controller="robin", action="chat", conditions={"function": not_in_sr}) mc("/robin/all", controller="robin", action="all", conditions={"function": not_in_sr}) mc("/robin/admin", controller="robin", action="admin", conditions={"function": not_in_sr}) mc("/robin/join", controller="robin", action="join", conditions={"function": not_in_sr}) mc("/robin/:room_id", controller="robin", action="force_room", conditions={"function": not_in_sr}) mc("/robin/user/:user", controller="robin", action="user_room", conditions={"function": not_in_sr}) mc("/api/robin/:room_id/:action", controller="robin", conditions={"function": not_in_sr}) mc("/api/join_room", controller="robin", action="join_room", conditions={"function": not_in_sr}) mc("/api/room_assignment", controller="robin", action="room_assignment", conditions={"function": not_in_sr}) mc("/api/admin_prompt", controller="robin", action="admin_prompt", conditions={"function": not_in_sr}) mc("/api/admin_reap", controller="robin", action="admin_reap", conditions={"function": not_in_sr}) mc("/api/admin_broadcast", controller="robin", action="admin_broadcast", conditions={"function": not_in_sr}) def load_controllers(self): from r2.lib.pages import Reddit from reddit_robin.controllers import ( RobinController, ) Reddit.extra_stylesheets.append('robin_global.less') from reddit_robin.hooks import hooks hooks.register_all()