Beispiel #1
0
def load_environment(global_conf={}, app_conf={}, setup_globals=True):
    # Setup our paths
    root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    paths = {'root': root_path,
             'controllers': os.path.join(root_path, 'controllers'),
             'templates': [os.path.join(root_path, 'templates')],
             }

    if ConfigValue.bool(global_conf.get('uncompressedJS')):
        paths['static_files'] = os.path.join(root_path, 'public')
    else:
        paths['static_files'] = os.path.join(os.path.dirname(root_path), 'build/public')

    config.init_app(global_conf, app_conf, package='r2',
                    template_engine='mako', paths=paths)

    g = config['pylons.g'] = Globals(global_conf, app_conf, paths)
    if setup_globals:
        g.setup()
        g.plugins.declare_queues(g.queues)
        r2.config.cache = g.cache
    g.plugins.load_plugins()
    config['r2.plugins'] = g.plugins
    g.startup_timer.intermediate("plugins")

    config['pylons.h'] = r2.lib.helpers
    config['routes.map'] = routing.make_map()

    #override the default response options
    config['pylons.response_options']['headers'] = {}

    # The following template options are passed to your template engines
    tmpl_options = config['buffet.template_options']
    tmpl_options['mako.filesystem_checks'] = getattr(g, 'reload_templates', False)
    tmpl_options['mako.default_filters'] = ["mako_websafe"]
    tmpl_options['mako.imports'] = \
                                 ["from r2.lib.filters import websafe, unsafe, mako_websafe",
                                  "from pylons import c, g, request",
                                  "from pylons.i18n import _, ungettext"]

    # when mako loads a previously compiled template file from its cache, it
    # doesn't check that the original template path matches the current path.
    # in the event that a new plugin defines a template overriding a reddit
    # template, unless the mtime newer, mako doesn't update the compiled
    # template. as a workaround, this makes mako store compiled templates with
    # the original path in the filename, forcing it to update with the path.
    def mako_module_path(filename, uri):
        module_directory = tmpl_options['mako.module_directory']
        filename = filename.lstrip('/').replace('/', '-')
        path = os.path.join(module_directory, filename + ".py")
        return os.path.abspath(path)

    tmpl_options['mako.modulename_callable'] = mako_module_path

    if setup_globals:
        g.setup_complete()
Beispiel #2
0
def load_environment(global_conf={}, app_conf={}, setup_globals=True):
    # Setup our paths
    root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    paths = {
        'root': root_path,
        'controllers': os.path.join(root_path, 'controllers'),
        'templates': tmpl_dirs,
    }

    if ConfigValue.bool(global_conf.get('uncompressedJS')):
        paths['static_files'] = os.path.join(root_path, 'public')
    else:
        paths['static_files'] = os.path.join(os.path.dirname(root_path),
                                             'build/public')

    config.init_app(global_conf,
                    app_conf,
                    package='r2',
                    template_engine='mako',
                    paths=paths)

    g = config['pylons.g'] = Globals(global_conf, app_conf, paths)
    if setup_globals:
        g.setup()
        r2.config.cache = g.cache

    config['pylons.h'] = r2.lib.helpers

    g.plugins = config['r2.plugins'] = PluginLoader().load_plugins(
        g.config.get('plugins', []))
    config['routes.map'] = routing.make_map()

    #override the default response options
    config['pylons.response_options']['headers'] = {}

    # The following template options are passed to your template engines
    #tmpl_options = {}
    #tmpl_options['myghty.log_errors'] = True
    #tmpl_options['myghty.escapes'] = dict(l=webhelpers.auto_link, s=webhelpers.simple_format)

    tmpl_options = config['buffet.template_options']
    tmpl_options['mako.filesystem_checks'] = getattr(g, 'reload_templates',
                                                     False)
    tmpl_options['mako.default_filters'] = ["mako_websafe"]
    tmpl_options['mako.imports'] = \
                                 ["from r2.lib.filters import websafe, unsafe, mako_websafe",
                                  "from pylons import c, g, request",
                                  "from pylons.i18n import _, ungettext"]
Beispiel #3
0
 def test_dict(self):
     self.assertEquals({}, ConfigValue.dict(str, str)(''))
     self.assertEquals({'a': ''}, ConfigValue.dict(str, str)('a'))
     self.assertEquals({'a': 3}, ConfigValue.dict(str, int)('a: 3'))
     self.assertEquals({'a': 3, 'b': 4},
                       ConfigValue.dict(str, int)('a: 3, b: 4'))
     self.assertEquals({'a': (3, 5), 'b': (4, 6)},
                       ConfigValue.dict(
                           str, ConfigValue.tuple_of(int), delim=';')
                       ('a: 3, 5;  b: 4, 6'))
class Adzerk(Plugin):
    needs_static_build = True

    config = {
        ConfigValue.int: [
            'az_selfserve_site_id',
            'az_selfserve_advertiser_id',
            'az_selfserve_channel_id',
            'az_selfserve_publisher_id',
            'az_selfserve_network_id',
            'az_selfserve_ad_type',
            'az_selfserve_num_request',
        ],
        ConfigValue.dict(ConfigValue.str, ConfigValue.int): [
            'az_selfserve_priorities',
        ],
    }

    js = {
        'reddit-init': Module(
            'reddit-init.js',
            'adzerk/adzerk.js',
        )
    }

    def add_routes(self, mc):
        mc('/api/request_promo/',
           controller='adzerkapi',
           action='request_promo')

    def declare_queues(self, queues):
        from r2.config.queues import MessageQueue
        queues.declare({
            "adzerk_q": MessageQueue(bind_to_self=True),
        })

    def load_controllers(self):
        # replace the standard Ads view with an Adzerk specific one.
        import r2.lib.pages.pages
        from adzerkads import Ads as AdzerkAds
        r2.lib.pages.pages.Ads = AdzerkAds

        # replace standard adserver with Adzerk.
        from adzerkpromote import AdzerkApiController
        from adzerkpromote import hooks as adzerkpromote_hooks
        adzerkpromote_hooks.register_all()
Beispiel #5
0
def load_environment(global_conf={}, app_conf={}, setup_globals=True):
    # Setup our paths
    root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    paths = {
        "root": root_path,
        "controllers": os.path.join(root_path, "controllers"),
        "templates": [os.path.join(root_path, "templates")],
    }

    if ConfigValue.bool(global_conf.get("uncompressedJS")):
        paths["static_files"] = os.path.join(root_path, "public")
    else:
        paths["static_files"] = os.path.join(os.path.dirname(root_path), "build/public")

    config.init_app(global_conf, app_conf, package="r2", template_engine="mako", paths=paths)

    g = config["pylons.g"] = Globals(global_conf, app_conf, paths)
    if setup_globals:
        g.setup()
        r2.config.cache = g.cache
    g.plugins.load_plugins()
    config["r2.plugins"] = g.plugins

    config["pylons.h"] = r2.lib.helpers
    config["routes.map"] = routing.make_map()

    # override the default response options
    config["pylons.response_options"]["headers"] = {}

    # The following template options are passed to your template engines
    # tmpl_options = {}
    # tmpl_options['myghty.log_errors'] = True
    # tmpl_options['myghty.escapes'] = dict(l=webhelpers.auto_link, s=webhelpers.simple_format)

    tmpl_options = config["buffet.template_options"]
    tmpl_options["mako.filesystem_checks"] = getattr(g, "reload_templates", False)
    tmpl_options["mako.default_filters"] = ["mako_websafe"]
    tmpl_options["mako.imports"] = [
        "from r2.lib.filters import websafe, unsafe, mako_websafe",
        "from pylons import c, g, request",
        "from pylons.i18n import _, ungettext",
    ]
Beispiel #6
0
def load_environment(global_conf={}, app_conf={}, setup_globals=True):
    # Setup our paths
    root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    paths = {'root': root_path,
             'controllers': os.path.join(root_path, 'controllers'),
             'templates': tmpl_dirs,
             }

    if ConfigValue.bool(global_conf.get('uncompressedJS')):
        paths['static_files'] = os.path.join(root_path, 'public')
    else:
        paths['static_files'] = os.path.join(os.path.dirname(root_path), 'build/public')

    config.init_app(global_conf, app_conf, package='r2',
                    template_engine='mako', paths=paths)

    g = config['pylons.g'] = Globals(global_conf, app_conf, paths)
    if setup_globals:
        g.setup()
        reddit_config.cache = g.cache

    config['pylons.h'] = r2.lib.helpers

    g.plugins = config['r2.plugins'] = PluginLoader().load_plugins(g.config.get('plugins', []))
    config['routes.map'] = routing.make_map()

    #override the default response options
    config['pylons.response_options']['headers'] = {}

    # The following template options are passed to your template engines
    #tmpl_options = {}
    #tmpl_options['myghty.log_errors'] = True
    #tmpl_options['myghty.escapes'] = dict(l=webhelpers.auto_link, s=webhelpers.simple_format)

    tmpl_options = config['buffet.template_options']
    tmpl_options['mako.filesystem_checks'] = getattr(g, 'reload_templates', False)
    tmpl_options['mako.default_filters'] = ["mako_websafe"]
    tmpl_options['mako.imports'] = \
                                 ["from r2.lib.filters import websafe, unsafe, mako_websafe",
                                  "from pylons import c, g, request",
                                  "from pylons.i18n import _, ungettext"]
Beispiel #7
0
 def test_dict(self):
     self.assertEquals({}, ConfigValue.dict(str, str)(''))
     self.assertEquals({'a': ''}, ConfigValue.dict(str, str)('a'))
     self.assertEquals({'a': 3}, ConfigValue.dict(str, int)('a: 3'))
     self.assertEquals({
         'a': 3,
         'b': 4
     },
                       ConfigValue.dict(str, int)('a: 3, b: 4'))
     self.assertEquals({
         'a': (3, 5),
         'b': (4, 6)
     },
                       ConfigValue.dict(str,
                                        ConfigValue.tuple_of(int),
                                        delim=';')('a: 3, 5;  b: 4, 6'))
Beispiel #8
0
class LiveUpdate(Plugin):
    needs_static_build = True

    errors = {
        "LIVEUPDATE_NOT_CONTRIBUTOR":
        N_("that user is not a contributor"),
        "LIVEUPDATE_NO_INVITE_FOUND":
        N_("there is no pending invite for that thread"),
        "LIVEUPDATE_TOO_MANY_INVITES":
        N_("there are too many pending invites outstanding"),
        "LIVEUPDATE_ALREADY_CONTRIBUTOR":
        N_("that user is already a contributor"),
        "LIVEUPDATE_LINK_IS_NOT_DISCUSSION":
        N_("the specified link is not a discussion about this live thread"),
    }

    config = {
        ConfigValue.int: [
            "liveupdate_invite_quota",
            "liveupdate_min_score_for_discussions",
        ],
        ConfigValue.str: [
            "liveupdate_pixel_domain",
        ],
        ConfigValue.baseplate(Date): [
            "liveupdate_min_date_viewcounts",
        ],
    }

    js = {
        "liveupdate":
        LocalizedModule(
            "liveupdate.js",
            "lib/page-visibility.js",
            "lib/tinycon.js",
            "lib/moment.js",
            "websocket.js",
            "liveupdate/init.js",
            "liveupdate/activity.js",
            "liveupdate/embeds.js",
            "liveupdate/event.js",
            "liveupdate/favicon.js",
            "liveupdate/listings.js",
            "liveupdate/notifications.js",
            "liveupdate/statusBar.js",
            "liveupdate/report.js",
            TemplateFileSource("liveupdate/update.html"),
            TemplateFileSource("liveupdate/separator.html"),
            TemplateFileSource("liveupdate/edit-button.html"),
            TemplateFileSource("liveupdate/reported.html"),
            PermissionsDataSource({
                "liveupdate_contributor":
                ContributorPermissionSet,
                "liveupdate_contributor_invite":
                ContributorPermissionSet,
            }),
            localized_appendices=[
                MomentTranslations(),
            ],
        ),
    }

    def add_routes(self, mc):
        mc(
            "/live",
            controller="liveupdateevents",
            action="home",
            conditions={"function": not_in_sr},
        )

        mc(
            "/live/create",
            controller="liveupdateevents",
            action="create",
            conditions={"function": not_in_sr},
        )

        mc(
            "/api/live/by_id/:names",
            action="listing",
            controller="liveupdatebyid",
        )

        mc(
            "/live/:filter",
            action="listing",
            controller="liveupdateevents",
            conditions={"function": not_in_sr},
            requirements={
                "filter": "open|closed|reported|active|happening_now|mine"
            },
        )

        mc(
            "/api/live/:action",
            controller="liveupdateevents",
            conditions={"function": not_in_sr},
            requirements={"action": "create|happening_now"},
        )

        mc("/live/:event",
           controller="liveupdate",
           action="listing",
           conditions={"function": not_in_sr},
           is_embed=False)

        mc("/live/:event/embed",
           controller="liveupdate",
           action="listing",
           conditions={"function": not_in_sr},
           is_embed=True)

        mc(
            "/live/:event/updates/:target",
            controller="liveupdate",
            action="focus",
            conditions={"function": not_in_sr},
        )

        mc("/live/:event/pixel",
           controller="liveupdatepixel",
           action="pixel",
           conditions={"function": not_in_sr})

        mc("/live/:event/:action",
           controller="liveupdate",
           conditions={"function": not_in_sr})

        mc("/api/live/:event/:action",
           controller="liveupdate",
           conditions={"function": not_in_sr})

        mc('/mediaembed/liveupdate/:event/:liveupdate/:embed_index',
           controller="liveupdateembed",
           action="mediaembed")

        mc('/admin/happening-now',
           controller='liveupdateadmin',
           action='happening_now')

    def load_controllers(self):
        from r2.controllers.api_docs import api_section, section_info
        api_section["live"] = "live"
        section_info["live"] = {
            "title": "live threads",
            "description": sys.modules[__name__].__doc__,
        }

        from r2.models.token import OAuth2Scope
        OAuth2Scope.scope_info["livemanage"] = {
            "id":
            "livemanage",
            "name":
            N_("Manage live threads"),
            "description":
            N_("Manage settings and contributors of live threads "
               "I contribute to."),
        }

        from reddit_liveupdate.controllers import (
            controller_hooks,
            LiveUpdateByIDController,
            LiveUpdateController,
            LiveUpdateEventsController,
            LiveUpdatePixelController,
        )

        from r2.config.templates import api
        from r2.lib.jsontemplates import ListingJsonTemplate
        from reddit_liveupdate import pages
        api('liveupdateeventapp', pages.LiveUpdateEventAppJsonTemplate)
        api('liveupdatefocusapp', pages.LiveUpdateEventAppJsonTemplate)
        api('liveupdateevent', pages.LiveUpdateEventJsonTemplate)
        api('liveupdatereportedeventrow', pages.LiveUpdateEventJsonTemplate)
        api('liveupdatefeaturedevent',
            pages.LiveUpdateFeaturedEventJsonTemplate)
        api('liveupdate', pages.LiveUpdateJsonTemplate)
        api('liveupdatecontributortableitem',
            pages.ContributorTableItemJsonTemplate)
        api('liveupdatediscussionslisting', ListingJsonTemplate)

        controller_hooks.register_all()

        from reddit_liveupdate import scraper
        scraper.hooks.register_all()

    def declare_queues(self, queues):
        from r2.config.queues import MessageQueue
        queues.declare({
            "liveupdate_scraper_q": MessageQueue(bind_to_self=True),
        })

        queues.liveupdate_scraper_q << ("new_liveupdate_update", )

    source_root_url = "https://github.com/reddit/reddit-plugin-liveupdate/blob/master/reddit_liveupdate/"

    def get_documented_controllers(self):
        from reddit_liveupdate.controllers import (
            LiveUpdateController,
            LiveUpdateEventsController,
            LiveUpdateByIDController,
        )

        yield LiveUpdateController, "/api/live/{thread}"
        yield LiveUpdateEventsController, ""
        yield LiveUpdateByIDController, ""
Beispiel #9
0
def load_environment(global_conf={}, app_conf={}, setup_globals=True):
    r2_path = get_r2_path()
    root_path = os.path.join(r2_path, "r2")

    paths = {
        "root": root_path,
        "controllers": os.path.join(root_path, "controllers"),
        "templates": [os.path.join(root_path, "templates")],
    }

    if ConfigValue.bool(global_conf.get("uncompressedJS")):
        paths["static_files"] = get_raw_statics_path()
    else:
        paths["static_files"] = get_built_statics_path()

    config = PylonsConfig()

    config.init_app(global_conf, app_conf, package="r2", paths=paths)

    # don't put action arguments onto c automatically
    config["pylons.c_attach_args"] = False

    # when accessing non-existent attributes on c, return "" instead of dying
    config["pylons.strict_tmpl_context"] = False

    g = Globals(config, global_conf, app_conf, paths)
    config["pylons.app_globals"] = g

    if setup_globals:
        config["r2.import_private"] = ConfigValue.bool(global_conf["import_private"])
        g.setup()
        g.plugins.declare_queues(g.queues)

    g.plugins.load_plugins(config)
    config["r2.plugins"] = g.plugins
    g.startup_timer.intermediate("plugins")

    config["pylons.h"] = r2.lib.helpers
    config["routes.map"] = make_map(config)

    # override the default response options
    config["pylons.response_options"]["headers"] = {}

    # when mako loads a previously compiled template file from its cache, it
    # doesn't check that the original template path matches the current path.
    # in the event that a new plugin defines a template overriding a reddit
    # template, unless the mtime newer, mako doesn't update the compiled
    # template. as a workaround, this makes mako store compiled templates with
    # the original path in the filename, forcing it to update with the path.
    if "cache_dir" in app_conf:
        module_directory = os.path.join(app_conf["cache_dir"], "templates")

        def mako_module_path(filename, uri):
            filename = filename.lstrip("/").replace("/", "-")
            path = os.path.join(module_directory, filename + ".py")
            return os.path.abspath(path)

    else:
        # disable caching templates since we don't know where they should go.
        module_directory = mako_module_path = None

    # set up the templating system
    config["pylons.app_globals"].mako_lookup = TemplateLookup(
        directories=paths["templates"],
        error_handler=handle_mako_error,
        module_directory=module_directory,
        input_encoding="utf-8",
        default_filters=["conditional_websafe"],
        filesystem_checks=getattr(g, "reload_templates", False),
        imports=[
            "from r2.lib.filters import websafe, unsafe, conditional_websafe",
            "from pylons import request",
            "from pylons import tmpl_context as c",
            "from pylons import app_globals as g",
            "from pylons.i18n import _, ungettext",
        ],
        modulename_callable=mako_module_path,
    )

    if setup_globals:
        g.setup_complete()

    return config
Beispiel #10
0
 def test_set(self):
     self.assertEquals(set([]), ConfigValue.set(''))
     self.assertEquals(set(['a', 'b']), ConfigValue.set('a, b'))
Beispiel #11
0
 def test_tuple_of(self):
     self.assertEquals((), ConfigValue.tuple_of(str)(''))
     self.assertEquals(('a', 'b'), ConfigValue.tuple_of(str)('a, b'))
     self.assertEquals(('a', 'b'),
                       ConfigValue.tuple_of(str, delim=':')('a : b'))
Beispiel #12
0
 def test_int(self):
     self.assertEquals(3, ConfigValue.int('3'))
     self.assertEquals(-3, ConfigValue.int('-3'))
     with self.assertRaises(ValueError):
         ConfigValue.int('asdf')
Beispiel #13
0
 def test_bool(self):
     self.assertEquals(True, ConfigValue.bool('TrUe'))
     self.assertEquals(False, ConfigValue.bool('fAlSe'))
     with self.assertRaises(ValueError):
         ConfigValue.bool('asdf')
Beispiel #14
0
 def test_tuple(self):
     self.assertEquals((), ConfigValue.tuple(''))
     self.assertEquals(('a', 'b'), ConfigValue.tuple('a, b'))
Beispiel #15
0
class Globals(object):
    spec = {

        ConfigValue.int: [
            'db_pool_size',
            'db_pool_overflow_size',
            'page_cache_time',
            'commentpane_cache_time',
            'num_mc_clients',
            'MAX_CAMPAIGNS_PER_LINK',
            'MIN_DOWN_LINK',
            'MIN_UP_KARMA',
            'MIN_DOWN_KARMA',
            'MIN_RATE_LIMIT_KARMA',
            'MIN_RATE_LIMIT_COMMENT_KARMA',
            'VOTE_AGE_LIMIT',
            'REPLY_AGE_LIMIT',
            'REPORT_AGE_LIMIT',
            'HOT_PAGE_AGE',
            'RATELIMIT',
            'QUOTA_THRESHOLD',
            'ADMIN_COOKIE_TTL',
            'ADMIN_COOKIE_MAX_IDLE',
            'OTP_COOKIE_TTL',
            'num_comments',
            'max_comments',
            'max_comments_gold',
            'num_default_reddits',
            'max_sr_images',
            'num_serendipity',
            'sr_dropdown_threshold',
            'comment_visits_period',
            'min_membership_create_community',
            'bcrypt_work_factor',
            'cassandra_pool_size',
            'sr_banned_quota',
            'sr_wikibanned_quota',
            'sr_wikicontributor_quota',
            'sr_moderator_invite_quota',
            'sr_contributor_quota',
            'sr_quota_time',
            'wiki_keep_recent_days',
            'wiki_max_page_length_bytes',
            'wiki_max_page_name_length',
            'wiki_max_page_separators',
        ],

        ConfigValue.float: [
            'min_promote_bid',
            'max_promote_bid',
            'statsd_sample_rate',
            'querycache_prune_chance',
        ],

        ConfigValue.bool: [
            'debug',
            'log_start',
            'sqlprinting',
            'template_debug',
            'reload_templates',
            'uncompressedJS',
            'css_killswitch',
            'db_create_tables',
            'disallow_db_writes',
            'disable_ratelimit',
            'amqp_logging',
            'read_only_mode',
            'disable_wiki',
            'heavy_load_mode',
            's3_media_direct',
            'disable_captcha',
            'disable_ads',
            'disable_require_admin_otp',
            'static_pre_gzipped',
            'static_secure_pre_gzipped',
            'trust_local_proxies',
            'shard_link_vote_queues',
        ],

        ConfigValue.tuple: [
            'plugins',
            'stalecaches',
            'memcaches',
            'lockcaches',
            'permacache_memcaches',
            'rendercaches',
            'pagecaches',
            'cassandra_seeds',
            'admins',
            'sponsors',
            'automatic_reddits',
            'agents',
            'allowed_css_linked_domains',
            'authorized_cnames',
            'hardcache_categories',
            's3_media_buckets',
            'allowed_pay_countries',
            'case_sensitive_domains',
            'reserved_subdomains',
            'TRAFFIC_LOG_HOSTS',
            'exempt_login_user_agents',
            'timed_templates',
        ],

        ConfigValue.str: [
            'wiki_page_registration_info',
            'wiki_page_privacy_policy',
            'wiki_page_user_agreement',
        ],

        ConfigValue.choice: {
             'cassandra_rcl': {
                 'ONE': CL_ONE,
                 'QUORUM': CL_QUORUM
             },
             'cassandra_wcl': {
                 'ONE': CL_ONE,
                 'QUORUM': CL_QUORUM
             },
        },

        config_gold_price: [
            'gold_month_price',
            'gold_year_price',
        ],
    }

    live_config_spec = {
        ConfigValue.bool: [
            'frontpage_dart',
        ],
        ConfigValue.float: [
            'spotlight_interest_sub_p',
            'spotlight_interest_nosub_p',
        ],
        ConfigValue.tuple: [
            'sr_discovery_links',
            'fastlane_links',
        ],
        ConfigValue.dict(ConfigValue.int, ConfigValue.float): [
            'comment_tree_version_weights',
        ],
        ConfigValue.messages: [
            'goldvertisement_blurbs',
            'goldvertisement_has_gold_blurbs',
            'welcomebar_messages',
            'sidebar_message',
            'gold_sidebar_message',
        ],
    }

    def __init__(self, global_conf, app_conf, paths, **extra):
        """
        Globals acts as a container for objects available throughout
        the life of the application.

        One instance of Globals is created by Pylons during
        application initialization and is available during requests
        via the 'g' variable.

        ``global_conf``
            The same variable used throughout ``config/middleware.py``
            namely, the variables from the ``[DEFAULT]`` section of the
            configuration file.

        ``app_conf``
            The same ``kw`` dictionary used throughout
            ``config/middleware.py`` namely, the variables from the
            section in the config file for your application.

        ``extra``
            The configuration returned from ``load_config`` in 
            ``config/middleware.py`` which may be of use in the setup of
            your global variables.

        """

        global_conf.setdefault("debug", False)

        self.config = ConfigValueParser(global_conf)
        self.config.add_spec(self.spec)
        self.plugins = PluginLoader(self.config.get("plugins", []))

        self.stats = Stats(self.config.get('statsd_addr'),
                           self.config.get('statsd_sample_rate'))
        self.startup_timer = self.stats.get_timer("app_startup")
        self.startup_timer.start()

        self.paths = paths

        self.running_as_script = global_conf.get('running_as_script', False)
        
        # turn on for language support
        self.lang = getattr(self, 'site_lang', 'en')
        self.languages, self.lang_name = \
            get_active_langs(default_lang=self.lang)

        all_languages = self.lang_name.keys()
        all_languages.sort()
        self.all_languages = all_languages
        
        # set default time zone if one is not set
        tz = global_conf.get('timezone', 'UTC')
        self.tz = pytz.timezone(tz)
        
        dtz = global_conf.get('display_timezone', tz)
        self.display_tz = pytz.timezone(dtz)

        self.startup_timer.intermediate("init")

    def __getattr__(self, name):
        if not name.startswith('_') and name in self.config:
            return self.config[name]
        else:
            raise AttributeError

    def setup(self):
        self.queues = queues.declare_queues(self)

        ################# CONFIGURATION
        # AMQP is required
        if not self.amqp_host:
            raise ValueError("amqp_host not set in the .ini")

        if not self.cassandra_seeds:
            raise ValueError("cassandra_seeds not set in the .ini")

        # heavy load mode is read only mode with a different infobar
        if self.heavy_load_mode:
            self.read_only_mode = True

        origin_prefix = self.domain_prefix + "." if self.domain_prefix else ""
        self.origin = "http://" + origin_prefix + self.domain
        self.secure_domains = set([urlparse(self.payment_domain).netloc])

        self.trusted_domains = set([self.domain])
        self.trusted_domains.update(self.authorized_cnames)
        if self.https_endpoint:
            https_url = urlparse(self.https_endpoint)
            self.secure_domains.add(https_url.netloc)
            self.trusted_domains.add(https_url.hostname)
        if getattr(self, 'oauth_domain', None):
            self.secure_domains.add(self.oauth_domain)

        # load the unique hashed names of files under static
        static_files = os.path.join(self.paths.get('static_files'), 'static')
        names_file_path = os.path.join(static_files, 'names.json')
        if os.path.exists(names_file_path):
            with open(names_file_path) as handle:
                self.static_names = json.load(handle)
        else:
            self.static_names = {}

        # make python warnings go through the logging system
        logging.captureWarnings(capture=True)

        log = logging.getLogger('reddit')

        # when we're a script (paster run) just set up super simple logging
        if self.running_as_script:
            log.setLevel(logging.INFO)
            log.addHandler(logging.StreamHandler())

        # if in debug mode, override the logging level to DEBUG
        if self.debug:
            log.setLevel(logging.DEBUG)

        # attempt to figure out which pool we're in and add that to the
        # LogRecords.
        try:
            with open("/etc/ec2_asg", "r") as f:
                pool = f.read().strip()
            # clean up the pool name since we're putting stuff after "-"
            pool = pool.partition("-")[0]
        except IOError:
            pool = "reddit-app"
        self.log = logging.LoggerAdapter(log, {"pool": pool})

        # make cssutils use the real logging system
        csslog = logging.getLogger("cssutils")
        cssutils.log.setLog(csslog)

        # load the country list
        countries_file_path = os.path.join(static_files, "countries.json")
        try:
            with open(countries_file_path) as handle:
                self.countries = json.load(handle)
            self.log.debug("Using countries.json.")
        except IOError:
            self.log.warning("Couldn't find countries.json. Using pycountry.")
            self.countries = get_countries_and_codes()

        if not self.media_domain:
            self.media_domain = self.domain
        if self.media_domain == self.domain:
            print ("Warning: g.media_domain == g.domain. " +
                   "This may give untrusted content access to user cookies")

        for arg in sys.argv:
            tokens = arg.split("=")
            if len(tokens) == 2:
                k, v = tokens
                self.log.debug("Overriding g.%s to %s" % (k, v))
                setattr(self, k, v)

        self.reddit_host = socket.gethostname()
        self.reddit_pid  = os.getpid()

        if hasattr(signal, 'SIGUSR1'):
            # not all platforms have user signals
            signal.signal(signal.SIGUSR1, thread_dump)

        self.startup_timer.intermediate("configuration")

        ################# ZOOKEEPER
        # for now, zookeeper will be an optional part of the stack.
        # if it's not configured, we will grab the expected config from the
        # [live_config] section of the ini file
        zk_hosts = self.config.get("zookeeper_connection_string")
        if zk_hosts:
            from r2.lib.zookeeper import (connect_to_zookeeper,
                                          LiveConfig, LiveList)
            zk_username = self.config["zookeeper_username"]
            zk_password = self.config["zookeeper_password"]
            self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username,
                                                             zk_password))
            self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE)
            self.throttles = LiveList(self.zookeeper, "/throttles",
                                      map_fn=ipaddress.ip_network,
                                      reduce_fn=ipaddress.collapse_addresses)
            self.banned_domains = LiveDict(self.zookeeper, 
                                           "/banned-domains",
                                           watch=True)
        else:
            self.zookeeper = None
            parser = ConfigParser.RawConfigParser()
            parser.read([self.config["__file__"]])
            self.live_config = extract_live_config(parser, self.plugins)
            self.throttles = tuple()  # immutable since it's not real
            self.banned_domains = dict()
        self.startup_timer.intermediate("zookeeper")

        ################# MEMCACHE
        num_mc_clients = self.num_mc_clients

        # the main memcache pool. used for most everything.
        self.memcache = CMemcache(
            self.memcaches,
            min_compress_len=50 * 1024,
            num_clients=num_mc_clients,
        )

        # a smaller pool of caches used only for distributed locks.
        # TODO: move this to ZooKeeper
        self.lock_cache = CMemcache(self.lockcaches,
                                    num_clients=num_mc_clients)
        self.make_lock = make_lock_factory(self.lock_cache, self.stats)

        # memcaches used in front of the permacache CF in cassandra.
        # XXX: this is a legacy thing; permacache was made when C* didn't have
        # a row cache.
        if self.permacache_memcaches:
            permacache_memcaches = CMemcache(self.permacache_memcaches,
                                             min_compress_len=50 * 1024,
                                             num_clients=num_mc_clients)
        else:
            permacache_memcaches = None

        # the stalecache is a memcached local to the current app server used
        # for data that's frequently fetched but doesn't need to be fresh.
        if self.stalecaches:
            stalecaches = CMemcache(self.stalecaches,
                                    num_clients=num_mc_clients)
        else:
            stalecaches = None

        # rendercache holds rendered partial templates.
        rendercaches = CMemcache(
            self.rendercaches,
            noreply=True,
            no_block=True,
            num_clients=num_mc_clients,
            min_compress_len=1400,
        )

        # pagecaches hold fully rendered pages
        pagecaches = CMemcache(
            self.pagecaches,
            noreply=True,
            no_block=True,
            num_clients=num_mc_clients,
            min_compress_len=1400,
        )

        self.startup_timer.intermediate("memcache")

        ################# CASSANDRA
        keyspace = "reddit"
        self.cassandra_pools = {
            "main":
                StatsCollectingConnectionPool(
                    keyspace,
                    stats=self.stats,
                    logging_name="main",
                    server_list=self.cassandra_seeds,
                    pool_size=self.cassandra_pool_size,
                    timeout=4,
                    max_retries=3,
                    prefill=False
                ),
        }

        permacache_cf = CassandraCache(
            'permacache',
            self.cassandra_pools[self.cassandra_default_pool],
            read_consistency_level=self.cassandra_rcl,
            write_consistency_level=self.cassandra_wcl
        )

        self.startup_timer.intermediate("cassandra")

        ################# POSTGRES
        event.listens_for(engine.Engine, 'before_cursor_execute')(
            self.stats.pg_before_cursor_execute)
        event.listens_for(engine.Engine, 'after_cursor_execute')(
            self.stats.pg_after_cursor_execute)

        self.dbm = self.load_db_params()
        self.startup_timer.intermediate("postgres")

        ################# CHAINS
        # initialize caches. Any cache-chains built here must be added
        # to cache_chains (closed around by reset_caches) so that they
        # can properly reset their local components
        cache_chains = {}
        localcache_cls = (SelfEmptyingCache if self.running_as_script
                          else LocalCache)

        if stalecaches:
            self.cache = StaleCacheChain(
                localcache_cls(),
                stalecaches,
                self.memcache,
            )
        else:
            self.cache = MemcacheChain((localcache_cls(), self.memcache))
        cache_chains.update(cache=self.cache)

        self.rendercache = MemcacheChain((
            localcache_cls(),
            rendercaches,
        ))
        cache_chains.update(rendercache=self.rendercache)

        self.pagecache = MemcacheChain((
            localcache_cls(),
            pagecaches,
        ))
        cache_chains.update(pagecache=self.pagecache)

        # the thing_cache is used in tdb_cassandra.
        self.thing_cache = CacheChain((localcache_cls(),))
        cache_chains.update(thing_cache=self.thing_cache)

        self.permacache = CassandraCacheChain(
            localcache_cls(),
            permacache_cf,
            memcache=permacache_memcaches,
            lock_factory=self.make_lock,
        )
        cache_chains.update(permacache=self.permacache)

        # hardcache is used for various things that tend to expire
        # TODO: replace hardcache w/ cassandra stuff
        self.hardcache = HardcacheChain(
            (localcache_cls(), self.memcache, HardCache(self)),
            cache_negative_results=True,
        )
        cache_chains.update(hardcache=self.hardcache)

        # I know this sucks, but we need non-request-threads to be
        # able to reset the caches, so we need them be able to close
        # around 'cache_chains' without being able to call getattr on
        # 'g'
        def reset_caches():
            for name, chain in cache_chains.iteritems():
                chain.reset()
                chain.stats = CacheStats(self.stats, name)
        self.cache_chains = cache_chains

        self.reset_caches = reset_caches
        self.reset_caches()

        self.startup_timer.intermediate("cache_chains")

        # try to set the source control revision numbers
        self.versions = {}
        r2_root = os.path.dirname(os.path.dirname(self.paths["root"]))
        r2_gitdir = os.path.join(r2_root, ".git")
        self.short_version = self.record_repo_version("r2", r2_gitdir)

        if I18N_PATH:
            i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git")
            self.record_repo_version("i18n", i18n_git_path)

        self.startup_timer.intermediate("revisions")

    def setup_complete(self):
        self.startup_timer.stop()
        self.stats.flush()

        if self.log_start:
            self.log.error(
                "%s:%s started %s at %s (took %.02fs)",
                self.reddit_host,
                self.reddit_pid,
                self.short_version,
                datetime.now().strftime("%H:%M:%S"),
                self.startup_timer.elapsed_seconds()
            )

    def record_repo_version(self, repo_name, git_dir):
        """Get the currently checked out git revision for a given repository,
        record it in g.versions, and return the short version of the hash."""
        try:
            subprocess.check_output
        except AttributeError:
            # python 2.6 compat
            pass
        else:
            try:
                revision = subprocess.check_output(["git",
                                                    "--git-dir", git_dir,
                                                    "rev-parse", "HEAD"])
            except subprocess.CalledProcessError, e:
                self.log.warning("Unable to fetch git revision: %r", e)
            else:
Beispiel #16
0
 def test_float(self):
     self.assertEquals(3.0, ConfigValue.float('3'))
     self.assertEquals(-3.0, ConfigValue.float('-3'))
     with self.assertRaises(ValueError):
         ConfigValue.float('asdf')
Beispiel #17
0
 def test_bool(self):
     self.assertEquals(True, ConfigValue.bool('TrUe'))
     self.assertEquals(False, ConfigValue.bool('fAlSe'))
     with self.assertRaises(ValueError):
         ConfigValue.bool('asdf')
Beispiel #18
0
 def test_int(self):
     self.assertEquals(3, ConfigValue.int('3'))
     self.assertEquals(-3, ConfigValue.int('-3'))
     with self.assertRaises(ValueError):
         ConfigValue.int('asdf')
Beispiel #19
0
 def test_str(self):
     self.assertEquals('x', ConfigValue.str('x'))
Beispiel #20
0
def load_environment(global_conf={}, app_conf={}, setup_globals=True):
    # Setup our paths
    root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    paths = {
        'root': root_path,
        'controllers': os.path.join(root_path, 'controllers'),
        'templates': [os.path.join(root_path, 'templates')],
    }

    if ConfigValue.bool(global_conf.get('uncompressedJS')):
        paths['static_files'] = os.path.join(root_path, 'public')
    else:
        paths['static_files'] = os.path.join(os.path.dirname(root_path),
                                             'build/public')

    config.init_app(global_conf,
                    app_conf,
                    package='r2',
                    template_engine='mako',
                    paths=paths)

    g = config['pylons.g'] = Globals(global_conf, app_conf, paths)
    if setup_globals:
        g.setup()
        g.plugins.declare_queues(g.queues)
        r2.config.cache = g.cache
    g.plugins.load_plugins()
    config['r2.plugins'] = g.plugins
    g.startup_timer.intermediate("plugins")

    config['pylons.h'] = r2.lib.helpers
    config['routes.map'] = routing.make_map()

    #override the default response options
    config['pylons.response_options']['headers'] = {}

    # The following template options are passed to your template engines
    tmpl_options = config['buffet.template_options']
    tmpl_options['mako.filesystem_checks'] = getattr(g, 'reload_templates',
                                                     False)
    tmpl_options['mako.default_filters'] = ["mako_websafe"]
    tmpl_options['mako.imports'] = \
                                 ["from r2.lib.filters import websafe, unsafe, mako_websafe",
                                  "from pylons import c, g, request",
                                  "from pylons.i18n import _, ungettext"]

    # when mako loads a previously compiled template file from its cache, it
    # doesn't check that the original template path matches the current path.
    # in the event that a new plugin defines a template overriding a reddit
    # template, unless the mtime newer, mako doesn't update the compiled
    # template. as a workaround, this makes mako store compiled templates with
    # the original path in the filename, forcing it to update with the path.
    def mako_module_path(filename, uri):
        module_directory = tmpl_options['mako.module_directory']
        filename = filename.lstrip('/').replace('/', '-')
        path = os.path.join(module_directory, filename + ".py")
        return os.path.abspath(path)

    tmpl_options['mako.modulename_callable'] = mako_module_path

    if setup_globals:
        g.setup_complete()
Beispiel #21
0
class Globals(object):
    spec = {
        ConfigValue.int: [
            'db_pool_size',
            'db_pool_overflow_size',
            'commentpane_cache_time',
            'num_mc_clients',
            'MAX_CAMPAIGNS_PER_LINK',
            'MIN_DOWN_LINK',
            'MIN_UP_KARMA',
            'MIN_DOWN_KARMA',
            'MIN_RATE_LIMIT_KARMA',
            'MIN_RATE_LIMIT_COMMENT_KARMA',
            'HOT_PAGE_AGE',
            'ADMIN_COOKIE_TTL',
            'ADMIN_COOKIE_MAX_IDLE',
            'OTP_COOKIE_TTL',
            'hsts_max_age',
            'num_comments',
            'max_comments',
            'max_comments_gold',
            'max_comment_parent_walk',
            'max_sr_images',
            'num_serendipity',
            'comment_visits_period',
            'butler_max_mentions',
            'min_membership_create_community',
            'bcrypt_work_factor',
            'cassandra_pool_size',
            'sr_banned_quota',
            'sr_muted_quota',
            'sr_wikibanned_quota',
            'sr_wikicontributor_quota',
            'sr_moderator_invite_quota',
            'sr_contributor_quota',
            'sr_quota_time',
            'sr_invite_limit',
            'thumbnail_hidpi_scaling',
            'wiki_keep_recent_days',
            'wiki_max_page_length_bytes',
            'wiki_max_config_stylesheet_length_bytes',
            'wiki_max_page_name_length',
            'wiki_max_page_separators',
            'RL_RESET_MINUTES',
            'RL_OAUTH_RESET_MINUTES',
            'comment_karma_display_floor',
            'link_karma_display_floor',
            'mobile_auth_gild_time',
            'default_total_budget_pennies',
            'min_total_budget_pennies',
            'max_total_budget_pennies',
            'default_bid_pennies',
            'min_bid_pennies',
            'max_bid_pennies',
            'frequency_cap_min',
            'frequency_cap_default',
            'eu_cookie_max_attempts',
            'captcha_sol_length',
            'captcha_font_size',
            'banner_variants',
            'precompute_limit',
            'precompute_limit_hot',
            'hot_max_links_per_subreddit',
            'fetch_title_max_download_kb',
        ],
        ConfigValue.float: [
            'statsd_sample_rate',
            'querycache_prune_chance',
            'RL_AVG_REQ_PER_SEC',
            'RL_OAUTH_AVG_REQ_PER_SEC',
            'RL_LOGIN_AVG_PER_SEC',
            'RL_LOGIN_IP_AVG_PER_SEC',
            'RL_SHARE_AVG_PER_SEC',
            'tracing_sample_rate',
            'hot_period_seconds',
        ],
        ConfigValue.bool: [
            'debug',
            'log_start',
            'sqlprinting',
            'template_debug',
            'reload_templates',
            'uncompressedJS',
            'css_killswitch',
            'db_create_tables',
            'disallow_db_writes',
            'disable_ratelimit',
            'amqp_logging',
            'read_only_mode',
            'disable_wiki',
            'heavy_load_mode',
            'disable_captcha',
            'disable_ads',
            'disable_require_admin_otp',
            'trust_local_proxies',
            'shard_commentstree_queues',
            'shard_author_query_queues',
            'shard_subreddit_query_queues',
            'shard_domain_query_queues',
            'authnet_validate',
            'ENFORCE_RATELIMIT',
            'RL_SITEWIDE_ENABLED',
            'RL_OAUTH_SITEWIDE_ENABLED',
            'enable_loggedout_experiments',
            'disable_geoip_service',
            'disable_remote_fetch',
            'disable_newsletter',
            'remote_fetch_proxy_enabled',
            'gold_gilding_enabled',
            'sub_muting_enabled',
            'allsr_prefilter_allow_top',
            'site_index_user_configurable',
            'allow_top_affects_new',
            'allow_top_false_subreddits_tab',
            'block_user_show_comments',
            'block_user_show_links',
            'chat_guest_chat_enabled',
            'chat_all',
            'chat_front',
        ],
        ConfigValue.tuple: [
            'plugins',
            'stalecaches',
            'lockcaches',
            'permacache_memcaches',
            'cassandra_seeds',
            'automatic_reddits',
            'hardcache_categories',
            'case_sensitive_domains',
            'known_image_domains',
            'reserved_subdomains',
            'offsite_subdomains',
            'TRAFFIC_LOG_HOSTS',
            'exempt_login_user_agents',
            'autoexpand_media_types',
            'media_preview_domain_whitelist',
            'multi_icons',
            'hide_subscribers_srs',
            'mcrouter_addr',
            'permacache_domain_priority',
        ],
        ConfigValue.tuple_of(ConfigValue.int): [
            'thumbnail_size',
            'preview_image_max_size',
            'preview_image_min_size',
            'mobile_ad_image_size',
        ],
        ConfigValue.tuple_of(ConfigValue.float): [
            'ios_versions',
            'android_versions',
        ],
        ConfigValue.dict(ConfigValue.str, ConfigValue.int): [
            'user_agent_ratelimit_regexes',
        ],
        ConfigValue.str: [
            'wiki_page_registration_info',
            'wiki_page_privacy_policy',
            'wiki_page_user_agreement',
            'wiki_page_gold_bottlecaps',
            'fraud_email',
            'feedback_email',
            'share_reply',
            'community_email',
            'smtp_server',
            'events_collector_url',
            'events_collector_test_url',
            'search_provider',
            'remote_fetch_proxy_url',
            'brander_community',
            'brander_community_plural',
            'imgur_client_id',
        ],
        ConfigValue.choice(ONE=CL_ONE, QUORUM=CL_QUORUM): [
            'cassandra_rcl',
            'cassandra_wcl',
        ],
        ConfigValue.choice(zookeeper="zookeeper", config="config"): [
            "liveconfig_source",
            "secrets_source",
        ],
        ConfigValue.timeinterval: [
            'ARCHIVE_AGE',
            "vote_queue_grace_period",
        ],
        config_gold_price: [
            'gold_month_price',
            'gold_year_price',
            'cpm_selfserve',
            'cpm_selfserve_geotarget_metro',
            'cpm_selfserve_geotarget_country',
            'cpm_selfserve_collection',
        ],
        ConfigValue.baseplate(
            baseplate_config.Optional(baseplate_config.Endpoint)): [
            "activity_endpoint",
            "tracing_endpoint",
        ],
        ConfigValue.dict(ConfigValue.str, ConfigValue.str): [
            'emr_traffic_tags',
        ],
    }

    live_config_spec = {
        ConfigValue.bool: [
            'frontend_logging',
            'mobile_gild_first_login',
            'precomputed_comment_suggested_sort',
        ],
        ConfigValue.int: [
            'captcha_exempt_comment_karma',
            'captcha_exempt_link_karma',
            'create_sr_account_age_days',
            'create_sr_comment_karma',
            'create_sr_link_karma',
            'cflag_min_votes',
            'ads_popularity_threshold',
            'precomputed_comment_sort_min_comments',
            'comment_vote_update_threshold',
            'comment_vote_update_period',
            'create_sr_ratelimit_once_per_days',
        ],
        ConfigValue.float: [
            'cflag_lower_bound',
            'cflag_upper_bound',
            'spotlight_interest_sub_p',
            'spotlight_interest_nosub_p',
            'gold_revenue_goal',
            'invalid_key_sample_rate',
            'events_collector_vote_sample_rate',
            'events_collector_poison_sample_rate',
            'events_collector_mod_sample_rate',
            'events_collector_quarantine_sample_rate',
            'events_collector_modmail_sample_rate',
            'events_collector_report_sample_rate',
            'events_collector_submit_sample_rate',
            'events_collector_comment_sample_rate',
            'events_collector_use_gzip_chance',
            'https_cert_testing_probability',
        ],
        ConfigValue.tuple: [
            'fastlane_links',
            'listing_chooser_sample_multis',
            'discovery_srs',
            'proxy_gilding_accounts',
            'mweb_blacklist_expressions',
            'global_loid_experiments',
            'precomputed_comment_sorts',
            'mailgun_domains',
        ],
        ConfigValue.str: [
            'listing_chooser_gold_multi',
            'listing_chooser_explore_sr',
        ],
        ConfigValue.messages: [
            'welcomebar_messages',
            'sidebar_message',
            'gold_sidebar_message',
        ],
        ConfigValue.dict(ConfigValue.str, ConfigValue.int): [
            'ticket_groups',
            'ticket_user_fields',
        ],
        ConfigValue.dict(ConfigValue.str, ConfigValue.float): [
            'pennies_per_server_second',
        ],
        ConfigValue.dict(ConfigValue.str, ConfigValue.str): [
            'employee_approved_clients',
            'mobile_auth_allowed_clients',
            'modmail_forwarding_email',
            'modmail_account_map',
        ],
        ConfigValue.dict(ConfigValue.str, ConfigValue.choice(**PERMISSIONS)): [
            'employees',
        ],
    }

    def __init__(self, config, global_conf, app_conf, paths, **extra):
        """
        Globals acts as a container for objects available throughout
        the life of the application.

        One instance of Globals is created by Pylons during
        application initialization and is available during requests
        via the 'g' variable.

        ``config``
            The PylonsConfig object passed in from ``config/environment.py``

        ``global_conf``
            The same variable used throughout ``config/middleware.py``
            namely, the variables from the ``[DEFAULT]`` section of the
            configuration file.

        ``app_conf``
            The same ``kw`` dictionary used throughout
            ``config/middleware.py`` namely, the variables from the
            section in the config file for your application.

        ``extra``
            The configuration returned from ``load_config`` in 
            ``config/middleware.py`` which may be of use in the setup of
            your global variables.

        """

        global_conf.setdefault("debug", False)

        # reloading site ensures that we have a fresh sys.path to build our
        # working set off of. this means that forked worker processes won't get
        # the sys.path that was current when the master process was spawned
        # meaning that new plugins will be picked up on regular app reload
        # rather than having to restart the master process as well.
        reload(site)
        self.pkg_resources_working_set = pkg_resources.WorkingSet()

        self.config = ConfigValueParser(global_conf)
        self.config.add_spec(self.spec)
        self.plugins = PluginLoader(self.pkg_resources_working_set,
                                    self.config.get("plugins", []))

        self.stats = Stats(self.config.get('statsd_addr'),
                           self.config.get('statsd_sample_rate'))
        self.startup_timer = self.stats.get_timer("app_startup")
        self.startup_timer.start()

        self.baseplate = Baseplate()
        self.baseplate.configure_logging()
        self.baseplate.register(R2BaseplateObserver())
        self.baseplate.configure_tracing(
            "r2",
            tracing_endpoint=self.config.get("tracing_endpoint"),
            sample_rate=self.config.get("tracing_sample_rate"),
        )

        self.paths = paths

        self.running_as_script = global_conf.get('running_as_script', False)

        # turn on for language support
        self.lang = getattr(self, 'site_lang', 'en')
        self.languages, self.lang_name = get_active_langs(
            config, default_lang=self.lang)

        all_languages = self.lang_name.keys()
        all_languages.sort()
        self.all_languages = all_languages

        # set default time zone if one is not set
        tz = global_conf.get('timezone', 'UTC')
        self.tz = pytz.timezone(tz)

        dtz = global_conf.get('display_timezone', tz)
        self.display_tz = pytz.timezone(dtz)

        self.startup_timer.intermediate("init")

    def __getattr__(self, name):
        if not name.startswith('_') and name in self.config:
            return self.config[name]
        else:
            raise AttributeError("g has no attr %r" % name)

    def setup(self):
        self.env = ''
        if (
                # handle direct invocation of "nosetests"
                "test" in sys.argv[0] or
                # handle "setup.py test" and all permutations thereof.
                "setup.py" in sys.argv[0] and "test" in sys.argv[1:]):
            self.env = "unit_test"

        self.queues = queues.declare_queues(self)

        self.extension_subdomains = dict(
            simple="mobile",
            i="compact",
            api="api",
            rss="rss",
            xml="xml",
            json="json",
        )
        # SaidIt CUSTOM
        self.extension_subdomains[
            self.config['extension_subdomain_mobile_v2']] = self.config[
                'extension_subdomain_mobile_v2_render_style']

        ################# PROVIDERS
        self.auth_provider = select_provider(
            self.config,
            self.pkg_resources_working_set,
            "r2.provider.auth",
            self.authentication_provider,
        )
        self.media_provider = select_provider(
            self.config,
            self.pkg_resources_working_set,
            "r2.provider.media",
            self.media_provider,
        )
        self.cdn_provider = select_provider(
            self.config,
            self.pkg_resources_working_set,
            "r2.provider.cdn",
            self.cdn_provider,
        )
        self.ticket_provider = select_provider(
            self.config,
            self.pkg_resources_working_set,
            "r2.provider.support",
            # TODO: fix this later, it refuses to pick up
            # g.config['ticket_provider'] value, so hardcoding for now.
            # really, the next uncommented line should be:
            #self.ticket_provider,
            # instead of:
            "zendesk",
        )
        self.image_resizing_provider = select_provider(
            self.config,
            self.pkg_resources_working_set,
            "r2.provider.image_resizing",
            self.image_resizing_provider,
        )
        self.email_provider = select_provider(
            self.config,
            self.pkg_resources_working_set,
            "r2.provider.email",
            self.email_provider,
        )
        self.startup_timer.intermediate("providers")

        ################# CONFIGURATION
        # AMQP is required
        if not self.amqp_host:
            raise ValueError("amqp_host not set in the .ini")

        if not self.cassandra_seeds:
            raise ValueError("cassandra_seeds not set in the .ini")

        # heavy load mode is read only mode with a different infobar
        if self.heavy_load_mode:
            self.read_only_mode = True

        origin_prefix = self.domain_prefix + "." if self.domain_prefix else ""
        self.origin = self.default_scheme + "://" + origin_prefix + self.domain

        self.trusted_domains = set([self.domain])
        if self.https_endpoint:
            https_url = urlparse(self.https_endpoint)
            self.trusted_domains.add(https_url.hostname)

        # load the unique hashed names of files under static
        static_files = os.path.join(self.paths.get('static_files'), 'static')
        names_file_path = os.path.join(static_files, 'names.json')
        if os.path.exists(names_file_path):
            with open(names_file_path) as handle:
                self.static_names = json.load(handle)
        else:
            self.static_names = {}

        # make python warnings go through the logging system
        logging.captureWarnings(capture=True)

        log = logging.getLogger('reddit')

        # when we're a script (paster run) just set up super simple logging
        if self.running_as_script:
            log.setLevel(logging.INFO)
            log.addHandler(logging.StreamHandler())

        # if in debug mode, override the logging level to DEBUG
        if self.debug:
            log.setLevel(logging.DEBUG)

        # attempt to figure out which pool we're in and add that to the
        # LogRecords.
        try:
            with open("/etc/ec2_asg", "r") as f:
                pool = f.read().strip()
            # clean up the pool name since we're putting stuff after "-"
            pool = pool.partition("-")[0]
        except IOError:
            pool = "reddit-app"
        self.log = logging.LoggerAdapter(log, {"pool": pool})

        # set locations
        locations = pkg_resources.resource_stream(__name__,
                                                  "../data/locations.json")
        self.locations = json.loads(locations.read())

        if not self.media_domain:
            self.media_domain = self.domain
        if self.media_domain == self.domain:
            print >> sys.stderr, (
                "Warning: g.media_domain == g.domain. " +
                "This may give untrusted content access to user cookies")
        if self.oauth_domain == self.domain:
            print >> sys.stderr, ("Warning: g.oauth_domain == g.domain. "
                                  "CORS requests to g.domain will be allowed")

        for arg in sys.argv:
            tokens = arg.split("=")
            if len(tokens) == 2:
                k, v = tokens
                self.log.debug("Overriding g.%s to %s" % (k, v))
                setattr(self, k, v)

        self.reddit_host = socket.gethostname()
        self.reddit_pid = os.getpid()

        if hasattr(signal, 'SIGUSR1'):
            # not all platforms have user signals
            signal.signal(signal.SIGUSR1, thread_dump)

        locale.setlocale(locale.LC_ALL, self.locale)

        # Pre-calculate ratelimit values
        self.RL_RESET_SECONDS = self.config["RL_RESET_MINUTES"] * 60
        self.RL_MAX_REQS = int(self.config["RL_AVG_REQ_PER_SEC"] *
                               self.RL_RESET_SECONDS)

        self.RL_OAUTH_RESET_SECONDS = self.config["RL_OAUTH_RESET_MINUTES"] * 60
        self.RL_OAUTH_MAX_REQS = int(self.config["RL_OAUTH_AVG_REQ_PER_SEC"] *
                                     self.RL_OAUTH_RESET_SECONDS)

        self.RL_LOGIN_MAX_REQS = int(self.config["RL_LOGIN_AVG_PER_SEC"] *
                                     self.RL_RESET_SECONDS)
        self.RL_LOGIN_IP_MAX_REQS = int(
            self.config["RL_LOGIN_IP_AVG_PER_SEC"] * self.RL_RESET_SECONDS)
        self.RL_SHARE_MAX_REQS = int(self.config["RL_SHARE_AVG_PER_SEC"] *
                                     self.RL_RESET_SECONDS)

        # Compile ratelimit regexs
        user_agent_ratelimit_regexes = {}
        for agent_re, limit in self.user_agent_ratelimit_regexes.iteritems():
            user_agent_ratelimit_regexes[re.compile(agent_re)] = limit
        self.user_agent_ratelimit_regexes = user_agent_ratelimit_regexes

        self.startup_timer.intermediate("configuration")

        ################# ZOOKEEPER
        zk_hosts = self.config["zookeeper_connection_string"]
        zk_username = self.config["zookeeper_username"]
        zk_password = self.config["zookeeper_password"]
        self.zookeeper = connect_to_zookeeper(zk_hosts,
                                              (zk_username, zk_password))

        self.throttles = IPNetworkLiveList(
            self.zookeeper,
            root="/throttles",
            reduced_data_node="/throttles_reduced",
        )

        parser = ConfigParser.RawConfigParser()
        parser.optionxform = str
        parser.read([self.config["__file__"]])

        if self.config["liveconfig_source"] == "zookeeper":
            self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE)
        else:
            self.live_config = extract_live_config(parser, self.plugins)

        if self.config["secrets_source"] == "zookeeper":
            self.secrets = fetch_secrets(self.zookeeper)
        else:
            self.secrets = extract_secrets(parser)

        ################# PRIVILEGED USERS
        self.admins = PermissionFilteredEmployeeList(self.live_config,
                                                     type="admin")
        self.sponsors = PermissionFilteredEmployeeList(self.live_config,
                                                       type="sponsor")
        self.employees = PermissionFilteredEmployeeList(self.live_config,
                                                        type="employee")

        # Store which OAuth clients employees may use, the keys are just for
        # readability.
        self.employee_approved_clients = \
            self.live_config["employee_approved_clients"].values()

        self.mobile_auth_allowed_clients = self.live_config[
            "mobile_auth_allowed_clients"].values()

        self.startup_timer.intermediate("zookeeper")

        ################# MEMCACHE
        num_mc_clients = self.num_mc_clients

        # a smaller pool of caches used only for distributed locks.
        self.lock_cache = CMemcache(
            "lock",
            self.lockcaches,
            num_clients=num_mc_clients,
        )
        self.make_lock = make_lock_factory(self.lock_cache, self.stats)

        # memcaches used in front of the permacache CF in cassandra.
        # XXX: this is a legacy thing; permacache was made when C* didn't have
        # a row cache.
        permacache_memcaches = CMemcache(
            "perma",
            self.permacache_memcaches,
            min_compress_len=1400,
            num_clients=num_mc_clients,
        )

        # the stalecache is a memcached local to the current app server used
        # for data that's frequently fetched but doesn't need to be fresh.
        if self.stalecaches:
            stalecaches = CMemcache(
                "stale",
                self.stalecaches,
                num_clients=num_mc_clients,
            )
        else:
            stalecaches = None

        self.startup_timer.intermediate("memcache")

        ################# MCROUTER
        self.mcrouter = Mcrouter(
            "mcrouter",
            self.mcrouter_addr,
            min_compress_len=1400,
            num_clients=num_mc_clients,
        )

        ################# THRIFT-BASED SERVICES
        activity_endpoint = self.config.get("activity_endpoint")
        if activity_endpoint:
            # make ActivityInfo objects rendercache-key friendly
            # TODO: figure out a more general solution for this if
            # we need to do this for other thrift-generated objects
            ActivityInfo.cache_key = lambda self, style: repr(self)

            activity_pool = ThriftConnectionPool(activity_endpoint,
                                                 timeout=0.1)
            self.baseplate.add_to_context(
                "activity_service",
                ThriftContextFactory(activity_pool, ActivityService.Client))

        self.startup_timer.intermediate("thrift")

        ################# CASSANDRA
        keyspace = "reddit"
        self.cassandra_pools = {
            "main":
            StatsCollectingConnectionPool(keyspace,
                                          stats=self.stats,
                                          logging_name="main",
                                          server_list=self.cassandra_seeds,
                                          pool_size=self.cassandra_pool_size,
                                          timeout=4,
                                          max_retries=3,
                                          prefill=False),
        }

        permacache_cf = Permacache._setup_column_family(
            'permacache',
            self.cassandra_pools[self.cassandra_default_pool],
        )

        self.startup_timer.intermediate("cassandra")

        ################# POSTGRES
        self.dbm = self.load_db_params()
        self.startup_timer.intermediate("postgres")

        ################# CHAINS
        # initialize caches. Any cache-chains built here must be added
        # to cache_chains (closed around by reset_caches) so that they
        # can properly reset their local components
        cache_chains = {}
        localcache_cls = (SelfEmptyingCache
                          if self.running_as_script else LocalCache)

        if stalecaches:
            self.gencache = StaleCacheChain(
                localcache_cls(),
                stalecaches,
                self.mcrouter,
            )
        else:
            self.gencache = CacheChain((localcache_cls(), self.mcrouter))
        cache_chains.update(gencache=self.gencache)

        if stalecaches:
            self.thingcache = StaleCacheChain(
                localcache_cls(),
                stalecaches,
                self.mcrouter,
            )
        else:
            self.thingcache = CacheChain((localcache_cls(), self.mcrouter))
        cache_chains.update(thingcache=self.thingcache)

        if stalecaches:
            self.memoizecache = StaleCacheChain(
                localcache_cls(),
                stalecaches,
                self.mcrouter,
            )
        else:
            self.memoizecache = MemcacheChain(
                (localcache_cls(), self.mcrouter))
        cache_chains.update(memoizecache=self.memoizecache)

        if stalecaches:
            self.srmembercache = StaleCacheChain(
                localcache_cls(),
                stalecaches,
                self.mcrouter,
            )
        else:
            self.srmembercache = MemcacheChain(
                (localcache_cls(), self.mcrouter))
        cache_chains.update(srmembercache=self.srmembercache)

        if stalecaches:
            self.relcache = StaleCacheChain(
                localcache_cls(),
                stalecaches,
                self.mcrouter,
            )
        else:
            self.relcache = MemcacheChain((localcache_cls(), self.mcrouter))
        cache_chains.update(relcache=self.relcache)

        self.ratelimitcache = MemcacheChain((localcache_cls(), self.mcrouter))
        cache_chains.update(ratelimitcache=self.ratelimitcache)

        # rendercache holds rendered partial templates.
        self.rendercache = MemcacheChain((
            localcache_cls(),
            self.mcrouter,
        ))
        cache_chains.update(rendercache=self.rendercache)

        # commentpanecaches hold fully rendered comment panes
        self.commentpanecache = MemcacheChain((
            localcache_cls(),
            self.mcrouter,
        ))
        cache_chains.update(commentpanecache=self.commentpanecache)

        # cassandra_local_cache is used for request-local caching in tdb_cassandra
        self.cassandra_local_cache = localcache_cls()
        cache_chains.update(cassandra_local_cache=self.cassandra_local_cache)

        if stalecaches:
            permacache_cache = StaleCacheChain(
                localcache_cls(),
                stalecaches,
                permacache_memcaches,
            )
        else:
            permacache_cache = CacheChain(
                (localcache_cls(), permacache_memcaches), )
        cache_chains.update(permacache=permacache_cache)

        self.permacache = Permacache(
            permacache_cache,
            permacache_cf,
            lock_factory=self.make_lock,
        )

        # hardcache is used for various things that tend to expire
        # TODO: replace hardcache w/ cassandra stuff
        self.hardcache = HardcacheChain(
            (localcache_cls(), HardCache(self)),
            cache_negative_results=True,
        )
        cache_chains.update(hardcache=self.hardcache)

        # I know this sucks, but we need non-request-threads to be
        # able to reset the caches, so we need them be able to close
        # around 'cache_chains' without being able to call getattr on
        # 'g'
        def reset_caches():
            for name, chain in cache_chains.iteritems():
                if isinstance(chain, TransitionalCache):
                    chain = chain.read_chain

                chain.reset()
                if isinstance(chain, LocalCache):
                    continue
                elif isinstance(chain, StaleCacheChain):
                    chain.stats = StaleCacheStats(self.stats, name)
                else:
                    chain.stats = CacheStats(self.stats, name)

        self.cache_chains = cache_chains

        self.reset_caches = reset_caches
        self.reset_caches()

        self.startup_timer.intermediate("cache_chains")

        # try to set the source control revision numbers
        self.versions = {}
        r2_root = os.path.dirname(os.path.dirname(self.paths["root"]))
        r2_gitdir = os.path.join(r2_root, ".git")
        self.short_version = self.record_repo_version("r2", r2_gitdir)

        if I18N_PATH:
            i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git")
            self.record_repo_version("i18n", i18n_git_path)

        # Initialize the amqp module globals, start the worker, etc.
        r2.lib.amqp.initialize(self)

        self.events = EventQueue()

        self.startup_timer.intermediate("revisions")

    def setup_complete(self):
        self.startup_timer.stop()
        self.stats.flush()

        if self.log_start:
            self.log.error("%s:%s started %s at %s (took %.02fs)",
                           self.reddit_host, self.reddit_pid,
                           self.short_version,
                           datetime.now().strftime("%H:%M:%S"),
                           self.startup_timer.elapsed_seconds())

        if einhorn.is_worker():
            einhorn.ack_startup()

    def record_repo_version(self, repo_name, git_dir):
        """Get the currently checked out git revision for a given repository,
        record it in g.versions, and return the short version of the hash."""
        try:
            subprocess.check_output
        except AttributeError:
            # python 2.6 compat
            pass
        else:
            try:
                revision = subprocess.check_output(
                    ["git", "--git-dir", git_dir, "rev-parse", "HEAD"])
            except subprocess.CalledProcessError, e:
                self.log.warning("Unable to fetch git revision: %r", e)
            else:
Beispiel #22
0
 def test_set(self):
     self.assertEquals(set([]), ConfigValue.set(''))
     self.assertEquals(set(['a', 'b']), ConfigValue.set('a, b'))
Beispiel #23
0
def load_environment(global_conf={}, app_conf={}, setup_globals=True):
    # Setup our paths
    root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    paths = {'root': root_path,
             'controllers': os.path.join(root_path, 'controllers'),
             'templates': [os.path.join(root_path, 'templates')],
             }

    if ConfigValue.bool(global_conf.get('uncompressedJS')):
        paths['static_files'] = os.path.join(root_path, 'public')
    else:
        paths['static_files'] = os.path.join(os.path.dirname(root_path), 'build/public')

    config.init_app(global_conf, app_conf, package='r2',
                    template_engine='mako', paths=paths)

    # don't put action arguments onto c automatically
    config['pylons.c_attach_args'] = False
    # when accessing non-existent attributes on c, return "" instead of dying
    config['pylons.strict_c'] = False

    g = config['pylons.g'] = Globals(global_conf, app_conf, paths)
    if setup_globals:
        g.setup()
        g.plugins.declare_queues(g.queues)
    g.plugins.load_plugins()
    config['r2.plugins'] = g.plugins
    g.startup_timer.intermediate("plugins")

    config['pylons.h'] = r2.lib.helpers
    config['routes.map'] = routing.make_map()

    #override the default response options
    config['pylons.response_options']['headers'] = {}

    # when mako loads a previously compiled template file from its cache, it
    # doesn't check that the original template path matches the current path.
    # in the event that a new plugin defines a template overriding a reddit
    # template, unless the mtime newer, mako doesn't update the compiled
    # template. as a workaround, this makes mako store compiled templates with
    # the original path in the filename, forcing it to update with the path.
    if "cache_dir" in app_conf:
        module_directory = os.path.join(app_conf['cache_dir'], 'templates')

        def mako_module_path(filename, uri):
            filename = filename.lstrip('/').replace('/', '-')
            path = os.path.join(module_directory, filename + ".py")
            return os.path.abspath(path)
    else:
        # we're probably in "paster run standalone" mode. we'll just avoid
        # caching templates since we don't know where they should go.
        module_directory = mako_module_path = None

    # set up the templating system
    config["pylons.g"].mako_lookup = TemplateLookup(
        directories=paths["templates"],
        error_handler=handle_mako_error,
        module_directory=module_directory,
        input_encoding="utf-8",
        default_filters=["mako_websafe"],
        filesystem_checks=getattr(g, "reload_templates", False),
        imports=[
            "from r2.lib.filters import websafe, unsafe, mako_websafe",
            "from pylons import c, g, request",
            "from pylons.i18n import _, ungettext",
        ],
        modulename_callable=mako_module_path,
    )

    if setup_globals:
        g.setup_complete()
Beispiel #24
0
 def test_set_of(self):
     self.assertEquals(set([]), ConfigValue.set_of(str)(''))
     self.assertEquals(set(['a', 'b']), ConfigValue.set_of(str)('a, b, b'))
     self.assertEquals(set(['a', 'b']),
                       ConfigValue.set_of(str, delim=':')('b : a : b'))
Beispiel #25
0
 def test_str(self):
     self.assertEquals('x', ConfigValue.str('x'))
Beispiel #26
0
 def test_choice(self):
     self.assertEquals(1, ConfigValue.choice(alpha=1)('alpha'))
     self.assertEquals(2, ConfigValue.choice(alpha=1, beta=2)('beta'))
     with self.assertRaises(ValueError):
         ConfigValue.choice(alpha=1)('asdf')
Beispiel #27
0
 def test_float(self):
     self.assertEquals(3.0, ConfigValue.float('3'))
     self.assertEquals(-3.0, ConfigValue.float('-3'))
     with self.assertRaises(ValueError):
         ConfigValue.float('asdf')
Beispiel #28
0
    def load_db_params(self):
        self.databases = tuple(ConfigValue.to_iter(self.config.raw_data["databases"]))
        self.db_params = {}
        self.predefined_type_ids = {}
        if not self.databases:
            return

        if self.env == "unit_test":
            from mock import MagicMock

            return MagicMock()

        dbm = db_manager.db_manager()
        db_param_names = ("name", "db_host", "db_user", "db_pass", "db_port", "pool_size", "max_overflow")
        for db_name in self.databases:
            conf_params = ConfigValue.to_iter(self.config.raw_data[db_name + "_db"])
            params = dict(zip(db_param_names, conf_params))
            if params["db_user"] == "*":
                params["db_user"] = self.db_user
            if params["db_pass"] == "*":
                params["db_pass"] = self.db_pass
            if params["db_port"] == "*":
                params["db_port"] = self.db_port

            if params["pool_size"] == "*":
                params["pool_size"] = self.db_pool_size
            if params["max_overflow"] == "*":
                params["max_overflow"] = self.db_pool_overflow_size

            dbm.setup_db(db_name, g_override=self, **params)
            self.db_params[db_name] = params

        dbm.type_db = dbm.get_engine(self.config.raw_data["type_db"])
        dbm.relation_type_db = dbm.get_engine(self.config.raw_data["rel_type_db"])

        def split_flags(raw_params):
            params = []
            flags = {}

            for param in raw_params:
                if not param.startswith("!"):
                    params.append(param)
                else:
                    key, sep, value = param[1:].partition("=")
                    if sep:
                        flags[key] = value
                    else:
                        flags[key] = True

            return params, flags

        prefix = "db_table_"
        for k, v in self.config.raw_data.iteritems():
            if not k.startswith(prefix):
                continue

            params, table_flags = split_flags(ConfigValue.to_iter(v))
            name = k[len(prefix) :]
            kind = params[0]
            server_list = self.config.raw_data["db_servers_" + name]
            engines, flags = split_flags(ConfigValue.to_iter(server_list))

            typeid = table_flags.get("typeid")
            if typeid:
                self.predefined_type_ids[name] = int(typeid)

            if kind == "thing":
                dbm.add_thing(name, dbm.get_engines(engines), **flags)
            elif kind == "relation":
                dbm.add_relation(name, params[1], params[2], dbm.get_engines(engines), **flags)
        return dbm
Beispiel #29
0
 def test_tuple(self):
     self.assertEquals((), ConfigValue.tuple(''))
     self.assertEquals(('a', 'b'), ConfigValue.tuple('a, b'))
Beispiel #30
0
class Globals(object):
    spec = {
        ConfigValue.int: [
            'db_pool_size',
            'db_pool_overflow_size',
            'page_cache_time',
            'commentpane_cache_time',
            'num_mc_clients',
            'MAX_CAMPAIGNS_PER_LINK',
            'MIN_DOWN_LINK',
            'MIN_UP_KARMA',
            'MIN_DOWN_KARMA',
            'MIN_RATE_LIMIT_KARMA',
            'MIN_RATE_LIMIT_COMMENT_KARMA',
            'HOT_PAGE_AGE',
            'QUOTA_THRESHOLD',
            'ADMIN_COOKIE_TTL',
            'ADMIN_COOKIE_MAX_IDLE',
            'OTP_COOKIE_TTL',
            'hsts_max_age',
            'num_comments',
            'max_comments',
            'max_comments_gold',
            'max_comment_parent_walk',
            'max_sr_images',
            'num_serendipity',
            'sr_dropdown_threshold',
            'comment_visits_period',
            'butler_max_mentions',
            'min_membership_create_community',
            'bcrypt_work_factor',
            'cassandra_pool_size',
            'sr_banned_quota',
            'sr_wikibanned_quota',
            'sr_wikicontributor_quota',
            'sr_moderator_invite_quota',
            'sr_contributor_quota',
            'sr_quota_time',
            'sr_invite_limit',
            'thumbnail_hidpi_scaling',
            'wiki_keep_recent_days',
            'wiki_max_page_length_bytes',
            'wiki_max_page_name_length',
            'wiki_max_page_separators',
            'RL_RESET_MINUTES',
            'RL_OAUTH_RESET_MINUTES',
            'comment_karma_display_floor',
            'link_karma_display_floor',
        ],
        ConfigValue.float: [
            'default_promote_bid',
            'min_promote_bid',
            'max_promote_bid',
            'statsd_sample_rate',
            'querycache_prune_chance',
            'RL_AVG_REQ_PER_SEC',
            'RL_OAUTH_AVG_REQ_PER_SEC',
            'RL_LOGIN_AVG_PER_SEC',
        ],
        ConfigValue.bool: [
            'debug',
            'log_start',
            'sqlprinting',
            'template_debug',
            'reload_templates',
            'uncompressedJS',
            'css_killswitch',
            'db_create_tables',
            'disallow_db_writes',
            'disable_ratelimit',
            'amqp_logging',
            'read_only_mode',
            'disable_wiki',
            'heavy_load_mode',
            'disable_captcha',
            'disable_ads',
            'disable_require_admin_otp',
            'trust_local_proxies',
            'shard_link_vote_queues',
            'shard_commentstree_queues',
            'ENFORCE_RATELIMIT',
            'RL_SITEWIDE_ENABLED',
            'RL_OAUTH_SITEWIDE_ENABLED',
        ],
        ConfigValue.tuple: [
            'plugins',
            'stalecaches',
            'memcaches',
            'lockcaches',
            'permacache_memcaches',
            'rendercaches',
            'pagecaches',
            'memoizecaches',
            'srmembercaches',
            'relcaches',
            'ratelimitcaches',
            'cassandra_seeds',
            'automatic_reddits',
            'hardcache_categories',
            'case_sensitive_domains',
            'known_image_domains',
            'reserved_subdomains',
            'offsite_subdomains',
            'TRAFFIC_LOG_HOSTS',
            'exempt_login_user_agents',
            'timed_templates',
            'autoexpand_media_types',
            'multi_icons',
            'hide_subscribers_srs',
        ],
        ConfigValue.tuple_of(ConfigValue.int): [
            'thumbnail_size',
        ],
        ConfigValue.dict(ConfigValue.str, ConfigValue.int): [
            'agents',
        ],
        ConfigValue.str: [
            'wiki_page_registration_info',
            'wiki_page_privacy_policy',
            'wiki_page_user_agreement',
            'wiki_page_gold_bottlecaps',
            'fraud_email',
            'feedback_email',
            'share_reply',
            'nerds_email',
            'community_email',
            'smtp_server',
        ],
        ConfigValue.choice(ONE=CL_ONE, QUORUM=CL_QUORUM): [
            'cassandra_rcl',
            'cassandra_wcl',
        ],
        ConfigValue.timeinterval: [
            'ARCHIVE_AGE',
            "vote_queue_grace_period",
        ],
        config_gold_price: [
            'gold_month_price',
            'gold_year_price',
            'cpm_selfserve',
            'cpm_selfserve_geotarget_metro',
            'cpm_selfserve_collection',
        ],
    }

    live_config_spec = {
        ConfigValue.bool: [
            'frontend_logging',
        ],
        ConfigValue.int: [
            'captcha_exempt_comment_karma',
            'captcha_exempt_link_karma',
            'create_sr_account_age_days',
            'create_sr_comment_karma',
            'create_sr_link_karma',
            'cflag_min_votes',
        ],
        ConfigValue.float: [
            'cflag_lower_bound',
            'cflag_upper_bound',
            'spotlight_interest_sub_p',
            'spotlight_interest_nosub_p',
            'gold_revenue_goal',
            'invalid_key_sample_rate',
        ],
        ConfigValue.tuple: [
            'fastlane_links',
            'listing_chooser_sample_multis',
            'discovery_srs',
            'proxy_gilding_accounts',
        ],
        ConfigValue.str: [
            'listing_chooser_gold_multi',
            'listing_chooser_explore_sr',
        ],
        ConfigValue.dict(ConfigValue.int, ConfigValue.float): [
            'comment_tree_version_weights',
        ],
        ConfigValue.messages: [
            'welcomebar_messages',
            'sidebar_message',
            'gold_sidebar_message',
        ],
        ConfigValue.dict(ConfigValue.str, ConfigValue.float): [
            'pennies_per_server_second',
        ],
        ConfigValue.dict(ConfigValue.str, ConfigValue.choice(**PERMISSIONS)): [
            'employees',
        ],
    }

    def __init__(self, global_conf, app_conf, paths, **extra):
        """
        Globals acts as a container for objects available throughout
        the life of the application.

        One instance of Globals is created by Pylons during
        application initialization and is available during requests
        via the 'g' variable.

        ``global_conf``
            The same variable used throughout ``config/middleware.py``
            namely, the variables from the ``[DEFAULT]`` section of the
            configuration file.

        ``app_conf``
            The same ``kw`` dictionary used throughout
            ``config/middleware.py`` namely, the variables from the
            section in the config file for your application.

        ``extra``
            The configuration returned from ``load_config`` in 
            ``config/middleware.py`` which may be of use in the setup of
            your global variables.

        """

        global_conf.setdefault("debug", False)

        # reloading site ensures that we have a fresh sys.path to build our
        # working set off of. this means that forked worker processes won't get
        # the sys.path that was current when the master process was spawned
        # meaning that new plugins will be picked up on regular app reload
        # rather than having to restart the master process as well.
        reload(site)
        self.pkg_resources_working_set = pkg_resources.WorkingSet()

        self.config = ConfigValueParser(global_conf)
        self.config.add_spec(self.spec)
        self.plugins = PluginLoader(self.pkg_resources_working_set,
                                    self.config.get("plugins", []))

        self.stats = Stats(self.config.get('statsd_addr'),
                           self.config.get('statsd_sample_rate'))
        self.startup_timer = self.stats.get_timer("app_startup")
        self.startup_timer.start()

        self.paths = paths

        self.running_as_script = global_conf.get('running_as_script', False)

        # turn on for language support
        self.lang = getattr(self, 'site_lang', 'en')
        self.languages, self.lang_name = \
            get_active_langs(default_lang=self.lang)

        all_languages = self.lang_name.keys()
        all_languages.sort()
        self.all_languages = all_languages

        # set default time zone if one is not set
        tz = global_conf.get('timezone', 'UTC')
        self.tz = pytz.timezone(tz)

        dtz = global_conf.get('display_timezone', tz)
        self.display_tz = pytz.timezone(dtz)

        self.startup_timer.intermediate("init")

    def __getattr__(self, name):
        if not name.startswith('_') and name in self.config:
            return self.config[name]
        else:
            raise AttributeError

    def setup(self):
        self.queues = queues.declare_queues(self)

        self.extension_subdomains = dict(
            m="mobile",
            i="compact",
            api="api",
            rss="rss",
            xml="xml",
            json="json",
        )

        ################# PROVIDERS
        self.auth_provider = select_provider(
            self.config,
            self.pkg_resources_working_set,
            "r2.provider.auth",
            self.authentication_provider,
        )
        self.media_provider = select_provider(
            self.config,
            self.pkg_resources_working_set,
            "r2.provider.media",
            self.media_provider,
        )
        self.cdn_provider = select_provider(
            self.config,
            self.pkg_resources_working_set,
            "r2.provider.cdn",
            self.cdn_provider,
        )
        self.startup_timer.intermediate("providers")

        ################# CONFIGURATION
        # AMQP is required
        if not self.amqp_host:
            raise ValueError("amqp_host not set in the .ini")

        if not self.cassandra_seeds:
            raise ValueError("cassandra_seeds not set in the .ini")

        # heavy load mode is read only mode with a different infobar
        if self.heavy_load_mode:
            self.read_only_mode = True

        origin_prefix = self.domain_prefix + "." if self.domain_prefix else ""
        self.origin = "http://" + origin_prefix + self.domain

        self.trusted_domains = set([self.domain])
        if self.https_endpoint:
            https_url = urlparse(self.https_endpoint)
            self.trusted_domains.add(https_url.hostname)

        # load the unique hashed names of files under static
        static_files = os.path.join(self.paths.get('static_files'), 'static')
        names_file_path = os.path.join(static_files, 'names.json')
        if os.path.exists(names_file_path):
            with open(names_file_path) as handle:
                self.static_names = json.load(handle)
        else:
            self.static_names = {}

        # make python warnings go through the logging system
        logging.captureWarnings(capture=True)

        log = logging.getLogger('reddit')

        # when we're a script (paster run) just set up super simple logging
        if self.running_as_script:
            log.setLevel(logging.INFO)
            log.addHandler(logging.StreamHandler())

        # if in debug mode, override the logging level to DEBUG
        if self.debug:
            log.setLevel(logging.DEBUG)

        # attempt to figure out which pool we're in and add that to the
        # LogRecords.
        try:
            with open("/etc/ec2_asg", "r") as f:
                pool = f.read().strip()
            # clean up the pool name since we're putting stuff after "-"
            pool = pool.partition("-")[0]
        except IOError:
            pool = "reddit-app"
        self.log = logging.LoggerAdapter(log, {"pool": pool})

        # set locations
        locations = pkg_resources.resource_stream(__name__,
                                                  "../data/locations.json")
        self.locations = json.loads(locations.read())

        if not self.media_domain:
            self.media_domain = self.domain
        if self.media_domain == self.domain:
            print >> sys.stderr, (
                "Warning: g.media_domain == g.domain. " +
                "This may give untrusted content access to user cookies")
        if self.oauth_domain == self.domain:
            print >> sys.stderr, ("Warning: g.oauth_domain == g.domain. "
                                  "CORS requests to g.domain will be allowed")

        for arg in sys.argv:
            tokens = arg.split("=")
            if len(tokens) == 2:
                k, v = tokens
                self.log.debug("Overriding g.%s to %s" % (k, v))
                setattr(self, k, v)

        self.reddit_host = socket.gethostname()
        self.reddit_pid = os.getpid()

        if hasattr(signal, 'SIGUSR1'):
            # not all platforms have user signals
            signal.signal(signal.SIGUSR1, thread_dump)

        locale.setlocale(locale.LC_ALL, self.locale)

        # Pre-calculate ratelimit values
        self.RL_RESET_SECONDS = self.config["RL_RESET_MINUTES"] * 60
        self.RL_MAX_REQS = int(self.config["RL_AVG_REQ_PER_SEC"] *
                               self.RL_RESET_SECONDS)

        self.RL_OAUTH_RESET_SECONDS = self.config["RL_OAUTH_RESET_MINUTES"] * 60
        self.RL_OAUTH_MAX_REQS = int(self.config["RL_OAUTH_AVG_REQ_PER_SEC"] *
                                     self.RL_OAUTH_RESET_SECONDS)

        self.RL_LOGIN_MAX_REQS = int(self.config["RL_LOGIN_AVG_PER_SEC"] *
                                     self.RL_RESET_SECONDS)

        self.startup_timer.intermediate("configuration")

        ################# ZOOKEEPER
        # for now, zookeeper will be an optional part of the stack.
        # if it's not configured, we will grab the expected config from the
        # [live_config] section of the ini file
        zk_hosts = self.config.get("zookeeper_connection_string")
        if zk_hosts:
            from r2.lib.zookeeper import (connect_to_zookeeper, LiveConfig,
                                          LiveList)
            zk_username = self.config["zookeeper_username"]
            zk_password = self.config["zookeeper_password"]
            self.zookeeper = connect_to_zookeeper(zk_hosts,
                                                  (zk_username, zk_password))
            self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE)
            self.secrets = fetch_secrets(self.zookeeper)
            self.throttles = LiveList(self.zookeeper,
                                      "/throttles",
                                      map_fn=ipaddress.ip_network,
                                      reduce_fn=ipaddress.collapse_addresses)

            # close our zk connection when the app shuts down
            SHUTDOWN_CALLBACKS.append(self.zookeeper.stop)
        else:
            self.zookeeper = None
            parser = ConfigParser.RawConfigParser()
            parser.optionxform = str
            parser.read([self.config["__file__"]])
            self.live_config = extract_live_config(parser, self.plugins)
            self.secrets = extract_secrets(parser)
            self.throttles = tuple()  # immutable since it's not real

        self.startup_timer.intermediate("zookeeper")

        ################# PRIVILEGED USERS
        self.admins = PermissionFilteredEmployeeList(self.live_config,
                                                     type="admin")
        self.sponsors = PermissionFilteredEmployeeList(self.live_config,
                                                       type="sponsor")
        self.employees = PermissionFilteredEmployeeList(self.live_config,
                                                        type="employee")

        ################# MEMCACHE
        num_mc_clients = self.num_mc_clients

        # the main memcache pool. used for most everything.
        memcache = CMemcache(
            self.memcaches,
            min_compress_len=1400,
            num_clients=num_mc_clients,
            binary=True,
        )

        # a pool just used for @memoize results
        memoizecaches = CMemcache(
            self.memoizecaches,
            min_compress_len=50 * 1024,
            num_clients=num_mc_clients,
            binary=True,
        )

        # a pool just for srmember rels
        srmembercaches = CMemcache(
            self.srmembercaches,
            min_compress_len=96,
            num_clients=num_mc_clients,
            binary=True,
        )

        # a pool just for rels
        relcaches = CMemcache(
            self.relcaches,
            min_compress_len=96,
            num_clients=num_mc_clients,
            binary=True,
        )

        ratelimitcaches = CMemcache(
            self.ratelimitcaches,
            min_compress_len=96,
            num_clients=num_mc_clients,
        )

        # a smaller pool of caches used only for distributed locks.
        # TODO: move this to ZooKeeper
        self.lock_cache = CMemcache(self.lockcaches,
                                    binary=True,
                                    num_clients=num_mc_clients)
        self.make_lock = make_lock_factory(self.lock_cache, self.stats)

        # memcaches used in front of the permacache CF in cassandra.
        # XXX: this is a legacy thing; permacache was made when C* didn't have
        # a row cache.
        permacache_memcaches = CMemcache(self.permacache_memcaches,
                                         min_compress_len=1400,
                                         num_clients=num_mc_clients)

        # the stalecache is a memcached local to the current app server used
        # for data that's frequently fetched but doesn't need to be fresh.
        if self.stalecaches:
            stalecaches = CMemcache(self.stalecaches,
                                    binary=True,
                                    num_clients=num_mc_clients)
        else:
            stalecaches = None

        # rendercache holds rendered partial templates.
        rendercaches = CMemcache(
            self.rendercaches,
            noreply=True,
            no_block=True,
            num_clients=num_mc_clients,
            min_compress_len=480,
        )

        # pagecaches hold fully rendered pages
        pagecaches = CMemcache(
            self.pagecaches,
            noreply=True,
            no_block=True,
            num_clients=num_mc_clients,
            min_compress_len=1400,
        )

        self.startup_timer.intermediate("memcache")

        ################# CASSANDRA
        keyspace = "reddit"
        self.cassandra_pools = {
            "main":
            StatsCollectingConnectionPool(keyspace,
                                          stats=self.stats,
                                          logging_name="main",
                                          server_list=self.cassandra_seeds,
                                          pool_size=self.cassandra_pool_size,
                                          timeout=4,
                                          max_retries=3,
                                          prefill=False),
        }

        permacache_cf = Permacache._setup_column_family(
            'permacache',
            self.cassandra_pools[self.cassandra_default_pool],
        )

        self.startup_timer.intermediate("cassandra")

        ################# POSTGRES
        self.dbm = self.load_db_params()
        self.startup_timer.intermediate("postgres")

        ################# CHAINS
        # initialize caches. Any cache-chains built here must be added
        # to cache_chains (closed around by reset_caches) so that they
        # can properly reset their local components
        cache_chains = {}
        localcache_cls = (SelfEmptyingCache
                          if self.running_as_script else LocalCache)

        if stalecaches:
            self.cache = StaleCacheChain(
                localcache_cls(),
                stalecaches,
                memcache,
            )
        else:
            self.cache = MemcacheChain((localcache_cls(), memcache))
        cache_chains.update(cache=self.cache)

        if stalecaches:
            self.memoizecache = StaleCacheChain(
                localcache_cls(),
                stalecaches,
                memoizecaches,
            )
        else:
            self.memoizecache = MemcacheChain(
                (localcache_cls(), memoizecaches))
        cache_chains.update(memoizecache=self.memoizecache)

        if stalecaches:
            self.srmembercache = StaleCacheChain(
                localcache_cls(),
                stalecaches,
                srmembercaches,
            )
        else:
            self.srmembercache = MemcacheChain(
                (localcache_cls(), srmembercaches))
        cache_chains.update(srmembercache=self.srmembercache)

        if stalecaches:
            self.relcache = StaleCacheChain(
                localcache_cls(),
                stalecaches,
                relcaches,
            )
        else:
            self.relcache = MemcacheChain((localcache_cls(), relcaches))
        cache_chains.update(relcache=self.relcache)

        self.ratelimitcache = MemcacheChain(
            (localcache_cls(), ratelimitcaches))
        cache_chains.update(ratelimitcache=self.ratelimitcache)

        self.rendercache = MemcacheChain((
            localcache_cls(),
            rendercaches,
        ))
        cache_chains.update(rendercache=self.rendercache)

        self.pagecache = MemcacheChain((
            localcache_cls(),
            pagecaches,
        ))
        cache_chains.update(pagecache=self.pagecache)

        # the thing_cache is used in tdb_cassandra.
        self.thing_cache = CacheChain((localcache_cls(), ), check_keys=False)
        cache_chains.update(thing_cache=self.thing_cache)

        if stalecaches:
            permacache_cache = StaleCacheChain(
                localcache_cls(),
                stalecaches,
                permacache_memcaches,
                check_keys=False,
            )
        else:
            permacache_cache = CacheChain(
                (localcache_cls(), permacache_memcaches),
                check_keys=False,
            )
        cache_chains.update(permacache=permacache_cache)

        self.permacache = Permacache(
            permacache_cache,
            permacache_cf,
            lock_factory=self.make_lock,
        )

        # hardcache is used for various things that tend to expire
        # TODO: replace hardcache w/ cassandra stuff
        self.hardcache = HardcacheChain(
            (localcache_cls(), memcache, HardCache(self)),
            cache_negative_results=True,
        )
        cache_chains.update(hardcache=self.hardcache)

        # I know this sucks, but we need non-request-threads to be
        # able to reset the caches, so we need them be able to close
        # around 'cache_chains' without being able to call getattr on
        # 'g'
        def reset_caches():
            for name, chain in cache_chains.iteritems():
                chain.reset()
                if isinstance(chain, StaleCacheChain):
                    chain.stats = StaleCacheStats(self.stats, name)
                else:
                    chain.stats = CacheStats(self.stats, name)

        self.cache_chains = cache_chains

        self.reset_caches = reset_caches
        self.reset_caches()

        self.startup_timer.intermediate("cache_chains")

        # try to set the source control revision numbers
        self.versions = {}
        r2_root = os.path.dirname(os.path.dirname(self.paths["root"]))
        r2_gitdir = os.path.join(r2_root, ".git")
        self.short_version = self.record_repo_version("r2", r2_gitdir)

        if I18N_PATH:
            i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git")
            self.record_repo_version("i18n", i18n_git_path)

        self.startup_timer.intermediate("revisions")

    def setup_complete(self):
        self.startup_timer.stop()
        self.stats.flush()

        if self.log_start:
            self.log.error("%s:%s started %s at %s (took %.02fs)",
                           self.reddit_host, self.reddit_pid,
                           self.short_version,
                           datetime.now().strftime("%H:%M:%S"),
                           self.startup_timer.elapsed_seconds())

    def record_repo_version(self, repo_name, git_dir):
        """Get the currently checked out git revision for a given repository,
        record it in g.versions, and return the short version of the hash."""
        try:
            subprocess.check_output
        except AttributeError:
            # python 2.6 compat
            pass
        else:
            try:
                revision = subprocess.check_output(
                    ["git", "--git-dir", git_dir, "rev-parse", "HEAD"])
            except subprocess.CalledProcessError, e:
                self.log.warning("Unable to fetch git revision: %r", e)
            else:
Beispiel #31
0
 def test_set_of(self):
     self.assertEquals(set([]), ConfigValue.set_of(str)(''))
     self.assertEquals(set(['a', 'b']), ConfigValue.set_of(str)('a, b, b'))
     self.assertEquals(set(['a', 'b']),
                       ConfigValue.set_of(str, delim=':')('b : a : b'))
Beispiel #32
0
class FreeToPlay(Plugin):
    needs_static_build = True

    config = {
        ConfigValue.tuple: [
            "f2pcaches",
        ],

        ConfigValue.dict(str, str): [
            "team_subreddits",
            "steam_promo_items",
        ],
    }

    js = {
        'reddit': Module('reddit.js',
            'lib/iso8601.js',
            'f2p/scrollupdater.js',
            'f2p/f2p.js',
            'f2p/utils.js',
            'f2p/items.js',
            TemplateFileSource('f2p/panel.html'),
            TemplateFileSource('f2p/login-message.html'),
            TemplateFileSource('f2p/item.html'),
            TemplateFileSource('f2p/item-bubble.html'),
            TemplateFileSource('f2p/scores.html'),
            TemplateFileSource('f2p/target-overlay.html'),
        )
    }

    live_config = {
        ConfigValue.float: [
            'drop_cooldown_mu',
            'drop_cooldown_sigma',
        ],

        ConfigValue.dict(str, int): [
            'f2p_rarity_weights',
        ],
    }

    def declare_queues(self, queues):
        # imported here so we don't depend on pyx files at import time
        # which allows "make" to work in a clean clone of the repos
        from r2.config.queues import MessageQueue

        queues.declare({
            "steam_q": MessageQueue(bind_to_self=True),
        })

    def on_load(self, g):
        from r2.lib.cache import CMemcache, MemcacheChain, LocalCache

        # TODO: use SelfEmptyingCache for localcache if we use this in jobs
        f2p_memcaches = CMemcache(
            'f2p',
            g.f2pcaches,
            num_clients=g.num_mc_clients,
        )
        g.f2pcache = MemcacheChain((
            LocalCache(),
            f2p_memcaches,
        ))
        g.cache_chains.update(f2p=g.f2pcache)

        compendium = pkg_resources.resource_stream(__name__,
                                                   "data/compendium.json")
        g.f2pitems = json.load(compendium)
        for kind, data in g.f2pitems.iteritems():
            data["kind"] = kind

    def add_routes(self, mc):
        mc('/f2p/gamelog', controller='gamelog', action='listing')
        mc('/api/f2p/:action', controller='freetoplayapi')
        mc('/f2p/steam/:action', controller='steam', action='start')

    def load_controllers(self):
        from r2.lib.pages import Reddit
        Reddit.extra_stylesheets.append('f2p.less')

        from reddit_f2p import f2p
        f2p.hooks.register_all()
        f2p.monkeypatch()

        from reddit_f2p.steam import SteamController
        from reddit_f2p.gamelog import GameLogController
Beispiel #33
0
 def test_tuple_of(self):
     self.assertEquals((), ConfigValue.tuple_of(str)(''))
     self.assertEquals(('a', 'b'), ConfigValue.tuple_of(str)('a, b'))
     self.assertEquals(('a', 'b'),
                       ConfigValue.tuple_of(str, delim=':')('a : b'))
class Adzerk(Plugin):
    needs_static_build = True

    config = {
        ConfigValue.str: [
            'adzerk_engine_domain',
        ],

        ConfigValue.int: [
            'az_selfserve_salesperson_id',
            'az_selfserve_network_id',
            'az_reporting_timeout',
        ],

        ConfigValue.float: [
            'display_ad_skip_probability',
        ],

        ConfigValue.tuple: [
            'display_ad_skip_keywords',
        ],

        ConfigValue.dict(ConfigValue.str, ConfigValue.int): [
            'az_selfserve_priorities',
            'az_selfserve_site_ids',
        ],

        ConfigValue.tuple_of(ConfigValue.int): [
            'adserver_campaign_ids',
        ],
    }

    js = {
        'reddit-init': Module('reddit-init.js',
            'adzerk/adzerk.js',
        ),

        'display': Module('display.js',
            'adzerk/display.js',
        ),

        'companion': Module('companion.js',
            'adzerk/companion.js',
        ),

        'ad-dependencies': Module('ad-dependencies.js',
            'adzerk/jquery.js',
        ),
    }

    def add_routes(self, mc):
        mc('/api/request_promo/', controller='adzerkapi', action='request_promo')
        mc('/ads/display/300x250/', controller='adserving', action='ad_300_250')
        mc('/ads/display/300x250-companion/', controller='adserving', action='ad_300_250_companion')

    def declare_queues(self, queues):
        from r2.config.queues import MessageQueue
        queues.declare({
            "adzerk_q": MessageQueue(bind_to_self=True),
            "adzerk_reporting_q": MessageQueue(bind_to_self=True),
        })

    def load_controllers(self):
        # replace the standard Ads view with an Adzerk specific one.
        import r2.lib.pages.pages
        from adzerkads import Ads as AdzerkAds
        r2.lib.pages.pages.Ads = AdzerkAds

        # replace standard adserver with Adzerk.
        from adzerkpromote import AdzerkApiController
        from adzerkpromote import hooks as adzerkpromote_hooks
        from adzerkads import AdServingController
        adzerkpromote_hooks.register_all()
Beispiel #35
0
 def test_timeinterval(self):
     self.assertEquals(datetime.timedelta(0, 60),
                       ConfigValue.timeinterval('1 minute'))
     with self.assertRaises(KeyError):
         ConfigValue.timeinterval('asdf')
Beispiel #36
0
 def test_choice(self):
     self.assertEquals(1, ConfigValue.choice(alpha=1)('alpha'))
     self.assertEquals(2, ConfigValue.choice(alpha=1, beta=2)('beta'))
     with self.assertRaises(ValueError):
         ConfigValue.choice(alpha=1)('asdf')
Beispiel #37
0
    def load_db_params(self):
        self.databases = tuple(ConfigValue.to_iter(self.config.raw_data['databases']))
        self.db_params = {}
        if not self.databases:
            return

        dbm = db_manager.db_manager()
        db_param_names = ('name', 'db_host', 'db_user', 'db_pass', 'db_port',
                          'pool_size', 'max_overflow')
        for db_name in self.databases:
            conf_params = ConfigValue.to_iter(self.config.raw_data[db_name + '_db'])
            params = dict(zip(db_param_names, conf_params))
            if params['db_user'] == "*":
                params['db_user'] = self.db_user
            if params['db_pass'] == "*":
                params['db_pass'] = self.db_pass
            if params['db_port'] == "*":
                params['db_port'] = self.db_port

            if params['pool_size'] == "*":
                params['pool_size'] = self.db_pool_size
            if params['max_overflow'] == "*":
                params['max_overflow'] = self.db_pool_overflow_size

            dbm.setup_db(db_name, g_override=self, **params)
            self.db_params[db_name] = params

        dbm.type_db = dbm.get_engine(self.config.raw_data['type_db'])
        dbm.relation_type_db = dbm.get_engine(self.config.raw_data['rel_type_db'])

        def split_flags(raw_params):
            params = []
            flags = {}

            for param in raw_params:
                if not param.startswith("!"):
                    params.append(param)
                else:
                    key, sep, value = param[1:].partition("=")
                    if sep:
                        flags[key] = value
                    else:
                        flags[key] = True

            return params, flags

        prefix = 'db_table_'
        self.predefined_type_ids = {}
        for k, v in self.config.raw_data.iteritems():
            if not k.startswith(prefix):
                continue

            params, table_flags = split_flags(ConfigValue.to_iter(v))
            name = k[len(prefix):]
            kind = params[0]
            server_list = self.config.raw_data["db_servers_" + name]
            engines, flags = split_flags(ConfigValue.to_iter(server_list))

            typeid = table_flags.get("typeid")
            if typeid:
                self.predefined_type_ids[name] = int(typeid)

            if kind == 'thing':
                dbm.add_thing(name, dbm.get_engines(engines),
                              **flags)
            elif kind == 'relation':
                dbm.add_relation(name, params[1], params[2],
                                 dbm.get_engines(engines),
                                 **flags)
        return dbm
Beispiel #38
0
 def test_timeinterval(self):
     self.assertEquals(datetime.timedelta(0, 60),
                       ConfigValue.timeinterval('1 minute'))
     with self.assertRaises(KeyError):
         ConfigValue.timeinterval('asdf')
Beispiel #39
0
    def load_db_params(self):
        self.databases = tuple(
            ConfigValue.to_iter(self.config.raw_data['databases']))
        self.db_params = {}
        if not self.databases:
            return

        dbm = db_manager.db_manager()
        db_param_names = ('name', 'db_host', 'db_user', 'db_pass', 'db_port',
                          'pool_size', 'max_overflow')
        for db_name in self.databases:
            conf_params = ConfigValue.to_iter(self.config.raw_data[db_name +
                                                                   '_db'])
            params = dict(zip(db_param_names, conf_params))
            if params['db_user'] == "*":
                params['db_user'] = self.db_user
            if params['db_pass'] == "*":
                params['db_pass'] = self.db_pass
            if params['db_port'] == "*":
                params['db_port'] = self.db_port

            if params['pool_size'] == "*":
                params['pool_size'] = self.db_pool_size
            if params['max_overflow'] == "*":
                params['max_overflow'] = self.db_pool_overflow_size

            dbm.setup_db(db_name, g_override=self, **params)
            self.db_params[db_name] = params

        dbm.type_db = dbm.get_engine(self.config.raw_data['type_db'])
        dbm.relation_type_db = dbm.get_engine(
            self.config.raw_data['rel_type_db'])

        def split_flags(raw_params):
            params = []
            flags = {}

            for param in raw_params:
                if not param.startswith("!"):
                    params.append(param)
                else:
                    key, sep, value = param[1:].partition("=")
                    if sep:
                        flags[key] = value
                    else:
                        flags[key] = True

            return params, flags

        prefix = 'db_table_'
        self.predefined_type_ids = {}
        for k, v in self.config.raw_data.iteritems():
            if not k.startswith(prefix):
                continue

            params, table_flags = split_flags(ConfigValue.to_iter(v))
            name = k[len(prefix):]
            kind = params[0]
            server_list = self.config.raw_data["db_servers_" + name]
            engines, flags = split_flags(ConfigValue.to_iter(server_list))

            typeid = table_flags.get("typeid")
            if typeid:
                self.predefined_type_ids[name] = int(typeid)

            if kind == 'thing':
                dbm.add_thing(name, dbm.get_engines(engines), **flags)
            elif kind == 'relation':
                dbm.add_relation(name, params[1], params[2],
                                 dbm.get_engines(engines), **flags)
        return dbm
Beispiel #40
0
class Robin(Plugin):
    needs_static_build = True

    js = {
        "robin": LocalizedModule("robin.js",
            "lib/page-visibility.js",
            "lib/tinycon.js",
            "websocket.js",
            TemplateFileSource("robin/robinmessage.html"),
            TemplateFileSource("robin/robinroomparticipant.html"),
            "errors.js",
            "models/validators.js",
            "robin/models.js",
            "robin/views.js",
            "robin/notifications.js",
            "robin/favicon.js",
            "robin/init.js",
        ),

        "robin-join": Module("robin-join.js",
            "robin/join.js",
        ),
    }

    live_config = {
        ConfigValue.int: [
            "robin_ratelimit_window",
        ],

        ConfigValue.dict(ConfigValue.int, ConfigValue.float): [
            "robin_ratelimit_avg_per_sec",
        ],
    }

    def declare_queues(self, queues):
        from r2.config.queues import MessageQueue

        queues.declare({
            "robin_presence_q": MessageQueue(),
            "robin_waitinglist_q": MessageQueue(bind_to_self=True),
            "robin_subreddit_maker_q": MessageQueue(bind_to_self=True),
        })

        queues.robin_presence_q << (
            "websocket.connect",
            "websocket.disconnect",
        )

    def add_routes(self, mc):
        mc("/robin", controller="robin", action="chat",
            conditions={"function": not_in_sr})
        mc("/robin/all", controller="robin", action="all",
            conditions={"function": not_in_sr})
        mc("/robin/admin", controller="robin", action="admin",
            conditions={"function": not_in_sr})
        mc("/robin/join", controller="robin", action="join",
            conditions={"function": not_in_sr})
        mc("/robin/:room_id", controller="robin", action="force_room",
            conditions={"function": not_in_sr})
        mc("/robin/user/:user", controller="robin", action="user_room",
            conditions={"function": not_in_sr})
        mc("/api/robin/:room_id/:action", controller="robin",
            conditions={"function": not_in_sr})
        mc("/api/join_room", controller="robin", action="join_room",
            conditions={"function": not_in_sr})
        mc("/api/room_assignment", controller="robin", action="room_assignment",
            conditions={"function": not_in_sr})
        mc("/api/admin_prompt", controller="robin", action="admin_prompt",
            conditions={"function": not_in_sr})
        mc("/api/admin_reap", controller="robin", action="admin_reap",
            conditions={"function": not_in_sr})
        mc("/api/admin_broadcast", controller="robin", action="admin_broadcast",
            conditions={"function": not_in_sr})

    def load_controllers(self):
        from r2.lib.pages import Reddit
        from reddit_robin.controllers import (
            RobinController,
        )

        Reddit.extra_stylesheets.append('robin_global.less')

        from reddit_robin.hooks import hooks
        hooks.register_all()