def setup(self): self.queues = queues.declare_queues(self) self.extension_subdomains = dict( m="mobile", i="compact", api="api", rss="rss", xml="xml", json="json", ) ################# PROVIDERS self.auth_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.auth", self.authentication_provider, ) self.media_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.media", self.media_provider, ) self.cdn_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.cdn", self.cdn_provider, ) self.startup_timer.intermediate("providers") ################# CONFIGURATION # AMQP is required if not self.amqp_host: raise ValueError("amqp_host not set in the .ini") if not self.cassandra_seeds: raise ValueError("cassandra_seeds not set in the .ini") # heavy load mode is read only mode with a different infobar if self.heavy_load_mode: self.read_only_mode = True origin_prefix = self.domain_prefix + "." if self.domain_prefix else "" self.origin = "http://" + origin_prefix + self.domain self.trusted_domains = set([self.domain]) if self.https_endpoint: https_url = urlparse(self.https_endpoint) self.trusted_domains.add(https_url.hostname) # load the unique hashed names of files under static static_files = os.path.join(self.paths.get('static_files'), 'static') names_file_path = os.path.join(static_files, 'names.json') if os.path.exists(names_file_path): with open(names_file_path) as handle: self.static_names = json.load(handle) else: self.static_names = {} # make python warnings go through the logging system logging.captureWarnings(capture=True) log = logging.getLogger('reddit') # when we're a script (paster run) just set up super simple logging if self.running_as_script: log.setLevel(logging.INFO) log.addHandler(logging.StreamHandler()) # if in debug mode, override the logging level to DEBUG if self.debug: log.setLevel(logging.DEBUG) # attempt to figure out which pool we're in and add that to the # LogRecords. try: with open("/etc/ec2_asg", "r") as f: pool = f.read().strip() # clean up the pool name since we're putting stuff after "-" pool = pool.partition("-")[0] except IOError: pool = "reddit-app" self.log = logging.LoggerAdapter(log, {"pool": pool}) # set locations locations = pkg_resources.resource_stream(__name__, "../data/locations.json") self.locations = json.loads(locations.read()) if not self.media_domain: self.media_domain = self.domain if self.media_domain == self.domain: print >> sys.stderr, ( "Warning: g.media_domain == g.domain. " + "This may give untrusted content access to user cookies") if self.oauth_domain == self.domain: print >> sys.stderr, ("Warning: g.oauth_domain == g.domain. " "CORS requests to g.domain will be allowed") for arg in sys.argv: tokens = arg.split("=") if len(tokens) == 2: k, v = tokens self.log.debug("Overriding g.%s to %s" % (k, v)) setattr(self, k, v) self.reddit_host = socket.gethostname() self.reddit_pid = os.getpid() if hasattr(signal, 'SIGUSR1'): # not all platforms have user signals signal.signal(signal.SIGUSR1, thread_dump) locale.setlocale(locale.LC_ALL, self.locale) # Pre-calculate ratelimit values self.RL_RESET_SECONDS = self.config["RL_RESET_MINUTES"] * 60 self.RL_MAX_REQS = int(self.config["RL_AVG_REQ_PER_SEC"] * self.RL_RESET_SECONDS) self.RL_OAUTH_RESET_SECONDS = self.config["RL_OAUTH_RESET_MINUTES"] * 60 self.RL_OAUTH_MAX_REQS = int(self.config["RL_OAUTH_AVG_REQ_PER_SEC"] * self.RL_OAUTH_RESET_SECONDS) self.RL_LOGIN_MAX_REQS = int(self.config["RL_LOGIN_AVG_PER_SEC"] * self.RL_RESET_SECONDS) self.startup_timer.intermediate("configuration") ################# ZOOKEEPER # for now, zookeeper will be an optional part of the stack. # if it's not configured, we will grab the expected config from the # [live_config] section of the ini file zk_hosts = self.config.get("zookeeper_connection_string") if zk_hosts: from r2.lib.zookeeper import (connect_to_zookeeper, LiveConfig, LiveList) zk_username = self.config["zookeeper_username"] zk_password = self.config["zookeeper_password"] self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username, zk_password)) self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE) self.secrets = fetch_secrets(self.zookeeper) self.throttles = LiveList(self.zookeeper, "/throttles", map_fn=ipaddress.ip_network, reduce_fn=ipaddress.collapse_addresses) # close our zk connection when the app shuts down SHUTDOWN_CALLBACKS.append(self.zookeeper.stop) else: self.zookeeper = None parser = ConfigParser.RawConfigParser() parser.optionxform = str parser.read([self.config["__file__"]]) self.live_config = extract_live_config(parser, self.plugins) self.secrets = extract_secrets(parser) self.throttles = tuple() # immutable since it's not real self.startup_timer.intermediate("zookeeper") ################# PRIVILEGED USERS self.admins = PermissionFilteredEmployeeList(self.live_config, type="admin") self.sponsors = PermissionFilteredEmployeeList(self.live_config, type="sponsor") self.employees = PermissionFilteredEmployeeList(self.live_config, type="employee") ################# MEMCACHE num_mc_clients = self.num_mc_clients # the main memcache pool. used for most everything. memcache = CMemcache( self.memcaches, min_compress_len=1400, num_clients=num_mc_clients, binary=True, ) # a pool just used for @memoize results memoizecaches = CMemcache( self.memoizecaches, min_compress_len=50 * 1024, num_clients=num_mc_clients, binary=True, ) # a pool just for srmember rels srmembercaches = CMemcache( self.srmembercaches, min_compress_len=96, num_clients=num_mc_clients, binary=True, ) # a pool just for rels relcaches = CMemcache( self.relcaches, min_compress_len=96, num_clients=num_mc_clients, binary=True, ) ratelimitcaches = CMemcache( self.ratelimitcaches, min_compress_len=96, num_clients=num_mc_clients, ) # a smaller pool of caches used only for distributed locks. # TODO: move this to ZooKeeper self.lock_cache = CMemcache(self.lockcaches, binary=True, num_clients=num_mc_clients) self.make_lock = make_lock_factory(self.lock_cache, self.stats) # memcaches used in front of the permacache CF in cassandra. # XXX: this is a legacy thing; permacache was made when C* didn't have # a row cache. permacache_memcaches = CMemcache(self.permacache_memcaches, min_compress_len=1400, num_clients=num_mc_clients) # the stalecache is a memcached local to the current app server used # for data that's frequently fetched but doesn't need to be fresh. if self.stalecaches: stalecaches = CMemcache(self.stalecaches, binary=True, num_clients=num_mc_clients) else: stalecaches = None # rendercache holds rendered partial templates. rendercaches = CMemcache( self.rendercaches, noreply=True, no_block=True, num_clients=num_mc_clients, min_compress_len=480, ) # pagecaches hold fully rendered pages pagecaches = CMemcache( self.pagecaches, noreply=True, no_block=True, num_clients=num_mc_clients, min_compress_len=1400, ) self.startup_timer.intermediate("memcache") ################# CASSANDRA keyspace = "reddit" self.cassandra_pools = { "main": StatsCollectingConnectionPool(keyspace, stats=self.stats, logging_name="main", server_list=self.cassandra_seeds, pool_size=self.cassandra_pool_size, timeout=4, max_retries=3, prefill=False), } permacache_cf = Permacache._setup_column_family( 'permacache', self.cassandra_pools[self.cassandra_default_pool], ) self.startup_timer.intermediate("cassandra") ################# POSTGRES self.dbm = self.load_db_params() self.startup_timer.intermediate("postgres") ################# CHAINS # initialize caches. Any cache-chains built here must be added # to cache_chains (closed around by reset_caches) so that they # can properly reset their local components cache_chains = {} localcache_cls = (SelfEmptyingCache if self.running_as_script else LocalCache) if stalecaches: self.cache = StaleCacheChain( localcache_cls(), stalecaches, memcache, ) else: self.cache = MemcacheChain((localcache_cls(), memcache)) cache_chains.update(cache=self.cache) if stalecaches: self.memoizecache = StaleCacheChain( localcache_cls(), stalecaches, memoizecaches, ) else: self.memoizecache = MemcacheChain( (localcache_cls(), memoizecaches)) cache_chains.update(memoizecache=self.memoizecache) if stalecaches: self.srmembercache = StaleCacheChain( localcache_cls(), stalecaches, srmembercaches, ) else: self.srmembercache = MemcacheChain( (localcache_cls(), srmembercaches)) cache_chains.update(srmembercache=self.srmembercache) if stalecaches: self.relcache = StaleCacheChain( localcache_cls(), stalecaches, relcaches, ) else: self.relcache = MemcacheChain((localcache_cls(), relcaches)) cache_chains.update(relcache=self.relcache) self.ratelimitcache = MemcacheChain( (localcache_cls(), ratelimitcaches)) cache_chains.update(ratelimitcache=self.ratelimitcache) self.rendercache = MemcacheChain(( localcache_cls(), rendercaches, )) cache_chains.update(rendercache=self.rendercache) self.pagecache = MemcacheChain(( localcache_cls(), pagecaches, )) cache_chains.update(pagecache=self.pagecache) # the thing_cache is used in tdb_cassandra. self.thing_cache = CacheChain((localcache_cls(), ), check_keys=False) cache_chains.update(thing_cache=self.thing_cache) if stalecaches: permacache_cache = StaleCacheChain( localcache_cls(), stalecaches, permacache_memcaches, check_keys=False, ) else: permacache_cache = CacheChain( (localcache_cls(), permacache_memcaches), check_keys=False, ) cache_chains.update(permacache=permacache_cache) self.permacache = Permacache( permacache_cache, permacache_cf, lock_factory=self.make_lock, ) # hardcache is used for various things that tend to expire # TODO: replace hardcache w/ cassandra stuff self.hardcache = HardcacheChain( (localcache_cls(), memcache, HardCache(self)), cache_negative_results=True, ) cache_chains.update(hardcache=self.hardcache) # I know this sucks, but we need non-request-threads to be # able to reset the caches, so we need them be able to close # around 'cache_chains' without being able to call getattr on # 'g' def reset_caches(): for name, chain in cache_chains.iteritems(): chain.reset() if isinstance(chain, StaleCacheChain): chain.stats = StaleCacheStats(self.stats, name) else: chain.stats = CacheStats(self.stats, name) self.cache_chains = cache_chains self.reset_caches = reset_caches self.reset_caches() self.startup_timer.intermediate("cache_chains") # try to set the source control revision numbers self.versions = {} r2_root = os.path.dirname(os.path.dirname(self.paths["root"])) r2_gitdir = os.path.join(r2_root, ".git") self.short_version = self.record_repo_version("r2", r2_gitdir) if I18N_PATH: i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git") self.record_repo_version("i18n", i18n_git_path) self.startup_timer.intermediate("revisions")
def setup(self): self.env = '' if ( # handle direct invocation of "nosetests" "test" in sys.argv[0] or # handle "setup.py test" and all permutations thereof. "setup.py" in sys.argv[0] and "test" in sys.argv[1:]): self.env = "unit_test" self.queues = queues.declare_queues(self) self.extension_subdomains = dict( simple="mobile", i="compact", api="api", rss="rss", xml="xml", json="json", ) ################# PROVIDERS self.auth_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.auth", self.authentication_provider, ) self.media_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.media", self.media_provider, ) self.cdn_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.cdn", self.cdn_provider, ) self.ticket_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.support", # TODO: fix this later, it refuses to pick up # g.config['ticket_provider'] value, so hardcoding for now. # really, the next uncommented line should be: #self.ticket_provider, # instead of: "zendesk", ) self.image_resizing_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.image_resizing", self.image_resizing_provider, ) self.email_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.email", self.email_provider, ) self.startup_timer.intermediate("providers") ################# CONFIGURATION # AMQP is required if not self.amqp_host: raise ValueError("amqp_host not set in the .ini") if not self.cassandra_seeds: raise ValueError("cassandra_seeds not set in the .ini") # heavy load mode is read only mode with a different infobar if self.heavy_load_mode: self.read_only_mode = True origin_prefix = self.domain_prefix + "." if self.domain_prefix else "" self.origin = self.default_scheme + "://" + origin_prefix + self.domain self.trusted_domains = set([self.domain]) if self.https_endpoint: https_url = urlparse(self.https_endpoint) self.trusted_domains.add(https_url.hostname) # load the unique hashed names of files under static static_files = os.path.join(self.paths.get('static_files'), 'static') names_file_path = os.path.join(static_files, 'names.json') if os.path.exists(names_file_path): with open(names_file_path) as handle: self.static_names = json.load(handle) else: self.static_names = {} # make python warnings go through the logging system logging.captureWarnings(capture=True) log = logging.getLogger('reddit') # when we're a script (paster run) just set up super simple logging if self.running_as_script: log.setLevel(logging.INFO) log.addHandler(logging.StreamHandler()) # if in debug mode, override the logging level to DEBUG if self.debug: log.setLevel(logging.DEBUG) # attempt to figure out which pool we're in and add that to the # LogRecords. try: with open("/etc/ec2_asg", "r") as f: pool = f.read().strip() # clean up the pool name since we're putting stuff after "-" pool = pool.partition("-")[0] except IOError: pool = "reddit-app" self.log = logging.LoggerAdapter(log, {"pool": pool}) # set locations locations = pkg_resources.resource_stream(__name__, "../data/locations.json") self.locations = json.loads(locations.read()) if not self.media_domain: self.media_domain = self.domain if self.media_domain == self.domain: print >> sys.stderr, ( "Warning: g.media_domain == g.domain. " + "This may give untrusted content access to user cookies") if self.oauth_domain == self.domain: print >> sys.stderr, ("Warning: g.oauth_domain == g.domain. " "CORS requests to g.domain will be allowed") for arg in sys.argv: tokens = arg.split("=") if len(tokens) == 2: k, v = tokens self.log.debug("Overriding g.%s to %s" % (k, v)) setattr(self, k, v) self.reddit_host = socket.gethostname() self.reddit_pid = os.getpid() if hasattr(signal, 'SIGUSR1'): # not all platforms have user signals signal.signal(signal.SIGUSR1, thread_dump) locale.setlocale(locale.LC_ALL, self.locale) # Pre-calculate ratelimit values self.RL_RESET_SECONDS = self.config["RL_RESET_MINUTES"] * 60 self.RL_MAX_REQS = int(self.config["RL_AVG_REQ_PER_SEC"] * self.RL_RESET_SECONDS) self.RL_OAUTH_RESET_SECONDS = self.config["RL_OAUTH_RESET_MINUTES"] * 60 self.RL_OAUTH_MAX_REQS = int(self.config["RL_OAUTH_AVG_REQ_PER_SEC"] * self.RL_OAUTH_RESET_SECONDS) self.RL_LOGIN_MAX_REQS = int(self.config["RL_LOGIN_AVG_PER_SEC"] * self.RL_RESET_SECONDS) self.RL_LOGIN_IP_MAX_REQS = int( self.config["RL_LOGIN_IP_AVG_PER_SEC"] * self.RL_RESET_SECONDS) self.RL_SHARE_MAX_REQS = int(self.config["RL_SHARE_AVG_PER_SEC"] * self.RL_RESET_SECONDS) # Compile ratelimit regexs user_agent_ratelimit_regexes = {} for agent_re, limit in self.user_agent_ratelimit_regexes.iteritems(): user_agent_ratelimit_regexes[re.compile(agent_re)] = limit self.user_agent_ratelimit_regexes = user_agent_ratelimit_regexes self.startup_timer.intermediate("configuration") ################# ZOOKEEPER # for now, zookeeper will be an optional part of the stack. # if it's not configured, we will grab the expected config from the # [live_config] section of the ini file zk_hosts = self.config.get("zookeeper_connection_string") if zk_hosts: from r2.lib.zookeeper import (connect_to_zookeeper, LiveConfig, LiveList) zk_username = self.config["zookeeper_username"] zk_password = self.config["zookeeper_password"] self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username, zk_password)) self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE) self.secrets = fetch_secrets(self.zookeeper) self.throttles = LiveList(self.zookeeper, "/throttles", map_fn=ipaddress.ip_network, reduce_fn=ipaddress.collapse_addresses) # close our zk connection when the app shuts down SHUTDOWN_CALLBACKS.append(self.zookeeper.stop) else: self.zookeeper = None parser = ConfigParser.RawConfigParser() parser.optionxform = str parser.read([self.config["__file__"]]) self.live_config = extract_live_config(parser, self.plugins) self.secrets = extract_secrets(parser) self.throttles = tuple() # immutable since it's not real ################# PRIVILEGED USERS self.admins = PermissionFilteredEmployeeList(self.live_config, type="admin") self.sponsors = PermissionFilteredEmployeeList(self.live_config, type="sponsor") self.employees = PermissionFilteredEmployeeList(self.live_config, type="employee") # Store which OAuth clients employees may use, the keys are just for # readability. self.employee_approved_clients = \ self.live_config["employee_approved_clients"].values() self.startup_timer.intermediate("zookeeper") ################# MEMCACHE num_mc_clients = self.num_mc_clients # the main memcache pool. used for most everything. memcaches = CMemcache( "main", self.memcaches, min_compress_len=1400, num_clients=num_mc_clients, validators=[validate_size_error], ) # a pool just used for @memoize results memoizecaches = CMemcache( "memoize", self.memoizecaches, min_compress_len=50 * 1024, num_clients=num_mc_clients, validators=[validate_size_error], ) # a pool just for srmember rels srmembercaches = CMemcache( "srmember", self.srmembercaches, min_compress_len=96, num_clients=num_mc_clients, validators=[validate_size_error], ) # a pool just for rels relcaches = CMemcache( "rel", self.relcaches, min_compress_len=96, num_clients=num_mc_clients, validators=[validate_size_error], ) ratelimitcaches = CMemcache( "ratelimit", self.ratelimitcaches, min_compress_len=96, num_clients=num_mc_clients, validators=[validate_size_error], ) # a smaller pool of caches used only for distributed locks. self.lock_cache = CMemcache( "lock", self.lockcaches, num_clients=num_mc_clients, validators=[validate_size_error], ) self.make_lock = make_lock_factory(self.lock_cache, self.stats) # memcaches used in front of the permacache CF in cassandra. # XXX: this is a legacy thing; permacache was made when C* didn't have # a row cache. permacache_memcaches = CMemcache( "perma", self.permacache_memcaches, min_compress_len=1400, num_clients=num_mc_clients, validators=[], ) # the stalecache is a memcached local to the current app server used # for data that's frequently fetched but doesn't need to be fresh. if self.stalecaches: stalecaches = CMemcache( "stale", self.stalecaches, num_clients=num_mc_clients, validators=[validate_size_error], ) else: stalecaches = None # hardcache memcache pool hardcache_memcaches = CMemcache( "hardcache", self.hardcache_memcaches, binary=True, min_compress_len=1400, num_clients=num_mc_clients, validators=[validate_size_error], ) self.startup_timer.intermediate("memcache") ################# MCROUTER self.mcrouter = Mcrouter( "mcrouter", self.mcrouter_addr, min_compress_len=1400, num_clients=1, ) ################# THRIFT-BASED SERVICES activity_endpoint = self.config.get("activity_endpoint") if activity_endpoint: # make ActivityInfo objects rendercache-key friendly # TODO: figure out a more general solution for this if # we need to do this for other thrift-generated objects ActivityInfo.cache_key = lambda self, style: repr(self) activity_pool = ThriftConnectionPool(activity_endpoint, timeout=0.1) self.baseplate.add_to_context( "activity_service", ThriftContextFactory(activity_pool, ActivityService.Client)) self.startup_timer.intermediate("thrift") ################# CASSANDRA keyspace = "reddit" self.cassandra_pools = { "main": StatsCollectingConnectionPool(keyspace, stats=self.stats, logging_name="main", server_list=self.cassandra_seeds, pool_size=self.cassandra_pool_size, timeout=4, max_retries=3, prefill=False), } permacache_cf = Permacache._setup_column_family( 'permacache', self.cassandra_pools[self.cassandra_default_pool], ) self.startup_timer.intermediate("cassandra") ################# POSTGRES self.dbm = self.load_db_params() self.startup_timer.intermediate("postgres") ################# CHAINS # initialize caches. Any cache-chains built here must be added # to cache_chains (closed around by reset_caches) so that they # can properly reset their local components cache_chains = {} localcache_cls = (SelfEmptyingCache if self.running_as_script else LocalCache) if stalecaches: self.cache = StaleCacheChain( localcache_cls(), stalecaches, memcaches, ) else: self.cache = CacheChain((localcache_cls(), memcaches)) cache_chains.update(cache=self.cache) if stalecaches: self.thingcache = StaleCacheChain( localcache_cls(), stalecaches, self.mcrouter, ) else: self.thingcache = CacheChain((localcache_cls(), self.mcrouter)) cache_chains.update(thingcache=self.thingcache) def get_new_account_prefix_and_key(key, prefix=''): old_prefix = "Account_" new_prefix = "account:" if prefix: assert prefix == old_prefix return new_prefix, key else: key = str(key) assert key.startswith(old_prefix) account_id = key[len(old_prefix):] return '', new_prefix + account_id self.account_transitionalcache = TransitionalCache( original_cache=self.cache, replacement_cache=self.thingcache, read_original=True, key_transform=get_new_account_prefix_and_key, ) if stalecaches: self.memoizecache = StaleCacheChain( localcache_cls(), stalecaches, memoizecaches, ) else: self.memoizecache = MemcacheChain( (localcache_cls(), memoizecaches)) cache_chains.update(memoizecache=self.memoizecache) if stalecaches: self.srmembercache = StaleCacheChain( localcache_cls(), stalecaches, srmembercaches, ) else: self.srmembercache = MemcacheChain( (localcache_cls(), srmembercaches)) cache_chains.update(srmembercache=self.srmembercache) if stalecaches: self.relcache = StaleCacheChain( localcache_cls(), stalecaches, relcaches, ) else: self.relcache = MemcacheChain((localcache_cls(), relcaches)) cache_chains.update(relcache=self.relcache) self.ratelimitcache = MemcacheChain( (localcache_cls(), ratelimitcaches)) cache_chains.update(ratelimitcache=self.ratelimitcache) # rendercache holds rendered partial templates. self.rendercache = MemcacheChain(( localcache_cls(), self.mcrouter, )) cache_chains.update(rendercache=self.rendercache) # pagecaches hold fully rendered pages (includes comment panes) self.pagecache = MemcacheChain(( localcache_cls(), self.mcrouter, )) cache_chains.update(pagecache=self.pagecache) # cassandra_local_cache is used for request-local caching in tdb_cassandra self.cassandra_local_cache = localcache_cls() cache_chains.update(cassandra_local_cache=self.cassandra_local_cache) if stalecaches: permacache_cache = StaleCacheChain( localcache_cls(), stalecaches, permacache_memcaches, ) else: permacache_cache = CacheChain( (localcache_cls(), permacache_memcaches), ) cache_chains.update(permacache=permacache_cache) self.permacache = Permacache( permacache_cache, permacache_cf, lock_factory=self.make_lock, ) # hardcache is used for various things that tend to expire # TODO: replace hardcache w/ cassandra stuff self.hardcache = HardcacheChain( (localcache_cls(), hardcache_memcaches, HardCache(self)), cache_negative_results=True, ) cache_chains.update(hardcache=self.hardcache) # I know this sucks, but we need non-request-threads to be # able to reset the caches, so we need them be able to close # around 'cache_chains' without being able to call getattr on # 'g' def reset_caches(): for name, chain in cache_chains.iteritems(): if isinstance(chain, TransitionalCache): chain = chain.read_chain chain.reset() if isinstance(chain, LocalCache): continue elif isinstance(chain, StaleCacheChain): chain.stats = StaleCacheStats(self.stats, name) else: chain.stats = CacheStats(self.stats, name) self.cache_chains = cache_chains self.reset_caches = reset_caches self.reset_caches() self.startup_timer.intermediate("cache_chains") # try to set the source control revision numbers self.versions = {} r2_root = os.path.dirname(os.path.dirname(self.paths["root"])) r2_gitdir = os.path.join(r2_root, ".git") self.short_version = self.record_repo_version("r2", r2_gitdir) if I18N_PATH: i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git") self.record_repo_version("i18n", i18n_git_path) # Initialize the amqp module globals, start the worker, etc. r2.lib.amqp.initialize(self) self.events = EventQueue() self.startup_timer.intermediate("revisions")
def setup(self): self.queues = queues.declare_queues(self) ################# CONFIGURATION # AMQP is required if not self.amqp_host: raise ValueError("amqp_host not set in the .ini") # This requirement doesn't *have* to be a requirement, but there are # bugs at the moment that will pop up if you violate it # XXX: get rid of these options. new query cache is always on. if self.write_query_queue and not self.use_query_cache: raise Exception("write_query_queue requires use_query_cache") if not self.cassandra_seeds: raise ValueError("cassandra_seeds not set in the .ini") # heavy load mode is read only mode with a different infobar if self.heavy_load_mode: self.read_only_mode = True origin_prefix = self.domain_prefix + "." if self.domain_prefix else "" self.origin = "http://" + origin_prefix + self.domain self.secure_domains = set([urlparse(self.payment_domain).netloc]) self.trusted_domains = set([self.domain]) self.trusted_domains.update(self.authorized_cnames) if self.https_endpoint: https_url = urlparse(self.https_endpoint) self.secure_domains.add(https_url.netloc) self.trusted_domains.add(https_url.hostname) if getattr(self, 'oauth_domain', None): self.secure_domains.add(self.oauth_domain) # load the unique hashed names of files under static static_files = os.path.join(self.paths.get('static_files'), 'static') names_file_path = os.path.join(static_files, 'names.json') if os.path.exists(names_file_path): with open(names_file_path) as handle: self.static_names = json.load(handle) else: self.static_names = {} #setup the logger self.log = logging.getLogger('reddit') self.log.addHandler(logging.StreamHandler()) if self.debug: self.log.setLevel(logging.DEBUG) else: self.log.setLevel(logging.INFO) # set log level for pycountry which is chatty logging.getLogger('pycountry.db').setLevel(logging.CRITICAL) if not self.media_domain: self.media_domain = self.domain if self.media_domain == self.domain: print ("Warning: g.media_domain == g.domain. " + "This may give untrusted content access to user cookies") for arg in sys.argv: tokens = arg.split("=") if len(tokens) == 2: k, v = tokens self.log.debug("Overriding g.%s to %s" % (k, v)) setattr(self, k, v) self.reddit_host = socket.gethostname() self.reddit_pid = os.getpid() if hasattr(signal, 'SIGUSR1'): # not all platforms have user signals signal.signal(signal.SIGUSR1, thread_dump) self.startup_timer.intermediate("configuration") ################# ZOOKEEPER # for now, zookeeper will be an optional part of the stack. # if it's not configured, we will grab the expected config from the # [live_config] section of the ini file zk_hosts = self.config.get("zookeeper_connection_string") if zk_hosts: from r2.lib.zookeeper import (connect_to_zookeeper, LiveConfig, LiveList) zk_username = self.config["zookeeper_username"] zk_password = self.config["zookeeper_password"] self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username, zk_password)) self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE) self.throttles = LiveList(self.zookeeper, "/throttles", map_fn=ipaddress.ip_network, reduce_fn=ipaddress.collapse_addresses) else: self.zookeeper = None parser = ConfigParser.RawConfigParser() parser.read([self.config["__file__"]]) self.live_config = extract_live_config(parser, self.plugins) self.throttles = tuple() # immutable since it's not real self.startup_timer.intermediate("zookeeper") ################# MEMCACHE num_mc_clients = self.num_mc_clients # the main memcache pool. used for most everything. self.memcache = CMemcache(self.memcaches, num_clients=num_mc_clients) # a smaller pool of caches used only for distributed locks. # TODO: move this to ZooKeeper self.lock_cache = CMemcache(self.lockcaches, num_clients=num_mc_clients) self.make_lock = make_lock_factory(self.lock_cache, self.stats) # memcaches used in front of the permacache CF in cassandra. # XXX: this is a legacy thing; permacache was made when C* didn't have # a row cache. if self.permacache_memcaches: permacache_memcaches = CMemcache(self.permacache_memcaches, num_clients=num_mc_clients) else: permacache_memcaches = None # the stalecache is a memcached local to the current app server used # for data that's frequently fetched but doesn't need to be fresh. if self.stalecaches: stalecaches = CMemcache(self.stalecaches, num_clients=num_mc_clients) else: stalecaches = None # rendercache holds rendered partial templates as well as fully # cached pages. rendercaches = CMemcache( self.rendercaches, noreply=True, no_block=True, num_clients=num_mc_clients, ) self.startup_timer.intermediate("memcache") ################# CASSANDRA keyspace = "reddit" self.cassandra_pools = { "main": StatsCollectingConnectionPool( keyspace, stats=self.stats, logging_name="main", server_list=self.cassandra_seeds, pool_size=self.cassandra_pool_size, timeout=2, max_retries=3, prefill=False ), } permacache_cf = CassandraCache( 'permacache', self.cassandra_pools[self.cassandra_default_pool], read_consistency_level=self.cassandra_rcl, write_consistency_level=self.cassandra_wcl ) self.startup_timer.intermediate("cassandra") ################# POSTGRES event.listens_for(engine.Engine, 'before_cursor_execute')( self.stats.pg_before_cursor_execute) event.listens_for(engine.Engine, 'after_cursor_execute')( self.stats.pg_after_cursor_execute) self.dbm = self.load_db_params() self.startup_timer.intermediate("postgres") ################# CHAINS # initialize caches. Any cache-chains built here must be added # to cache_chains (closed around by reset_caches) so that they # can properly reset their local components self.cache_chains = {} localcache_cls = (SelfEmptyingCache if self.running_as_script else LocalCache) if stalecaches: self.cache = StaleCacheChain( localcache_cls(), stalecaches, self.memcache, ) else: self.cache = MemcacheChain((localcache_cls(), self.memcache)) self.cache_chains.update(cache=self.cache) self.rendercache = MemcacheChain(( localcache_cls(), rendercaches, )) self.cache_chains.update(rendercache=self.rendercache) # the thing_cache is used in tdb_cassandra. self.thing_cache = CacheChain((localcache_cls(),)) self.cache_chains.update(thing_cache=self.thing_cache) self.permacache = CassandraCacheChain( localcache_cls(), permacache_cf, memcache=permacache_memcaches, lock_factory=self.make_lock, ) self.cache_chains.update(permacache=self.permacache) # hardcache is used for various things that tend to expire # TODO: replace hardcache w/ cassandra stuff self.hardcache = HardcacheChain( (localcache_cls(), self.memcache, HardCache(self)), cache_negative_results=True, ) self.cache_chains.update(hardcache=self.hardcache) # I know this sucks, but we need non-request-threads to be # able to reset the caches, so we need them be able to close # around 'cache_chains' without being able to call getattr on # 'g' cache_chains = self.cache_chains.copy() def reset_caches(): for name, chain in cache_chains.iteritems(): chain.reset() chain.stats = CacheStats(self.stats, name) self.reset_caches = reset_caches self.reset_caches() self.startup_timer.intermediate("cache_chains") # try to set the source control revision numbers self.versions = {} r2_root = os.path.dirname(os.path.dirname(self.paths["root"])) r2_gitdir = os.path.join(r2_root, ".git") self.short_version = self.record_repo_version("r2", r2_gitdir) if I18N_PATH: i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git") self.record_repo_version("i18n", i18n_git_path) self.startup_timer.intermediate("revisions")
def setup(self): # heavy load mode is read only mode with a different infobar if self.heavy_load_mode: self.read_only_mode = True if hasattr(signal, 'SIGUSR1'): # not all platforms have user signals signal.signal(signal.SIGUSR1, thread_dump) # initialize caches. Any cache-chains built here must be added # to cache_chains (closed around by reset_caches) so that they # can properly reset their local components localcache_cls = (SelfEmptyingCache if self.running_as_script else LocalCache) num_mc_clients = self.num_mc_clients self.cache_chains = {} # for now, zookeeper will be an optional part of the stack. # if it's not configured, we will grab the expected config from the # [live_config] section of the ini file zk_hosts = self.config.get("zookeeper_connection_string") if zk_hosts: from r2.lib.zookeeper import (connect_to_zookeeper, LiveConfig, LiveList) zk_username = self.config["zookeeper_username"] zk_password = self.config["zookeeper_password"] self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username, zk_password)) self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE) self.throttles = LiveList(self.zookeeper, "/throttles", map_fn=ipaddress.ip_network, reduce_fn=ipaddress.collapse_addresses) else: self.zookeeper = None parser = ConfigParser.RawConfigParser() parser.read([self.config["__file__"]]) self.live_config = extract_live_config(parser, self.plugins) self.throttles = tuple() # immutable since it's not real self.memcache = CMemcache(self.memcaches, num_clients=num_mc_clients) self.lock_cache = CMemcache(self.lockcaches, num_clients=num_mc_clients) self.stats = Stats(self.config.get('statsd_addr'), self.config.get('statsd_sample_rate')) event.listens_for(engine.Engine, 'before_cursor_execute')( self.stats.pg_before_cursor_execute) event.listens_for(engine.Engine, 'after_cursor_execute')( self.stats.pg_after_cursor_execute) self.make_lock = make_lock_factory(self.lock_cache, self.stats) if not self.cassandra_seeds: raise ValueError("cassandra_seeds not set in the .ini") keyspace = "reddit" self.cassandra_pools = { "main": StatsCollectingConnectionPool(keyspace, stats=self.stats, logging_name="main", server_list=self.cassandra_seeds, pool_size=self.cassandra_pool_size, timeout=2, max_retries=3, prefill=False), } perma_memcache = (CMemcache(self.permacache_memcaches, num_clients=num_mc_clients) if self.permacache_memcaches else None) self.permacache = CassandraCacheChain( localcache_cls(), CassandraCache('permacache', self.cassandra_pools[self.cassandra_default_pool], read_consistency_level=self.cassandra_rcl, write_consistency_level=self.cassandra_wcl), memcache=perma_memcache, lock_factory=self.make_lock) self.cache_chains.update(permacache=self.permacache) # hardcache is done after the db info is loaded, and then the # chains are reset to use the appropriate initial entries if self.stalecaches: self.cache = StaleCacheChain( localcache_cls(), CMemcache(self.stalecaches, num_clients=num_mc_clients), self.memcache) else: self.cache = MemcacheChain((localcache_cls(), self.memcache)) self.cache_chains.update(cache=self.cache) self.rendercache = MemcacheChain( (localcache_cls(), CMemcache(self.rendercaches, noreply=True, no_block=True, num_clients=num_mc_clients))) self.cache_chains.update(rendercache=self.rendercache) self.thing_cache = CacheChain((localcache_cls(), )) self.cache_chains.update(thing_cache=self.thing_cache) #load the database info self.dbm = self.load_db_params() # can't do this until load_db_params() has been called self.hardcache = HardcacheChain( (localcache_cls(), self.memcache, HardCache(self)), cache_negative_results=True) self.cache_chains.update(hardcache=self.hardcache) # I know this sucks, but we need non-request-threads to be # able to reset the caches, so we need them be able to close # around 'cache_chains' without being able to call getattr on # 'g' cache_chains = self.cache_chains.copy() def reset_caches(): for name, chain in cache_chains.iteritems(): chain.reset() chain.stats = CacheStats(self.stats, name) self.reset_caches = reset_caches self.reset_caches() # set the modwindow self.MODWINDOW = timedelta(self.MODWINDOW) self.REDDIT_MAIN = bool(os.environ.get('REDDIT_MAIN')) origin_prefix = self.domain_prefix + "." if self.domain_prefix else "" self.origin = "http://" + origin_prefix + self.domain self.secure_domains = set([urlparse(self.payment_domain).netloc]) self.trusted_domains = set([self.domain]) self.trusted_domains.update(self.authorized_cnames) if self.https_endpoint: https_url = urlparse(self.https_endpoint) self.secure_domains.add(https_url.netloc) self.trusted_domains.add(https_url.hostname) if getattr(self, 'oauth_domain', None): self.secure_domains.add(self.oauth_domain) # load the unique hashed names of files under static static_files = os.path.join(self.paths.get('static_files'), 'static') names_file_path = os.path.join(static_files, 'names.json') if os.path.exists(names_file_path): with open(names_file_path) as handle: self.static_names = json.load(handle) else: self.static_names = {} #setup the logger self.log = logging.getLogger('reddit') self.log.addHandler(logging.StreamHandler()) if self.debug: self.log.setLevel(logging.DEBUG) else: self.log.setLevel(logging.INFO) # set log level for pycountry which is chatty logging.getLogger('pycountry.db').setLevel(logging.CRITICAL) if not self.media_domain: self.media_domain = self.domain if self.media_domain == self.domain: print("Warning: g.media_domain == g.domain. " + "This may give untrusted content access to user cookies") self.reddit_host = socket.gethostname() self.reddit_pid = os.getpid() for arg in sys.argv: tokens = arg.split("=") if len(tokens) == 2: k, v = tokens self.log.debug("Overriding g.%s to %s" % (k, v)) setattr(self, k, v) #if we're going to use the query_queue, we need amqp if self.write_query_queue and not self.amqp_host: raise Exception("amqp_host must be defined to use the query queue") # This requirement doesn't *have* to be a requirement, but there are # bugs at the moment that will pop up if you violate it if self.write_query_queue and not self.use_query_cache: raise Exception("write_query_queue requires use_query_cache") # try to set the source control revision numbers self.versions = {} r2_root = os.path.dirname(os.path.dirname(self.paths["root"])) r2_gitdir = os.path.join(r2_root, ".git") self.short_version = self.record_repo_version("r2", r2_gitdir) if I18N_PATH: i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git") self.record_repo_version("i18n", i18n_git_path) if self.log_start: self.log.error("reddit app %s:%s started %s at %s" % (self.reddit_host, self.reddit_pid, self.short_version, datetime.now()))