def make_wsgi_app(app_config): cfg = config.parse_config(app_config, { "activity": { "endpoint": config.Endpoint, }, }) metrics_client = make_metrics_client(app_config) pool = ThriftConnectionPool(cfg.activity.endpoint) baseplate = Baseplate() baseplate.configure_logging() baseplate.configure_metrics(metrics_client) baseplate.add_to_context( "activity", ThriftContextFactory(pool, ActivityService.Client)) configurator = Configurator(settings=app_config) baseplate_configurator = BaseplateConfigurator(baseplate) configurator.include(baseplate_configurator.includeme) controller = ActivityGateway() configurator.add_route("health", "/health", request_method="GET") configurator.add_view(controller.is_healthy, route_name="health", renderer="json") configurator.add_route("pixel", "/{context_id:[A-Za-z0-9_]{,40}}.png", request_method="GET") configurator.add_view(controller.pixel, route_name="pixel") return configurator.make_wsgi_app()
def baseplate_thrift_client(endpoint, client_span_observer=None): pool = ThriftConnectionPool(endpoint) context = RequestContext() server_span = ServerSpan( trace_id=1234, parent_id=2345, span_id=3456, flags=4567, sampled=1, name="example_service.example", context=context, ) if client_span_observer: class TestServerSpanObserver(ServerSpanObserver): def on_child_span_created(self, span): span.register(client_span_observer) server_span.register(TestServerSpanObserver()) edge_context_factory = make_edge_context_factory() edge_context = edge_context_factory.from_upstream( SERIALIZED_EDGECONTEXT_WITH_VALID_AUTH) edge_context.attach_context(context) context_factory = ThriftContextFactory(pool, TestService.Client) client = context_factory.make_object_for_context("example_service", server_span) setattr(context, "example_service", client) yield context
def setup(self): self.env = '' if ( # handle direct invocation of "nosetests" "test" in sys.argv[0] or # handle "setup.py test" and all permutations thereof. "setup.py" in sys.argv[0] and "test" in sys.argv[1:] ): self.env = "unit_test" self.queues = queues.declare_queues(self) self.extension_subdomains = dict( simple="mobile", i="compact", api="api", rss="rss", xml="xml", json="json", ) ################# PROVIDERS self.auth_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.auth", self.authentication_provider, ) self.media_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.media", self.media_provider, ) self.cdn_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.cdn", self.cdn_provider, ) self.ticket_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.support", # TODO: fix this later, it refuses to pick up # g.config['ticket_provider'] value, so hardcoding for now. # really, the next uncommented line should be: #self.ticket_provider, # instead of: "zendesk", ) self.image_resizing_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.image_resizing", self.image_resizing_provider, ) self.email_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.email", self.email_provider, ) self.startup_timer.intermediate("providers") ################# CONFIGURATION # AMQP is required if not self.amqp_host: raise ValueError("amqp_host not set in the .ini") if not self.cassandra_seeds: raise ValueError("cassandra_seeds not set in the .ini") # heavy load mode is read only mode with a different infobar if self.heavy_load_mode: self.read_only_mode = True origin_prefix = self.domain_prefix + "." if self.domain_prefix else "" self.origin = self.default_scheme + "://" + origin_prefix + self.domain self.trusted_domains = set([self.domain]) if self.https_endpoint: https_url = urlparse(self.https_endpoint) self.trusted_domains.add(https_url.hostname) # load the unique hashed names of files under static static_files = os.path.join(self.paths.get('static_files'), 'static') names_file_path = os.path.join(static_files, 'names.json') if os.path.exists(names_file_path): with open(names_file_path) as handle: self.static_names = json.load(handle) else: self.static_names = {} # make python warnings go through the logging system logging.captureWarnings(capture=True) log = logging.getLogger('reddit') # when we're a script (paster run) just set up super simple logging if self.running_as_script: log.setLevel(logging.INFO) log.addHandler(logging.StreamHandler()) # if in debug mode, override the logging level to DEBUG if self.debug: log.setLevel(logging.DEBUG) # attempt to figure out which pool we're in and add that to the # LogRecords. try: with open("/etc/ec2_asg", "r") as f: pool = f.read().strip() # clean up the pool name since we're putting stuff after "-" pool = pool.partition("-")[0] except IOError: pool = "reddit-app" self.log = logging.LoggerAdapter(log, {"pool": pool}) # set locations locations = pkg_resources.resource_stream(__name__, "../data/locations.json") self.locations = json.loads(locations.read()) if not self.media_domain: self.media_domain = self.domain if self.media_domain == self.domain: print >> sys.stderr, ("Warning: g.media_domain == g.domain. " + "This may give untrusted content access to user cookies") if self.oauth_domain == self.domain: print >> sys.stderr, ("Warning: g.oauth_domain == g.domain. " "CORS requests to g.domain will be allowed") for arg in sys.argv: tokens = arg.split("=") if len(tokens) == 2: k, v = tokens self.log.debug("Overriding g.%s to %s" % (k, v)) setattr(self, k, v) self.reddit_host = socket.gethostname() self.reddit_pid = os.getpid() if hasattr(signal, 'SIGUSR1'): # not all platforms have user signals signal.signal(signal.SIGUSR1, thread_dump) locale.setlocale(locale.LC_ALL, self.locale) # Pre-calculate ratelimit values self.RL_RESET_SECONDS = self.config["RL_RESET_MINUTES"] * 60 self.RL_MAX_REQS = int(self.config["RL_AVG_REQ_PER_SEC"] * self.RL_RESET_SECONDS) self.RL_OAUTH_RESET_SECONDS = self.config["RL_OAUTH_RESET_MINUTES"] * 60 self.RL_OAUTH_MAX_REQS = int(self.config["RL_OAUTH_AVG_REQ_PER_SEC"] * self.RL_OAUTH_RESET_SECONDS) self.RL_LOGIN_MAX_REQS = int(self.config["RL_LOGIN_AVG_PER_SEC"] * self.RL_RESET_SECONDS) self.RL_LOGIN_IP_MAX_REQS = int(self.config["RL_LOGIN_IP_AVG_PER_SEC"] * self.RL_RESET_SECONDS) self.RL_SHARE_MAX_REQS = int(self.config["RL_SHARE_AVG_PER_SEC"] * self.RL_RESET_SECONDS) # Compile ratelimit regexs user_agent_ratelimit_regexes = {} for agent_re, limit in self.user_agent_ratelimit_regexes.iteritems(): user_agent_ratelimit_regexes[re.compile(agent_re)] = limit self.user_agent_ratelimit_regexes = user_agent_ratelimit_regexes self.startup_timer.intermediate("configuration") ################# ZOOKEEPER zk_hosts = self.config["zookeeper_connection_string"] zk_username = self.config["zookeeper_username"] zk_password = self.config["zookeeper_password"] self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username, zk_password)) self.throttles = IPNetworkLiveList( self.zookeeper, root="/throttles", reduced_data_node="/throttles_reduced", ) parser = ConfigParser.RawConfigParser() parser.optionxform = str parser.read([self.config["__file__"]]) if self.config["liveconfig_source"] == "zookeeper": self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE) else: self.live_config = extract_live_config(parser, self.plugins) if self.config["secrets_source"] == "zookeeper": self.secrets = fetch_secrets(self.zookeeper) else: self.secrets = extract_secrets(parser) ################# PRIVILEGED USERS self.admins = PermissionFilteredEmployeeList( self.live_config, type="admin") self.sponsors = PermissionFilteredEmployeeList( self.live_config, type="sponsor") self.employees = PermissionFilteredEmployeeList( self.live_config, type="employee") # Store which OAuth clients employees may use, the keys are just for # readability. self.employee_approved_clients = \ self.live_config["employee_approved_clients"].values() self.startup_timer.intermediate("zookeeper") ################# MEMCACHE num_mc_clients = self.num_mc_clients # a smaller pool of caches used only for distributed locks. self.lock_cache = CMemcache( "lock", self.lockcaches, num_clients=num_mc_clients, ) self.make_lock = make_lock_factory(self.lock_cache, self.stats) # memcaches used in front of the permacache CF in cassandra. # XXX: this is a legacy thing; permacache was made when C* didn't have # a row cache. permacache_memcaches = CMemcache( "perma", self.permacache_memcaches, min_compress_len=1400, num_clients=num_mc_clients, ) # the stalecache is a memcached local to the current app server used # for data that's frequently fetched but doesn't need to be fresh. if self.stalecaches: stalecaches = CMemcache( "stale", self.stalecaches, num_clients=num_mc_clients, ) else: stalecaches = None self.startup_timer.intermediate("memcache") ################# MCROUTER self.mcrouter = Mcrouter( "mcrouter", self.mcrouter_addr, min_compress_len=1400, num_clients=num_mc_clients, ) ################# THRIFT-BASED SERVICES activity_endpoint = self.config.get("activity_endpoint") if activity_endpoint: # make ActivityInfo objects rendercache-key friendly # TODO: figure out a more general solution for this if # we need to do this for other thrift-generated objects ActivityInfo.cache_key = lambda self, style: repr(self) activity_pool = ThriftConnectionPool(activity_endpoint, timeout=0.1) self.baseplate.add_to_context("activity_service", ThriftContextFactory(activity_pool, ActivityService.Client)) self.startup_timer.intermediate("thrift") ################# CASSANDRA keyspace = "reddit" self.cassandra_pools = { "main": StatsCollectingConnectionPool( keyspace, stats=self.stats, logging_name="main", server_list=self.cassandra_seeds, pool_size=self.cassandra_pool_size, timeout=4, max_retries=3, prefill=False ), } permacache_cf = Permacache._setup_column_family( 'permacache', self.cassandra_pools[self.cassandra_default_pool], ) self.startup_timer.intermediate("cassandra") ################# POSTGRES self.dbm = self.load_db_params() self.startup_timer.intermediate("postgres") ################# CHAINS # initialize caches. Any cache-chains built here must be added # to cache_chains (closed around by reset_caches) so that they # can properly reset their local components cache_chains = {} localcache_cls = (SelfEmptyingCache if self.running_as_script else LocalCache) if stalecaches: self.gencache = StaleCacheChain( localcache_cls(), stalecaches, self.mcrouter, ) else: self.gencache = CacheChain((localcache_cls(), self.mcrouter)) cache_chains.update(gencache=self.gencache) if stalecaches: self.thingcache = StaleCacheChain( localcache_cls(), stalecaches, self.mcrouter, ) else: self.thingcache = CacheChain((localcache_cls(), self.mcrouter)) cache_chains.update(thingcache=self.thingcache) if stalecaches: self.memoizecache = StaleCacheChain( localcache_cls(), stalecaches, self.mcrouter, ) else: self.memoizecache = MemcacheChain( (localcache_cls(), self.mcrouter)) cache_chains.update(memoizecache=self.memoizecache) if stalecaches: self.srmembercache = StaleCacheChain( localcache_cls(), stalecaches, self.mcrouter, ) else: self.srmembercache = MemcacheChain( (localcache_cls(), self.mcrouter)) cache_chains.update(srmembercache=self.srmembercache) if stalecaches: self.relcache = StaleCacheChain( localcache_cls(), stalecaches, self.mcrouter, ) else: self.relcache = MemcacheChain( (localcache_cls(), self.mcrouter)) cache_chains.update(relcache=self.relcache) self.ratelimitcache = MemcacheChain( (localcache_cls(), self.mcrouter)) cache_chains.update(ratelimitcache=self.ratelimitcache) # rendercache holds rendered partial templates. self.rendercache = MemcacheChain(( localcache_cls(), self.mcrouter, )) cache_chains.update(rendercache=self.rendercache) # commentpanecaches hold fully rendered comment panes self.commentpanecache = MemcacheChain(( localcache_cls(), self.mcrouter, )) cache_chains.update(commentpanecache=self.commentpanecache) # cassandra_local_cache is used for request-local caching in tdb_cassandra self.cassandra_local_cache = localcache_cls() cache_chains.update(cassandra_local_cache=self.cassandra_local_cache) if stalecaches: permacache_cache = StaleCacheChain( localcache_cls(), stalecaches, permacache_memcaches, ) else: permacache_cache = CacheChain( (localcache_cls(), permacache_memcaches), ) cache_chains.update(permacache=permacache_cache) self.permacache = Permacache( permacache_cache, permacache_cf, lock_factory=self.make_lock, ) # hardcache is used for various things that tend to expire # TODO: replace hardcache w/ cassandra stuff self.hardcache = HardcacheChain( (localcache_cls(), HardCache(self)), cache_negative_results=True, ) cache_chains.update(hardcache=self.hardcache) # I know this sucks, but we need non-request-threads to be # able to reset the caches, so we need them be able to close # around 'cache_chains' without being able to call getattr on # 'g' def reset_caches(): for name, chain in cache_chains.iteritems(): if isinstance(chain, TransitionalCache): chain = chain.read_chain chain.reset() if isinstance(chain, LocalCache): continue elif isinstance(chain, StaleCacheChain): chain.stats = StaleCacheStats(self.stats, name) else: chain.stats = CacheStats(self.stats, name) self.cache_chains = cache_chains self.reset_caches = reset_caches self.reset_caches() self.startup_timer.intermediate("cache_chains") # try to set the source control revision numbers self.versions = {} r2_root = os.path.dirname(os.path.dirname(self.paths["root"])) r2_gitdir = os.path.join(r2_root, ".git") self.short_version = self.record_repo_version("r2", r2_gitdir) if I18N_PATH: i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git") self.record_repo_version("i18n", i18n_git_path) # Initialize the amqp module globals, start the worker, etc. r2.lib.amqp.initialize(self) self.events = EventQueue() self.startup_timer.intermediate("revisions")
def check_thrift_service(endpoint): pool = ThriftConnectionPool(endpoint, size=1, timeout=TIMEOUT) with pool.connection() as protocol: client = BaseplateService.Client(protocol) assert client.is_healthy(), "service indicated unhealthiness"
def raw_thrift_client(endpoint): pool = ThriftConnectionPool(endpoint) with pool.connection() as client_protocol: yield TestService.Client(client_protocol)
def check_thrift_service(endpoint): pool = ThriftConnectionPool(endpoint, size=1, timeout=TIMEOUT) with pool.connection() as protocol: client = BaseplateService.Client(protocol) assert client.is_healthy(), "service indicated unhealthiness"