def pre(self): c.start_time = datetime.now(g.tz) g.reset_caches() c.domain_prefix = request.environ.get("reddit-domain-prefix", g.domain_prefix) c.secure = request.host in g.secure_domains #check if user-agent needs a dose of rate-limiting if not c.error_page: ratelimit_throttled() ratelimit_agents() c.allow_loggedin_cache = False c.show_wiki_actions = False # the domain has to be set before Cookies get initialized set_subreddit() c.errors = ErrorSet() c.cookies = Cookies() # if an rss feed, this will also log the user in if a feed= # GET param is included set_content_type()
def pre(self): action = request.environ["pylons.routes_dict"].get("action") if action: c.request_timer = g.stats.get_timer(request_timer_name(action)) else: c.request_timer = SimpleSillyStub() c.response_wrappers = [] c.start_time = datetime.now(g.tz) c.request_timer.start() g.reset_caches() c.domain_prefix = request.environ.get("reddit-domain-prefix", g.domain_prefix) c.secure = request.host in g.secure_domains #check if user-agent needs a dose of rate-limiting if not c.error_page: ratelimit_throttled() ratelimit_agents() c.allow_loggedin_cache = False c.show_wiki_actions = False # the domain has to be set before Cookies get initialized set_subreddit() c.errors = ErrorSet() c.cookies = Cookies() # if an rss feed, this will also log the user in if a feed= # GET param is included set_content_type() c.request_timer.intermediate("minimal-pre")
def handle_items(queue, callback, ack = True, limit = 1, drain = False, verbose=True, sleep_time = 1): """Call callback() on every item in a particular queue. If the connection to the queue is lost, it will die. Intended to be used as a long-running process.""" from pylons import c chan = connection_manager.get_channel() countdown = None while True: # NB: None != 0, so we don't need an "is not None" check here if countdown == 0: break msg = chan.basic_get(queue) if not msg and drain: return elif not msg: time.sleep(sleep_time) continue if countdown is None and drain and 'message_count' in msg.delivery_info: countdown = 1 + msg.delivery_info['message_count'] g.reset_caches() c.use_write_db = {} items = [] while msg and countdown != 0: items.append(msg) if countdown is not None: countdown -= 1 if len(items) >= limit: break # the innermost loop only msg = chan.basic_get(queue) try: count_str = '' if 'message_count' in items[-1].delivery_info: # the count from the last message, if the count is # available count_str = '(%d remaining)' % items[-1].delivery_info['message_count'] if verbose: print "%s: %d items %s" % (queue, len(items), count_str) callback(items, chan) if ack: # ack *all* outstanding messages chan.basic_ack(0, multiple=True) # flush any log messages printed by the callback sys.stdout.flush() except: for item in items: # explicitly reject the items that we've not processed chan.basic_reject(item.delivery_tag, requeue = True) raise
def pre(self): action = request.environ["pylons.routes_dict"].get("action") if action: c.request_timer = g.stats.get_timer(request_timer_name(action)) else: c.request_timer = SimpleSillyStub() c.response_wrapper = None c.start_time = datetime.now(g.tz) c.request_timer.start() g.reset_caches() c.domain_prefix = request.environ.get("reddit-domain-prefix", g.domain_prefix) c.secure = request.host in g.secure_domains #check if user-agent needs a dose of rate-limiting if not c.error_page: ratelimit_throttled() ratelimit_agents() c.allow_loggedin_cache = False # the domain has to be set before Cookies get initialized set_subreddit() c.errors = ErrorSet() c.cookies = Cookies() # if an rss feed, this will also log the user in if a feed= # GET param is included set_content_type() c.request_timer.intermediate("minimal-pre")
def pre(self): action = request.environ["pylons.routes_dict"].get("action") if action: if not self._get_action_handler(): action = 'invalid' controller = request.environ["pylons.routes_dict"]["controller"] key = "{}.{}".format(controller, action) c.request_timer = g.stats.get_timer(request_timer_name(key)) else: c.request_timer = SimpleSillyStub() c.response_wrapper = None c.start_time = datetime.now(g.tz) c.request_timer.start() g.reset_caches() c.domain_prefix = request.environ.get("reddit-domain-prefix", g.domain_prefix) c.secure = request.environ["wsgi.url_scheme"] == "https" c.request_origin = request.host_url #check if user-agent needs a dose of rate-limiting if not c.error_page: ratelimit_throttled() ratelimit_agents() c.allow_loggedin_cache = False c.allow_framing = False c.cdn_cacheable = (request.via_cdn and g.login_cookie not in request.cookies) # the domain has to be set before Cookies get initialized set_subreddit() c.errors = ErrorSet() c.cookies = Cookies() # if an rss feed, this will also log the user in if a feed= # GET param is included set_content_type() c.request_timer.intermediate("minimal-pre") # True/False forces. None updates for most non-POST requests c.update_last_visit = None g.stats.count_string('user_agents', request.user_agent) if not self.defer_ratelimiting: self.run_sitewide_ratelimits() c.request_timer.intermediate("minimal-ratelimits") hooks.get_hook("reddit.request.minimal_begin").call()
def pre(self): action = request.environ["pylons.routes_dict"].get("action") if action: if not self._get_action_handler(): action = 'invalid' c.request_timer = g.stats.get_timer(request_timer_name(action)) else: c.request_timer = SimpleSillyStub() c.response_wrapper = None c.start_time = datetime.now(g.tz) c.request_timer.start() g.reset_caches() c.domain_prefix = request.environ.get("reddit-domain-prefix", g.domain_prefix) c.secure = request.host in g.secure_domains # wsgi.url_scheme is used in generating absolute urls, such as by webob # for translating some of our relative-url redirects to rfc compliant # absolute-url ones. TODO: consider using one of webob's methods of # setting wsgi.url_scheme based on incoming request headers added by # upstream things like stunnel/haproxy. if c.secure: request.environ["wsgi.url_scheme"] = "https" c.request_origin = request.host_url #check if user-agent needs a dose of rate-limiting if not c.error_page: ratelimit_throttled() ratelimit_agents() c.allow_loggedin_cache = False # the domain has to be set before Cookies get initialized set_subreddit() c.errors = ErrorSet() c.cookies = Cookies() # if an rss feed, this will also log the user in if a feed= # GET param is included set_content_type() c.request_timer.intermediate("minimal-pre") # True/False forces. None updates for most non-POST requests c.update_last_visit = None g.stats.count_string('user_agents', request.user_agent) hooks.get_hook("reddit.request.minimal_begin").call()
def pre(self): action = request.environ["pylons.routes_dict"].get("action") if action: if not self._get_action_handler(): action = 'invalid' c.request_timer = g.stats.get_timer(request_timer_name(action)) else: c.request_timer = SimpleSillyStub() c.response_wrapper = None c.start_time = datetime.now(g.tz) c.request_timer.start() g.reset_caches() c.domain_prefix = request.environ.get("reddit-domain-prefix", g.domain_prefix) c.secure = request.environ["wsgi.url_scheme"] == "https" c.request_origin = request.host_url #check if user-agent needs a dose of rate-limiting if not c.error_page: ratelimit_throttled() ratelimit_agents() c.allow_loggedin_cache = False c.allow_framing = False c.cdn_cacheable = (request.via_cdn and g.login_cookie not in request.cookies) # the domain has to be set before Cookies get initialized set_subreddit() c.errors = ErrorSet() c.cookies = Cookies() # if an rss feed, this will also log the user in if a feed= # GET param is included set_content_type() c.request_timer.intermediate("minimal-pre") # True/False forces. None updates for most non-POST requests c.update_last_visit = None g.stats.count_string('user_agents', request.user_agent) if not self.defer_ratelimiting: self.run_sitewide_ratelimits() c.request_timer.intermediate("minimal-ratelimits") hooks.get_hook("reddit.request.minimal_begin").call()
def _callback(msg): if verbose: count_str = '' if 'message_count' in msg.delivery_info: # the count from the last message, if the count is # available count_str = '(%d remaining)' % msg.delivery_info['message_count'] print "%s: 1 item %s" % (queue, count_str) g.reset_caches() ret = callback(msg) msg.channel.basic_ack(msg.delivery_tag) sys.stdout.flush() return ret
def get_step_state(emr_connection, jobflowid, step_name, update=False): """Return the state of a step. If jobflowid/step_name combination is not unique this will return the state of the most recent step. """ g.reset_caches() steps = get_step_states(emr_connection, jobflowid, _update=update) for name, state in reversed(steps): if name == step_name: return state else: return NOTFOUND
def _callback(msg): if verbose: count_str = "" if "message_count" in msg.delivery_info: # the count from the last message, if the count is # available count_str = "(%d remaining)" % msg.delivery_info["message_count"] print "%s: 1 item %s" % (queue, count_str) g.reset_caches() c.use_write_db = {} ret = callback(msg) msg.channel.basic_ack(msg.delivery_tag) sys.stdout.flush() return ret
def pre(self): c.start_time = datetime.now(g.tz) g.reset_caches() c.domain_prefix = request.environ.get("reddit-domain-prefix", g.domain_prefix) #check if user-agent needs a dose of rate-limiting if not c.error_page: ratelimit_agents() ratelimit_throttled() c.allow_loggedin_cache = False # the domain has to be set before Cookies get initialized set_subreddit() c.errors = ErrorSet() c.cookies = Cookies()
def main(): now = datetime.datetime.now(TIMEZONE) # calculate and store the new day's gold goal determine_gold_goal(now.date()) # post a new thread if we met our revenue goal yesterday = (now - datetime.timedelta(days=1)).date() post_if_goal_reached(yesterday) # look at old (now complete) threads if any activate_requested_names(but_not=yesterday) # wait until all our amqp / permacache changes are flushed from the # in-process queue. worker.join() g.reset_caches() # update the sidebar with a list of names update_sidebar()
def post(self): c.request_timer.intermediate("action") if c.response_wrapper: content = "".join(_force_utf8(x) for x in tup(response.content) if x) wrapped_content = c.response_wrapper(content) response.content = wrapped_content if c.user_is_loggedin and not c.allow_loggedin_cache: response.headers['Cache-Control'] = 'no-cache' response.headers['Pragma'] = 'no-cache' if c.deny_frames: response.headers["X-Frame-Options"] = "DENY" #set content cache if (g.page_cache_time and request.method.upper() == 'GET' and (not c.user_is_loggedin or c.allow_loggedin_cache) and not c.used_cache and response.status_code not in (429, 503)): try: g.pagecache.set(self.request_key(), (response._current_obj(), c.cookies), g.page_cache_time) except MemcachedError as e: # this codepath will actually never be hit as long as # the pagecache memcached client is in no_reply mode. g.log.warning("Ignored exception (%r) on pagecache " "write for %r", e, request.path) # send cookies for k, v in c.cookies.iteritems(): if v.dirty: response.set_cookie(key=k, value=quote(v.value), domain=v.domain, expires=v.expires, secure=getattr(v, 'secure', False), httponly=getattr(v, 'httponly', False)) end_time = datetime.now(g.tz) # update last_visit if (c.user_is_loggedin and not g.disallow_db_writes and request.method.upper() != "POST" and not c.dont_update_last_visit and request.path != '/validuser'): c.user.update_last_visit(c.start_time) check_request(end_time) # this thread is probably going to be reused, but it could be # a while before it is. So we might as well dump the cache in # the mean time so that we don't have dead objects hanging # around taking up memory g.reset_caches() # push data to statsd c.request_timer.stop() g.stats.flush()
def post(self): response = c.response content = filter(None, response.content) if isinstance(content, (list, tuple)): content = ''.join(content) for w in c.response_wrappers: content = w(content) response.content = content if c.response_content_type: response.headers['Content-Type'] = c.response_content_type if c.user_is_loggedin and not c.allow_loggedin_cache: response.headers['Cache-Control'] = 'no-cache' response.headers['Pragma'] = 'no-cache' if c.deny_frames: response.headers["X-Frame-Options"] = "DENY" #return #set content cache if (g.page_cache_time and request.method.upper() == 'GET' and (not c.user_is_loggedin or c.allow_loggedin_cache) and not c.used_cache and response.status_code not in (429, 503) and response.content and response.content[0]): try: g.rendercache.set(self.request_key(), (response, c.cookies), g.page_cache_time) except MemcachedError: # the key was too big to set in the rendercache g.log.debug("Ignored too-big render cache") # send cookies for k, v in c.cookies.iteritems(): if v.dirty: response.set_cookie(key=k, value=quote(v.value), domain=v.domain, expires=v.expires) end_time = datetime.now(g.tz) if ('pylons.routes_dict' in request.environ and 'action' in request.environ['pylons.routes_dict']): action = str(request.environ['pylons.routes_dict']['action']) else: action = "unknown" log_text("unknown action", "no action for %r" % path_info, "warning") if g.usage_sampling >= 1.0 or rand.random() < g.usage_sampling: amqp.add_kw("usage_q", start_time=c.start_time, end_time=end_time, sampling_rate=g.usage_sampling, action=action) check_request(end_time) # this thread is probably going to be reused, but it could be # a while before it is. So we might as well dump the cache in # the mean time so that we don't have dead objects hanging # around taking up memory g.reset_caches() # push data to statsd if 'pylons.action_method' in request.environ: # only report web timing data if an action handler was called g.stats.transact('web.%s' % action, (end_time - c.start_time).total_seconds()) g.stats.flush_timing_stats()
def post(self): c.request_timer.intermediate("action") # if the action raised an HTTPException (i.e. it aborted) then pylons # will have replaced response with the exception itself. c.is_exception_response = getattr(response, "_exception", False) if c.response_wrapper and not c.is_exception_response: content = flatten_response(response.content) wrapped_content = c.response_wrapper(content) response.content = wrapped_content if c.user_is_loggedin and not c.allow_loggedin_cache: response.headers['Cache-Control'] = 'no-cache' response.headers['Pragma'] = 'no-cache' if c.deny_frames: response.headers["X-Frame-Options"] = "DENY" # save the result of this page to the pagecache if possible. we # mustn't cache things that rely on state not tracked by request_key # such as If-Modified-Since headers for 304s or requesting IP for 429s. if (g.page_cache_time and request.method.upper() == 'GET' and c.can_use_pagecache and not c.used_cache and response.status_int not in (304, 429) and not response.status.startswith("5") and not c.is_exception_response): try: g.pagecache.set(self.request_key(), (response._current_obj(), c.cookies), g.page_cache_time) except MemcachedError as e: # this codepath will actually never be hit as long as # the pagecache memcached client is in no_reply mode. g.log.warning( "Ignored exception (%r) on pagecache " "write for %r", e, request.path) pragmas = [ p.strip() for p in request.headers.get("Pragma", "").split(",") ] if g.debug or "x-reddit-pagecache" in pragmas: if c.can_use_pagecache: pagecache_state = "hit" if c.used_cache else "miss" else: pagecache_state = "disallowed" response.headers["X-Reddit-Pagecache"] = pagecache_state # send cookies for k, v in c.cookies.iteritems(): if v.dirty: response.set_cookie(key=k, value=quote(v.value), domain=v.domain, expires=v.expires, secure=getattr(v, 'secure', False), httponly=getattr(v, 'httponly', False)) if self.should_update_last_visit(): c.user.update_last_visit(c.start_time) hooks.get_hook("reddit.request.end").call() # this thread is probably going to be reused, but it could be # a while before it is. So we might as well dump the cache in # the mean time so that we don't have dead objects hanging # around taking up memory g.reset_caches() c.request_timer.intermediate("post") # push data to statsd c.request_timer.stop() g.stats.flush()
def describe_jobflows_by_ids(emr_connection, jobflow_ids, _update=False): g.reset_caches() jobflows = describe_jobflows_cached(emr_connection, _update=_update) return [jf for jf in jobflows if jf.jobflowid in jobflow_ids]
def post(self): response = c.response content = filter(None, response.content) if isinstance(content, (list, tuple)): content = "".join(content) for w in c.response_wrappers: content = w(content) response.content = content if c.response_content_type: response.headers["Content-Type"] = c.response_content_type if c.user_is_loggedin and not c.allow_loggedin_cache: response.headers["Cache-Control"] = "no-cache" response.headers["Pragma"] = "no-cache" if c.deny_frames: response.headers["X-Frame-Options"] = "DENY" # return # set content cache if ( g.page_cache_time and request.method.upper() == "GET" and (not c.user_is_loggedin or c.allow_loggedin_cache) and not c.used_cache and response.status_code not in (429, 503) and response.content and response.content[0] ): try: g.rendercache.set(self.request_key(), (response, c.cookies), g.page_cache_time) except MemcachedError as e: # this codepath will actually never be hit as long as # the pagecache memcached client is in no_reply mode. g.log.warning("Ignored exception (%r) on pagecache " "write for %r", e, request.path) # send cookies for k, v in c.cookies.iteritems(): if v.dirty: response.set_cookie( key=k, value=quote(v.value), domain=v.domain, expires=v.expires, secure=getattr(v, "secure", False), httponly=getattr(v, "httponly", False), ) end_time = datetime.now(g.tz) # update last_visit if c.user_is_loggedin and not g.disallow_db_writes and request.path != "/validuser": c.user.update_last_visit(c.start_time) if "pylons.routes_dict" in request.environ and "action" in request.environ["pylons.routes_dict"]: action = str(request.environ["pylons.routes_dict"]["action"]) else: action = "unknown" log_text("unknown action", "no action for %r" % path_info, "warning") if g.usage_sampling >= 1.0 or rand.random() < g.usage_sampling: amqp.add_kw( "usage_q", start_time=c.start_time, end_time=end_time, sampling_rate=g.usage_sampling, action=action ) check_request(end_time) # this thread is probably going to be reused, but it could be # a while before it is. So we might as well dump the cache in # the mean time so that we don't have dead objects hanging # around taking up memory g.reset_caches() # push data to statsd if "pylons.action_method" in request.environ: # only report web timing data if an action handler was called g.stats.transact("web.%s" % action, (end_time - c.start_time).total_seconds()) g.stats.flush()
def describe_jobflows(emr_connection, _update=False): g.reset_caches() jobflows = describe_jobflows_cached(emr_connection, _update=_update) return jobflows
return True resume_id = long(sys.argv[1]) if len(sys.argv) > 1 else None msg_accounts = Account._query(sort=desc("_date"), data=True) if resume_id: msg_accounts._filter(Account.c._id < resume_id) for account in progress(fetch_things2(msg_accounts), estimate=resume_id): current_inbox_count = account.inbox_count unread_messages = list(queries.get_unread_inbox(account)) if account._id % 100000 == 0: g.reset_caches() if not len(unread_messages): if current_inbox_count: account._incr('inbox_count', -current_inbox_count) else: msgs = Message._by_fullname( unread_messages, data=True, return_dict=False, ignore_missing=True, ) kept_msgs = sum(1 for msg in msgs if _keep(msg, account)) if kept_msgs or current_inbox_count: account._incr('inbox_count', kept_msgs - current_inbox_count)
def handle_items(queue, callback, ack=True, limit=1, min_size=0, drain=False, verbose=True, sleep_time=1): """Call callback() on every item in a particular queue. If the connection to the queue is lost, it will die. Intended to be used as a long-running process.""" if limit < min_size: raise ValueError("min_size must be less than limit") from pylons import c from raven import Client if "sentry_dsn" in g.config: raven_client = Client(g.config["sentry_dsn"]) else: raven_client = None chan = connection_manager.get_channel() countdown = None while True: # NB: None != 0, so we don't need an "is not None" check here if countdown == 0: break msg = chan.basic_get(queue) if not msg and drain: return elif not msg: time.sleep(sleep_time) continue if countdown is None and drain and "message_count" in msg.delivery_info: countdown = 1 + msg.delivery_info["message_count"] g.reset_caches() c.use_write_db = {} items = [msg] while countdown != 0: if countdown is not None: countdown -= 1 if len(items) >= limit: break # the innermost loop only msg = chan.basic_get(queue) if msg is None: if len(items) < min_size: time.sleep(sleep_time) else: break else: items.append(msg) try: count_str = "" if "message_count" in items[-1].delivery_info: # the count from the last message, if the count is # available count_str = "(%d remaining)" % items[-1].delivery_info["message_count"] if verbose: print "%s: %d items %s" % (queue, len(items), count_str) callback(items, chan) if ack: # ack *all* outstanding messages chan.basic_ack(ALL_READ_MESSAGES, multiple=True) # flush any log messages printed by the callback sys.stdout.flush() except: if raven_client: raven_client.captureException() for item in items: # explicitly reject the items that we've not processed chan.basic_reject(item.delivery_tag, requeue=True) raise
def pushup_permacache(verbosity=1000): """When putting cassandra into the permacache chain, we need to push everything up into the rest of the chain, so this is everything that uses the permacache, as of that check-in.""" from pylons import g from r2.models import Link, Subreddit, Account from r2.lib.db.operators import desc from r2.lib.comment_tree import comments_key, messages_key from r2.lib.utils import fetch_things2, in_chunks from r2.lib.utils import last_modified_key from r2.lib.promote import promoted_memo_key from r2.lib.subreddit_search import load_all_reddits from r2.lib.db import queries from r2.lib.cache import CassandraCacheChain authority = g.permacache.caches[-1] nonauthority = CassandraCacheChain(g.permacache.caches[1:-1]) def populate(keys): vals = authority.simple_get_multi(keys) if vals: nonauthority.set_multi(vals) def gen_keys(): yield promoted_memo_key # just let this one do its own writing load_all_reddits() yield queries.get_all_comments().iden l_q = Link._query(Link.c._spam == (True, False), Link.c._deleted == (True, False), sort=desc('_date'), data=True, ) for link in fetch_things2(l_q, verbosity): yield comments_key(link._id) yield last_modified_key(link, 'comments') a_q = Account._query(Account.c._spam == (True, False), sort=desc('_date'), ) for account in fetch_things2(a_q, verbosity): yield messages_key(account._id) yield last_modified_key(account, 'overview') yield last_modified_key(account, 'commented') yield last_modified_key(account, 'submitted') yield last_modified_key(account, 'liked') yield last_modified_key(account, 'disliked') yield queries.get_comments(account, 'new', 'all').iden yield queries.get_submitted(account, 'new', 'all').iden yield queries.get_liked(account).iden yield queries.get_disliked(account).iden yield queries.get_hidden(account).iden yield queries.get_saved(account).iden yield queries.get_inbox_messages(account).iden yield queries.get_unread_messages(account).iden yield queries.get_inbox_comments(account).iden yield queries.get_unread_comments(account).iden yield queries.get_inbox_selfreply(account).iden yield queries.get_unread_selfreply(account).iden yield queries.get_sent(account).iden sr_q = Subreddit._query(Subreddit.c._spam == (True, False), sort=desc('_date'), ) for sr in fetch_things2(sr_q, verbosity): yield last_modified_key(sr, 'stylesheet_contents') yield queries.get_links(sr, 'hot', 'all').iden yield queries.get_links(sr, 'new', 'all').iden for sort in 'top', 'controversial': for time in 'hour', 'day', 'week', 'month', 'year', 'all': yield queries.get_links(sr, sort, time, merge_batched=False).iden yield queries.get_spam_links(sr).iden yield queries.get_spam_comments(sr).iden yield queries.get_reported_links(sr).iden yield queries.get_reported_comments(sr).iden yield queries.get_subreddit_messages(sr).iden yield queries.get_unread_subreddit_messages(sr).iden done = 0 for keys in in_chunks(gen_keys(), verbosity): g.reset_caches() done += len(keys) print 'Done %d: %r' % (done, keys[-1]) populate(keys)
def post(self): response = c.response content = filter(None, response.content) if isinstance(content, (list, tuple)): content = ''.join(content) for w in c.response_wrappers: content = w(content) response.content = content if c.response_content_type: response.headers['Content-Type'] = c.response_content_type if c.user_is_loggedin and not c.allow_loggedin_cache: response.headers['Cache-Control'] = 'no-cache' response.headers['Pragma'] = 'no-cache' if c.deny_frames: response.headers["X-Frame-Options"] = "DENY" #return #set content cache if (g.page_cache_time and request.method.upper() == 'GET' and (not c.user_is_loggedin or c.allow_loggedin_cache) and not c.used_cache and response.status_code not in (429, 503) and response.content and response.content[0]): try: g.rendercache.set(self.request_key(), (response, c.cookies), g.page_cache_time) except MemcachedError: # the key was too big to set in the rendercache g.log.debug("Ignored too-big render cache") # send cookies for k,v in c.cookies.iteritems(): if v.dirty: response.set_cookie(key = k, value = quote(v.value), domain = v.domain, expires = v.expires) end_time = datetime.now(g.tz) if ('pylons.routes_dict' in request.environ and 'action' in request.environ['pylons.routes_dict']): action = str(request.environ['pylons.routes_dict']['action']) else: action = "unknown" log_text("unknown action", "no action for %r" % path_info, "warning") if g.usage_sampling >= 1.0 or rand.random() < g.usage_sampling: amqp.add_kw("usage_q", start_time = c.start_time, end_time = end_time, sampling_rate = g.usage_sampling, action = action) check_request(end_time) # this thread is probably going to be reused, but it could be # a while before it is. So we might as well dump the cache in # the mean time so that we don't have dead objects hanging # around taking up memory g.reset_caches() # push data to statsd if 'pylons.action_method' in request.environ: # only report web timing data if an action handler was called g.stats.transact('web.%s' % action, (end_time - c.start_time).total_seconds()) g.stats.flush_timing_stats()
def post(self): c.request_timer.intermediate("action") response = c.response content = filter(None, response.content) if isinstance(content, (list, tuple)): content = ''.join(content) for w in c.response_wrappers: content = w(content) response.content = content if c.response_content_type: response.headers['Content-Type'] = c.response_content_type if c.user_is_loggedin and not c.allow_loggedin_cache: response.headers['Cache-Control'] = 'no-cache' response.headers['Pragma'] = 'no-cache' if c.deny_frames: response.headers["X-Frame-Options"] = "DENY" #return #set content cache if (g.page_cache_time and request.method.upper() == 'GET' and (not c.user_is_loggedin or c.allow_loggedin_cache) and not c.used_cache and response.status_code not in (429, 503) and response.content and response.content[0]): try: g.rendercache.set(self.request_key(), (response, c.cookies), g.page_cache_time) except MemcachedError as e: # this codepath will actually never be hit as long as # the pagecache memcached client is in no_reply mode. g.log.warning( "Ignored exception (%r) on pagecache " "write for %r", e, request.path) # send cookies for k, v in c.cookies.iteritems(): if v.dirty: response.set_cookie(key=k, value=quote(v.value), domain=v.domain, expires=v.expires, secure=getattr(v, 'secure', False), httponly=getattr(v, 'httponly', False)) end_time = datetime.now(g.tz) # update last_visit if (c.user_is_loggedin and not g.disallow_db_writes and request.method.upper() != "POST" and not c.dont_update_last_visit and request.path != '/validuser'): c.user.update_last_visit(c.start_time) check_request(end_time) # this thread is probably going to be reused, but it could be # a while before it is. So we might as well dump the cache in # the mean time so that we don't have dead objects hanging # around taking up memory g.reset_caches() # push data to statsd c.request_timer.stop() g.stats.flush()
def post(self): response = c.response content = filter(None, response.content) if isinstance(content, (list, tuple)): content = ''.join(content) for w in c.response_wrappers: content = w(content) response.content = content if c.response_content_type: response.headers['Content-Type'] = c.response_content_type if c.response_access_control: c.response.headers['Access-Control'] = c.response_access_control if c.user_is_loggedin and not c.allow_loggedin_cache: response.headers['Cache-Control'] = 'no-cache' response.headers['Pragma'] = 'no-cache' #return #set content cache if (g.page_cache_time and request.method.upper() == 'GET' and (not c.user_is_loggedin or c.allow_loggedin_cache) and not c.used_cache and response.status_code != 503 and response.content and response.content[0]): try: g.rendercache.set(self.request_key(), (response, c.cookies), g.page_cache_time) except MemcachedError: # the key was too big to set in the rendercache g.log.debug("Ignored too-big render cache") # send cookies for k,v in c.cookies.iteritems(): if v.dirty: response.set_cookie(key = k, value = quote(v.value), domain = v.domain, expires = v.expires) if g.usage_sampling <= 0.0: return if g.usage_sampling >= 1.0 or rand.random() < g.usage_sampling: if ('pylons.routes_dict' in request.environ and 'action' in request.environ['pylons.routes_dict']): action = str(request.environ['pylons.routes_dict']['action']) else: action = "unknown" log_text("unknown action", "no action for %r" % path_info, "warning") amqp.add_kw("usage_q", start_time = c.start_time, end_time = datetime.now(g.tz), sampling_rate = g.usage_sampling, action = action) # this thread is probably going to be reused, but it could be # a while before it is. So we might as well dump the cache in # the mean time so that we don't have dead objects hanging # around taking up memory g.reset_caches()
def update_timer(): while True: g.reset_caches() _update_timer() sleep(UPDATE_INTERVAL_SECONDS)
def describe_jobflows_by_state(emr_connection, states, _update=False): g.reset_caches() jobflows = describe_jobflows_cached(emr_connection, _update=_update) return [jf for jf in jobflows if jf.state in states]
def post(self): c.request_timer.intermediate("action") # if the action raised an HTTPException (i.e. it aborted) then pylons # will have replaced response with the exception itself. c.is_exception_response = getattr(response, "_exception", False) if c.response_wrapper and not c.is_exception_response: content = flatten_response(response.content) wrapped_content = c.response_wrapper(content) response.content = wrapped_content if c.user_is_loggedin and not c.allow_loggedin_cache: response.headers['Cache-Control'] = 'no-cache' response.headers['Pragma'] = 'no-cache' if c.deny_frames: response.headers["X-Frame-Options"] = "DENY" # save the result of this page to the pagecache if possible. we # mustn't cache things that rely on state not tracked by request_key # such as If-Modified-Since headers for 304s or requesting IP for 429s. if (g.page_cache_time and request.method.upper() == 'GET' and c.can_use_pagecache and not c.used_cache and response.status_int not in (304, 429) and not response.status.startswith("5") and not c.is_exception_response): try: g.pagecache.set(self.request_key(), (response._current_obj(), c.cookies), g.page_cache_time) except MemcachedError as e: # this codepath will actually never be hit as long as # the pagecache memcached client is in no_reply mode. g.log.warning("Ignored exception (%r) on pagecache " "write for %r", e, request.path) pragmas = [p.strip() for p in request.headers.get("Pragma", "").split(",")] if g.debug or "x-reddit-pagecache" in pragmas: if c.can_use_pagecache: pagecache_state = "hit" if c.used_cache else "miss" else: pagecache_state = "disallowed" response.headers["X-Reddit-Pagecache"] = pagecache_state # send cookies for k, v in c.cookies.iteritems(): if v.dirty: response.set_cookie(key=k, value=quote(v.value), domain=v.domain, expires=v.expires, secure=getattr(v, 'secure', False), httponly=getattr(v, 'httponly', False)) if self.should_update_last_visit(): c.user.update_last_visit(c.start_time) hooks.get_hook("reddit.request.end").call() # this thread is probably going to be reused, but it could be # a while before it is. So we might as well dump the cache in # the mean time so that we don't have dead objects hanging # around taking up memory g.reset_caches() c.request_timer.intermediate("post") # push data to statsd c.request_timer.stop() g.stats.flush()
def post(self): response = c.response content = filter(None, response.content) if isinstance(content, (list, tuple)): content = ''.join(content) for w in c.response_wrappers: content = w(content) response.content = content if c.response_content_type: response.headers['Content-Type'] = c.response_content_type if c.response_access_control: c.response.headers['Access-Control'] = c.response_access_control if c.user_is_loggedin and not c.allow_loggedin_cache: response.headers['Cache-Control'] = 'no-cache' response.headers['Pragma'] = 'no-cache' #return #set content cache if (g.page_cache_time and request.method.upper() == 'GET' and (not c.user_is_loggedin or c.allow_loggedin_cache) and not c.used_cache and response.status_code != 503 and response.content and response.content[0]): try: g.rendercache.set(self.request_key(), (response, c.cookies), g.page_cache_time) except MemcachedError: # the key was too big to set in the rendercache g.log.debug("Ignored too-big render cache") # send cookies for k,v in c.cookies.iteritems(): if v.dirty: response.set_cookie(key = k, value = quote(v.value), domain = v.domain, expires = v.expires) if g.logans_run_limit: if c.start_time > g.logans_run_limit and not g.shutdown: g.log.info("Time to restart. It's been an honor serving with you.") g.shutdown = 'init' if g.usage_sampling <= 0.0: return if g.usage_sampling >= 1.0 or rand.random() < g.usage_sampling: if ('pylons.routes_dict' in request.environ and 'action' in request.environ['pylons.routes_dict']): action = str(request.environ['pylons.routes_dict']['action']) else: action = "unknown" log_text("unknown action", "no action for %r" % path_info, "warning") amqp.add_kw("usage_q", start_time = c.start_time, end_time = datetime.now(g.tz), sampling_rate = g.usage_sampling, action = action) # this thread is probably going to be reused, but it could be # a while before it is. So we might as well dump the cache in # the mean time so that we don't have dead objects hanging # around taking up memory g.reset_caches()
def post(self): c.request_timer.intermediate("action") # if the action raised an HTTPException (i.e. it aborted) then pylons # will have replaced response with the exception itself. c.is_exception_response = getattr(response, "_exception", False) if c.response_wrapper and not c.is_exception_response: content = flatten_response(response.content) wrapped_content = c.response_wrapper(content) response.content = wrapped_content if c.user_is_loggedin and not c.allow_loggedin_cache: response.headers['Cache-Control'] = 'no-cache' response.headers['Pragma'] = 'no-cache' if c.deny_frames: response.headers["X-Frame-Options"] = "DENY" #set content cache if (g.page_cache_time and request.method.upper() == 'GET' and (not c.user_is_loggedin or c.allow_loggedin_cache) and not c.used_cache and response.status_int != 429 and not response.status.startswith("5") and not c.is_exception_response): try: g.pagecache.set(self.request_key(), (response._current_obj(), c.cookies), g.page_cache_time) except MemcachedError as e: # this codepath will actually never be hit as long as # the pagecache memcached client is in no_reply mode. g.log.warning("Ignored exception (%r) on pagecache " "write for %r", e, request.path) # send cookies for k, v in c.cookies.iteritems(): if v.dirty: response.set_cookie(key=k, value=quote(v.value), domain=v.domain, expires=v.expires, secure=getattr(v, 'secure', False), httponly=getattr(v, 'httponly', False)) if self.should_update_last_visit(): c.user.update_last_visit(c.start_time) hooks.get_hook("reddit.request.end").call() # this thread is probably going to be reused, but it could be # a while before it is. So we might as well dump the cache in # the mean time so that we don't have dead objects hanging # around taking up memory g.reset_caches() c.request_timer.intermediate("post") # push data to statsd c.request_timer.stop() g.stats.flush()
def batch_fn(items): g.reset_caches() return items
def post(self): response = c.response content = filter(None, response.content) if isinstance(content, (list, tuple)): content = "".join(content) for w in c.response_wrappers: content = w(content) response.content = content if c.response_content_type: response.headers["Content-Type"] = c.response_content_type if c.user_is_loggedin and not c.allow_loggedin_cache: response.headers["Cache-Control"] = "no-cache" response.headers["Pragma"] = "no-cache" if c.deny_frames: response.headers["X-Frame-Options"] = "DENY" # return # set content cache if ( g.page_cache_time and request.method.upper() == "GET" and (not c.user_is_loggedin or c.allow_loggedin_cache) and not c.used_cache and response.status_code != 503 and response.content and response.content[0] ): try: g.rendercache.set(self.request_key(), (response, c.cookies), g.page_cache_time) except MemcachedError: # the key was too big to set in the rendercache g.log.debug("Ignored too-big render cache") # send cookies for k, v in c.cookies.iteritems(): if v.dirty: response.set_cookie(key=k, value=quote(v.value), domain=v.domain, expires=v.expires) if g.usage_sampling <= 0.0: return if g.usage_sampling >= 1.0 or rand.random() < g.usage_sampling: if "pylons.routes_dict" in request.environ and "action" in request.environ["pylons.routes_dict"]: action = str(request.environ["pylons.routes_dict"]["action"]) else: action = "unknown" log_text("unknown action", "no action for %r" % path_info, "warning") amqp.add_kw( "usage_q", start_time=c.start_time, end_time=datetime.now(g.tz), sampling_rate=g.usage_sampling, action=action, ) # this thread is probably going to be reused, but it could be # a while before it is. So we might as well dump the cache in # the mean time so that we don't have dead objects hanging # around taking up memory g.reset_caches()