def get_step_state(emr_connection, jobflowid, step_name, update=False): """Return the state of a step. If jobflowid/step_name combination is not unique this will return the state of the most recent step. """ g.reset_caches() steps = get_step_states(emr_connection, jobflowid, _update=update) for name, state, start in sorted(steps, key=lambda t: t[2], reverse=True): if name == step_name: return state else: return NOTFOUND
def main(): now = datetime.datetime.now(TIMEZONE) # calculate and store the new day's gold goal determine_gold_goal(now.date()) # post a new thread if we met our revenue goal yesterday = (now - datetime.timedelta(days=1)).date() post_if_goal_reached(yesterday) # look at old (now complete) threads if any activate_requested_names(but_not=yesterday) # wait until all our amqp / permacache changes are flushed from the # in-process queue. worker.join() g.reset_caches() # update the sidebar with a list of names update_sidebar()
def batch_fn(items): g.reset_caches() return items
def pushup_permacache(verbosity=1000): """When putting cassandra into the permacache chain, we need to push everything up into the rest of the chain, so this is everything that uses the permacache, as of that check-in.""" from pylons import app_globals as g from r2.models import Link, Subreddit, Account from r2.lib.db.operators import desc from r2.lib.comment_tree import comments_key, messages_key from r2.lib.utils import fetch_things2, in_chunks from r2.lib.utils import last_modified_key from r2.lib.promote import promoted_memo_key from r2.lib.subreddit_search import load_all_reddits from r2.lib.db import queries from r2.lib.cache import CassandraCacheChain authority = g.permacache.caches[-1] nonauthority = CassandraCacheChain(g.permacache.caches[1:-1]) def populate(keys): vals = authority.simple_get_multi(keys) if vals: nonauthority.set_multi(vals) def gen_keys(): yield promoted_memo_key # just let this one do its own writing load_all_reddits() yield queries.get_all_comments().iden l_q = Link._query( Link.c._spam == (True, False), Link.c._deleted == (True, False), sort=desc("_date"), data=True ) for link in fetch_things2(l_q, verbosity): yield comments_key(link._id) yield last_modified_key(link, "comments") a_q = Account._query(Account.c._spam == (True, False), sort=desc("_date")) for account in fetch_things2(a_q, verbosity): yield messages_key(account._id) yield last_modified_key(account, "overview") yield last_modified_key(account, "commented") yield last_modified_key(account, "submitted") yield last_modified_key(account, "liked") yield last_modified_key(account, "disliked") yield queries.get_comments(account, "new", "all").iden yield queries.get_submitted(account, "new", "all").iden yield queries.get_liked(account).iden yield queries.get_disliked(account).iden yield queries.get_hidden(account).iden yield queries.get_saved(account).iden yield queries.get_inbox_messages(account).iden yield queries.get_unread_messages(account).iden yield queries.get_inbox_comments(account).iden yield queries.get_unread_comments(account).iden yield queries.get_inbox_selfreply(account).iden yield queries.get_unread_selfreply(account).iden yield queries.get_sent(account).iden sr_q = Subreddit._query(Subreddit.c._spam == (True, False), sort=desc("_date")) for sr in fetch_things2(sr_q, verbosity): yield last_modified_key(sr, "stylesheet_contents") yield queries.get_links(sr, "hot", "all").iden yield queries.get_links(sr, "new", "all").iden for sort in "top", "controversial": for time in "hour", "day", "week", "month", "year", "all": yield queries.get_links(sr, sort, time, merge_batched=False).iden yield queries.get_spam_links(sr).iden yield queries.get_spam_comments(sr).iden yield queries.get_reported_links(sr).iden yield queries.get_reported_comments(sr).iden yield queries.get_subreddit_messages(sr).iden yield queries.get_unread_subreddit_messages(sr).iden done = 0 for keys in in_chunks(gen_keys(), verbosity): g.reset_caches() done += len(keys) print "Done %d: %r" % (done, keys[-1]) populate(keys)
def pushup_permacache(verbosity=1000): """When putting cassandra into the permacache chain, we need to push everything up into the rest of the chain, so this is everything that uses the permacache, as of that check-in.""" from pylons import app_globals as g from v1.models import Link, Subverbify, Account from v1.lib.db.operators import desc from v1.lib.comment_tree import comments_key, messages_key from v1.lib.utils import fetch_things2, in_chunks from v1.lib.utils import last_modified_key from v1.lib.promote import promoted_memo_key from v1.lib.subverbify_search import load_all_verbifys from v1.lib.db import queries from v1.lib.cache import CassandraCacheChain authority = g.permacache.caches[-1] nonauthority = CassandraCacheChain(g.permacache.caches[1:-1]) def populate(keys): vals = authority.simple_get_multi(keys) if vals: nonauthority.set_multi(vals) def gen_keys(): yield promoted_memo_key # just let this one do its own writing load_all_verbifys() yield queries.get_all_comments().iden l_q = Link._query( Link.c._spam == (True, False), Link.c._deleted == (True, False), sort=desc('_date'), data=True, ) for link in fetch_things2(l_q, verbosity): yield comments_key(link._id) yield last_modified_key(link, 'comments') a_q = Account._query( Account.c._spam == (True, False), sort=desc('_date'), ) for account in fetch_things2(a_q, verbosity): yield messages_key(account._id) yield last_modified_key(account, 'overview') yield last_modified_key(account, 'commented') yield last_modified_key(account, 'submitted') yield last_modified_key(account, 'liked') yield last_modified_key(account, 'disliked') yield queries.get_comments(account, 'new', 'all').iden yield queries.get_submitted(account, 'new', 'all').iden yield queries.get_liked(account).iden yield queries.get_disliked(account).iden yield queries.get_hidden(account).iden yield queries.get_saved(account).iden yield queries.get_inbox_messages(account).iden yield queries.get_unread_messages(account).iden yield queries.get_inbox_comments(account).iden yield queries.get_unread_comments(account).iden yield queries.get_inbox_selfreply(account).iden yield queries.get_unread_selfreply(account).iden yield queries.get_sent(account).iden sr_q = Subverbify._query( Subverbify.c._spam == (True, False), sort=desc('_date'), ) for sr in fetch_things2(sr_q, verbosity): yield last_modified_key(sr, 'stylesheet_contents') yield queries.get_links(sr, 'hot', 'all').iden yield queries.get_links(sr, 'new', 'all').iden for sort in 'top', 'controversial': for time in 'hour', 'day', 'week', 'month', 'year', 'all': yield queries.get_links(sr, sort, time, merge_batched=False).iden yield queries.get_spam_links(sr).iden yield queries.get_spam_comments(sr).iden yield queries.get_reported_links(sr).iden yield queries.get_reported_comments(sr).iden yield queries.get_subverbify_messages(sr).iden yield queries.get_unread_subverbify_messages(sr).iden done = 0 for keys in in_chunks(gen_keys(), verbosity): g.reset_caches() done += len(keys) print 'Done %d: %r' % (done, keys[-1]) populate(keys)
return True resume_id = long(sys.argv[1]) if len(sys.argv) > 1 else None msg_accounts = Account._query(sort=desc("_date"), data=True) if resume_id: msg_accounts._filter(Account.c._id < resume_id) for account in progress(fetch_things2(msg_accounts), estimate=resume_id): current_inbox_count = account.inbox_count unread_messages = list(queries.get_unread_inbox(account)) if account._id % 100000 == 0: g.reset_caches() if not len(unread_messages): if current_inbox_count: account._incr('inbox_count', -current_inbox_count) else: msgs = Message._by_fullname( unread_messages, data=True, return_dict=False, ignore_missing=True, ) kept_msgs = sum(1 for msg in msgs if _keep(msg, account)) if kept_msgs or current_inbox_count: account._incr('inbox_count', kept_msgs - current_inbox_count)
def update_timer(): while True: g.reset_caches() _update_timer() sleep(UPDATE_INTERVAL_SECONDS)