def get_step_state(emr_connection, jobflowid, step_name, update=False):
"""Return the state of a step.
If jobflowid/step_name combination is not unique this will return the state
of the most recent step.
"""
g.reset_caches()
steps = get_step_states(emr_connection, jobflowid, _update=update)
for name, state, start in sorted(steps, key=lambda t: t[2], reverse=True):
if name == step_name:
return state
else:
return NOTFOUND
def get_step_state(emr_connection, jobflowid, step_name, update=False):
"""Return the state of a step.
If jobflowid/step_name combination is not unique this will return the state
of the most recent step.
"""
g.reset_caches()
steps = get_step_states(emr_connection, jobflowid, _update=update)
for name, state, start in sorted(steps, key=lambda t: t[2], reverse=True):
if name == step_name:
return state
else:
return NOTFOUND
def main():
now = datetime.datetime.now(TIMEZONE)
# calculate and store the new day's gold goal
determine_gold_goal(now.date())
# post a new thread if we met our revenue goal
yesterday = (now - datetime.timedelta(days=1)).date()
post_if_goal_reached(yesterday)
# look at old (now complete) threads if any
activate_requested_names(but_not=yesterday)
# wait until all our amqp / permacache changes are flushed from the
# in-process queue.
worker.join()
g.reset_caches()
# update the sidebar with a list of names
update_sidebar()
def main():
now = datetime.datetime.now(TIMEZONE)
# calculate and store the new day's gold goal
determine_gold_goal(now.date())
# post a new thread if we met our revenue goal
yesterday = (now - datetime.timedelta(days=1)).date()
post_if_goal_reached(yesterday)
# look at old (now complete) threads if any
activate_requested_names(but_not=yesterday)
# wait until all our amqp / permacache changes are flushed from the
# in-process queue.
worker.join()
g.reset_caches()
# update the sidebar with a list of names
update_sidebar()
def batch_fn(items):
g.reset_caches()
return items
def pushup_permacache(verbosity=1000):
"""When putting cassandra into the permacache chain, we need to
push everything up into the rest of the chain, so this is
everything that uses the permacache, as of that check-in."""
from pylons import app_globals as g
from r2.models import Link, Subreddit, Account
from r2.lib.db.operators import desc
from r2.lib.comment_tree import comments_key, messages_key
from r2.lib.utils import fetch_things2, in_chunks
from r2.lib.utils import last_modified_key
from r2.lib.promote import promoted_memo_key
from r2.lib.subreddit_search import load_all_reddits
from r2.lib.db import queries
from r2.lib.cache import CassandraCacheChain
authority = g.permacache.caches[-1]
nonauthority = CassandraCacheChain(g.permacache.caches[1:-1])
def populate(keys):
vals = authority.simple_get_multi(keys)
if vals:
nonauthority.set_multi(vals)
def gen_keys():
yield promoted_memo_key
# just let this one do its own writing
load_all_reddits()
yield queries.get_all_comments().iden
l_q = Link._query(
Link.c._spam == (True, False), Link.c._deleted == (True, False), sort=desc("_date"), data=True
)
for link in fetch_things2(l_q, verbosity):
yield comments_key(link._id)
yield last_modified_key(link, "comments")
a_q = Account._query(Account.c._spam == (True, False), sort=desc("_date"))
for account in fetch_things2(a_q, verbosity):
yield messages_key(account._id)
yield last_modified_key(account, "overview")
yield last_modified_key(account, "commented")
yield last_modified_key(account, "submitted")
yield last_modified_key(account, "liked")
yield last_modified_key(account, "disliked")
yield queries.get_comments(account, "new", "all").iden
yield queries.get_submitted(account, "new", "all").iden
yield queries.get_liked(account).iden
yield queries.get_disliked(account).iden
yield queries.get_hidden(account).iden
yield queries.get_saved(account).iden
yield queries.get_inbox_messages(account).iden
yield queries.get_unread_messages(account).iden
yield queries.get_inbox_comments(account).iden
yield queries.get_unread_comments(account).iden
yield queries.get_inbox_selfreply(account).iden
yield queries.get_unread_selfreply(account).iden
yield queries.get_sent(account).iden
sr_q = Subreddit._query(Subreddit.c._spam == (True, False), sort=desc("_date"))
for sr in fetch_things2(sr_q, verbosity):
yield last_modified_key(sr, "stylesheet_contents")
yield queries.get_links(sr, "hot", "all").iden
yield queries.get_links(sr, "new", "all").iden
for sort in "top", "controversial":
for time in "hour", "day", "week", "month", "year", "all":
yield queries.get_links(sr, sort, time, merge_batched=False).iden
yield queries.get_spam_links(sr).iden
yield queries.get_spam_comments(sr).iden
yield queries.get_reported_links(sr).iden
yield queries.get_reported_comments(sr).iden
yield queries.get_subreddit_messages(sr).iden
yield queries.get_unread_subreddit_messages(sr).iden
done = 0
for keys in in_chunks(gen_keys(), verbosity):
g.reset_caches()
done += len(keys)
print "Done %d: %r" % (done, keys[-1])
populate(keys)
def pushup_permacache(verbosity=1000):
"""When putting cassandra into the permacache chain, we need to
push everything up into the rest of the chain, so this is
everything that uses the permacache, as of that check-in."""
from pylons import app_globals as g
from v1.models import Link, Subverbify, Account
from v1.lib.db.operators import desc
from v1.lib.comment_tree import comments_key, messages_key
from v1.lib.utils import fetch_things2, in_chunks
from v1.lib.utils import last_modified_key
from v1.lib.promote import promoted_memo_key
from v1.lib.subverbify_search import load_all_verbifys
from v1.lib.db import queries
from v1.lib.cache import CassandraCacheChain
authority = g.permacache.caches[-1]
nonauthority = CassandraCacheChain(g.permacache.caches[1:-1])
def populate(keys):
vals = authority.simple_get_multi(keys)
if vals:
nonauthority.set_multi(vals)
def gen_keys():
yield promoted_memo_key
# just let this one do its own writing
load_all_verbifys()
yield queries.get_all_comments().iden
l_q = Link._query(
Link.c._spam == (True, False),
Link.c._deleted == (True, False),
sort=desc('_date'),
data=True,
)
for link in fetch_things2(l_q, verbosity):
yield comments_key(link._id)
yield last_modified_key(link, 'comments')
a_q = Account._query(
Account.c._spam == (True, False),
sort=desc('_date'),
)
for account in fetch_things2(a_q, verbosity):
yield messages_key(account._id)
yield last_modified_key(account, 'overview')
yield last_modified_key(account, 'commented')
yield last_modified_key(account, 'submitted')
yield last_modified_key(account, 'liked')
yield last_modified_key(account, 'disliked')
yield queries.get_comments(account, 'new', 'all').iden
yield queries.get_submitted(account, 'new', 'all').iden
yield queries.get_liked(account).iden
yield queries.get_disliked(account).iden
yield queries.get_hidden(account).iden
yield queries.get_saved(account).iden
yield queries.get_inbox_messages(account).iden
yield queries.get_unread_messages(account).iden
yield queries.get_inbox_comments(account).iden
yield queries.get_unread_comments(account).iden
yield queries.get_inbox_selfreply(account).iden
yield queries.get_unread_selfreply(account).iden
yield queries.get_sent(account).iden
sr_q = Subverbify._query(
Subverbify.c._spam == (True, False),
sort=desc('_date'),
)
for sr in fetch_things2(sr_q, verbosity):
yield last_modified_key(sr, 'stylesheet_contents')
yield queries.get_links(sr, 'hot', 'all').iden
yield queries.get_links(sr, 'new', 'all').iden
for sort in 'top', 'controversial':
for time in 'hour', 'day', 'week', 'month', 'year', 'all':
yield queries.get_links(sr,
sort,
time,
merge_batched=False).iden
yield queries.get_spam_links(sr).iden
yield queries.get_spam_comments(sr).iden
yield queries.get_reported_links(sr).iden
yield queries.get_reported_comments(sr).iden
yield queries.get_subverbify_messages(sr).iden
yield queries.get_unread_subverbify_messages(sr).iden
done = 0
for keys in in_chunks(gen_keys(), verbosity):
g.reset_caches()
done += len(keys)
print 'Done %d: %r' % (done, keys[-1])
populate(keys)
def batch_fn(items):
g.reset_caches()
return items
return True
resume_id = long(sys.argv[1]) if len(sys.argv) > 1 else None
msg_accounts = Account._query(sort=desc("_date"), data=True)
if resume_id:
msg_accounts._filter(Account.c._id < resume_id)
for account in progress(fetch_things2(msg_accounts), estimate=resume_id):
current_inbox_count = account.inbox_count
unread_messages = list(queries.get_unread_inbox(account))
if account._id % 100000 == 0:
g.reset_caches()
if not len(unread_messages):
if current_inbox_count:
account._incr('inbox_count', -current_inbox_count)
else:
msgs = Message._by_fullname(
unread_messages,
data=True,
return_dict=False,
ignore_missing=True,
)
kept_msgs = sum(1 for msg in msgs if _keep(msg, account))
if kept_msgs or current_inbox_count:
account._incr('inbox_count', kept_msgs - current_inbox_count)
def update_timer():
while True:
g.reset_caches()
_update_timer()
sleep(UPDATE_INTERVAL_SECONDS)
return True
resume_id = long(sys.argv[1]) if len(sys.argv) > 1 else None
msg_accounts = Account._query(sort=desc("_date"), data=True)
if resume_id:
msg_accounts._filter(Account.c._id < resume_id)
for account in progress(fetch_things2(msg_accounts), estimate=resume_id):
current_inbox_count = account.inbox_count
unread_messages = list(queries.get_unread_inbox(account))
if account._id % 100000 == 0:
g.reset_caches()
if not len(unread_messages):
if current_inbox_count:
account._incr('inbox_count', -current_inbox_count)
else:
msgs = Message._by_fullname(
unread_messages,
data=True,
return_dict=False,
ignore_missing=True,
)
kept_msgs = sum(1 for msg in msgs if _keep(msg, account))
if kept_msgs or current_inbox_count:
account._incr('inbox_count', kept_msgs - current_inbox_count)
