def recalculate_karma_points():
"""Go through all karma action data and recalculate points."""
if not waffle.switch_is_active('karma'):
return
for chunk in chunked(list(KarmaManager().user_ids()), 2500):
_process_recalculate_chunk.apply_async(args=[chunk])
def auto_lock_old_questions():
"""Locks all questions that were created over 180 days ago"""
# Set up logging so it doesn't send Ricky email.
logging.basicConfig(level=logging.ERROR)
# Get a list of ids of questions we're going to go change. We need
# a list of ids so that we can feed it to the update, but then
# also know what we need to update in the index.
days_180 = datetime.now() - timedelta(days=180)
q_ids = list(
Question.objects.filter(is_locked=False).filter(
created__lte=days_180).values_list('id', flat=True))
if q_ids:
log.info('Updating %d questions', len(q_ids))
sql = """
UPDATE questions_question
SET is_locked = 1
WHERE id IN (%s)
""" % ','.join(map(str, q_ids))
cursor = connection.cursor()
cursor.execute(sql)
transaction.commit_unless_managed()
if settings.ES_LIVE_INDEXING:
try:
# So... the first time this runs, it'll handle 160K
# questions or so which stresses everything. Thus we
# do it in chunks because otherwise this won't work.
#
# After we've done this for the first time, we can nix
# the chunking code.
from search.utils import chunked
for chunk in chunked(q_ids, 100):
# Fetch all the documents we need to update.
es_docs = get_documents(QuestionMappingType, chunk)
log.info('Updating %d index documents', len(es_docs))
documents = []
# For each document, update the data and stick it
# back in the index.
for doc in es_docs:
doc[u'question_is_locked'] = True
doc[u'indexed_on'] = int(time.time())
documents.append(doc)
QuestionMappingType.bulk_index(documents,
id_field='document_id')
except ES_EXCEPTIONS:
# Something happened with ES, so let's push index
# updating into an index_task which retries when it
# fails because of ES issues.
index_task.delay(QuestionMappingType, q_ids)
def auto_lock_old_questions():
"""Locks all questions that were created over 180 days ago"""
# Set up logging so it doesn't send Ricky email.
logging.basicConfig(level=logging.ERROR)
# Get a list of ids of questions we're going to go change. We need
# a list of ids so that we can feed it to the update, but then
# also know what we need to update in the index.
days_180 = datetime.now() - timedelta(days=180)
q_ids = list(Question.objects.filter(is_locked=False)
.filter(created__lte=days_180)
.values_list('id', flat=True))
if q_ids:
log.info('Updating %d questions', len(q_ids))
sql = """
UPDATE questions_question
SET is_locked = 1
WHERE id IN (%s)
""" % ','.join(map(str, q_ids))
cursor = connection.cursor()
cursor.execute(sql)
transaction.commit_unless_managed()
if settings.ES_LIVE_INDEXING:
try:
# So... the first time this runs, it'll handle 160K
# questions or so which stresses everything. Thus we
# do it in chunks because otherwise this won't work.
#
# After we've done this for the first time, we can nix
# the chunking code.
from search.utils import chunked
for chunk in chunked(q_ids, 100):
# Fetch all the documents we need to update.
es_docs = get_documents(QuestionMappingType, chunk)
log.info('Updating %d index documents', len(es_docs))
documents = []
# For each document, update the data and stick it
# back in the index.
for doc in es_docs:
doc[u'question_is_locked'] = True
doc[u'indexed_on'] = int(time.time())
documents.append(doc)
QuestionMappingType.bulk_index(
documents, id_field='document_id')
except ES_EXCEPTIONS:
# Something happened with ES, so let's push index
# updating into an index_task which retries when it
# fails because of ES issues.
index_task.delay(QuestionMappingType, q_ids)
def rebuild_kb():
"""Re-render all documents in the KB in chunks."""
cache.delete(settings.WIKI_REBUILD_TOKEN)
d = Document.objects.using("default").filter(current_revision__isnull=False).values_list("id", flat=True)
for chunk in chunked(d, 100):
_rebuild_kb_chunk.apply_async(args=[chunk])
def init_karma():
"""Flushes the karma redis backend and populates with fresh data.
Goes through all questions/answers/votes and save karma actions for them.
"""
if not waffle.switch_is_active('karma'):
return
redis_client('karma').flushdb()
questions = Question.objects.all()
for chunk in chunked(questions.values_list('pk', flat=True), 200):
_process_question_chunk.apply_async(args=[chunk])
votes = AnswerVote.objects.all()
for chunk in chunked(votes.values_list('pk', flat=True), 1000):
_process_answer_vote_chunk.apply_async(args=[chunk])
def recalculate_karma_points():
"""Go through all karma action data and recalculate points."""
if not waffle.switch_is_active("karma"):
return
qs = User.objects.filter(is_active=True).values_list("id", flat=True)
for chunk in chunked(qs, 2500):
_process_recalculate_chunk.apply_async(args=[chunk])
def rebuild_kb():
"""Re-render all documents in the KB in chunks."""
cache.delete(settings.WIKI_REBUILD_TOKEN)
d = (Document.objects.using('default').filter(
current_revision__isnull=False).values_list('id', flat=True))
for chunk in chunked(d, 100):
_rebuild_kb_chunk.apply_async(args=[chunk])
def update_weekly_votes():
"""Keep the num_votes_past_week value accurate."""
questions = Question.objects.all().values_list('pk', flat=True)
with establish_connection() as conn:
for chunk in chunked(questions, 200):
update_question_vote_chunk.apply_async(args=[chunk],
connection=conn)
def update_weekly_votes():
"""Keep the num_votes_past_week value accurate."""
recent = datetime.now() - timedelta(days=14)
q = QuestionVote.objects.filter(created__gte=recent)
q = q.values_list('question_id', flat=True).order_by('question')
q = q.distinct()
for chunk in chunked(q, 50):
update_question_vote_chunk.apply_async(args=[chunk])
def rebuild_kb():
"""Re-render all documents in the KB in chunks."""
cache.delete(settings.WIKI_REBUILD_TOKEN)
d = (Document.objects.using('default')
.filter(current_revision__isnull=False).values_list('id', flat=True))
with establish_connection() as conn:
for chunk in chunked(d, 100):
_rebuild_kb_chunk.apply_async(args=[chunk],
connection=conn)
def update_weekly_votes():
"""Keep the num_votes_past_week value accurate."""
recent = datetime.now() - timedelta(days=14)
q = QuestionVote.objects.filter(created__gte=recent)
q = q.values_list("question_id", flat=True).order_by("question")
q = q.distinct()
for chunk in chunked(q, 50):
update_question_vote_chunk.apply_async(args=[chunk])
def delete_expired_users(self):
"""
Remove expired instances of this manager's object class.
Accounts to be deleted are identified by searching for
instances of this manager's object class with expired activation
keys, and then checking to see if their associated ``User``
instances have the field ``is_active`` set to ``False``; any
``User`` who is both inactive and has an expired activation
key will be deleted.
"""
days_valid = settings.ACCOUNT_ACTIVATION_DAYS
expired = datetime.now() - timedelta(days=days_valid)
prof_ids = self.filter(user__date_joined__lt=expired)
prof_ids = prof_ids.values_list('id', flat=True)
for chunk in chunked(prof_ids, 1000):
_delete_registration_profiles_chunk.apply_async(args=[chunk])
def delete_expired_users(self):
"""
Remove expired instances of this manager's object class.
Accounts to be deleted are identified by searching for
instances of this manager's object class with expired activation
keys, and then checking to see if their associated ``User``
instances have the field ``is_active`` set to ``False``; any
``User`` who is both inactive and has an expired activation
key will be deleted.
"""
days_valid = settings.ACCOUNT_ACTIVATION_DAYS
expired = datetime.now() - timedelta(days=days_valid)
prof_ids = self.filter(user__date_joined__lt=expired)
prof_ids = prof_ids.values_list('id', flat=True)
for chunk in chunked(prof_ids, 1000):
_delete_registration_profiles_chunk.apply_async(args=[chunk])
def update_weekly_votes():
"""Keep the num_votes_past_week value accurate."""
# Get all questions (id) with a vote in the last week.
recent = datetime.now() - timedelta(days=7)
q = QuestionVote.objects.filter(created__gte=recent)
q = q.values_list('question_id', flat=True).order_by('question')
q = q.distinct()
q_with_recent_votes = list(q)
# Get all questions with num_votes_past_week > 0
q = Question.objects.filter(num_votes_past_week__gt=0)
q = q.values_list('id', flat=True)
q_with_nonzero_votes = list(q)
# Union.
qs_to_update = list(set(q_with_recent_votes + q_with_nonzero_votes))
# Chunk them for tasks.
for chunk in chunked(qs_to_update, 50):
update_question_vote_chunk.apply_async(args=[chunk])
def update_weekly_votes():
"""Keep the num_votes_past_week value accurate."""
# Get all questions (id) with a vote in the last week.
recent = datetime.now() - timedelta(days=7)
q = QuestionVote.objects.filter(created__gte=recent)
q = q.values_list('question_id', flat=True).order_by('question')
q = q.distinct()
q_with_recent_votes = list(q)
# Get all questions with num_votes_past_week > 0
q = Question.objects.filter(num_votes_past_week__gt=0)
q = q.values_list('id', flat=True)
q_with_nonzero_votes = list(q)
# Union.
qs_to_update = list(set(q_with_recent_votes + q_with_nonzero_votes))
# Chunk them for tasks.
for chunk in chunked(qs_to_update, 50):
update_question_vote_chunk.apply_async(args=[chunk])
