def process(task):
ctx = context.get()
params = ctx.mapreduce_spec.mapper.params
program_key = params["program_key"]
try:
program = GCIProgram.get_by_key_name(program_key)
except db.BadValueError:
yield operation.counters.Increment("program_key_is_empty_or_invalid")
return
def subscribe_to_task_txn(task_key, subscribe):
task = GCITask.get(task_key)
task.subscribers = list(set(task.subscribers + subscribe))
task.put()
return task
if task.program.key() != program.key():
yield operation.counters.Increment("old_program_task_not_updated")
return
mentors = db.get(task.mentors)
entities = mentors + [task.created_by, task.modified_by]
subscribe = [ent.key() for ent in entities if ent.automatic_task_subscription]
result = db.run_in_transaction(subscribe_to_task_txn, task.key(), subscribe)
if result:
yield operation.counters.Increment("task_updated")
else:
yield operation.counters.Increment("task_not_updated")
def process(comment):
ctx = context.get()
params = ctx.mapreduce_spec.mapper.params
program_key = params['program_key']
program = GCIProgram.get_by_key_name(program_key)
if comment.parent().program.key() != program.key():
yield operation.counters.Increment(
"prev_program_comment_not_converted")
return
if comment.title not in ACTION_TITLES:
yield operation.counters.Increment("user_comment_not_converted")
return
comment_title = ACTION_TITLES[comment.title]
changes = ACTION_TITLES[comment_title]
# Task reopening is a special case which could have been performed
# either by a mentor or by the automated system after the passing of
# the deadline. So additional inference of the user has to be made.
if comment_title == 'Task Reopened':
if comment.created_by:
user_info = ugettext('User-Mentor')
else:
user_info = ugettext('MelangeAutomatic')
changes = [user_info] + changes
comment.changes = changes
yield operation.db.Put(comment)
yield operation.counters.Increment("action_comment_converted")
def process(comment):
ctx = context.get()
params = ctx.mapreduce_spec.mapper.params
program_key = params['program_key']
program = GCIProgram.get_by_key_name(program_key)
if comment.parent().program.key() != program.key():
yield operation.counters.Increment("prev_program_comment_not_converted")
return
if comment.title not in ACTION_TITLES:
yield operation.counters.Increment("user_comment_not_converted")
return
comment_title = ACTION_TITLES[comment.title]
changes = ACTION_TITLES[comment_title]
# Task reopening is a special case which could have been performed
# either by a mentor or by the automated system after the passing of
# the deadline. So additional inference of the user has to be made.
if comment_title == 'Task Reopened':
if comment.created_by:
user_info = ugettext('User-Mentor')
else:
user_info = ugettext('MelangeAutomatic')
changes = [user_info] + changes
comment.changes = changes
yield operation.db.Put(comment)
yield operation.counters.Increment("action_comment_converted")
def process(org_app):
ctx = context.get()
params = ctx.mapreduce_spec.mapper.params
program_type = params['program_type']
program_key_str = params['program_key']
# now the script is used only for GCI
if program_type != 'gci':
return
program = GCIProgram.get_by_key_name(program_key_str)
survey_query = OrgAppSurvey.all(keys_only=True).filter('program', program)
survey_key = survey_query.get()
# We can skip the survey records not belonging to the given program.
if org_app.survey.key() != survey_key:
return
# TODO(daniel): create a MapReduce/Task RequestData
data = MapreduceRequestData(program, Site.get_by_key_name('site'))
absolute_url = links.ABSOLUTE_LINKER.program(
program, gci_url_names.CREATE_GCI_ORG_PROFILE)
if org_app.status == 'pre-accepted':
org_app_logic.setStatus(data, org_app, 'accepted', absolute_url)
yield operation.counters.Increment("proposals_accepted")
elif org_app.status == 'pre-rejected':
org_app_logic.setStatus(data, org_app, 'rejected', absolute_url)
yield operation.counters.Increment("proposals_rejected")
else:
yield operation.counters.Increment("proposals_ignored")
def process(task):
ctx = context.get()
params = ctx.mapreduce_spec.mapper.params
program_key = params['program_key']
try:
program = GCIProgram.get_by_key_name(program_key)
except db.BadValueError:
yield operation.counters.Increment('program_key_is_empty_or_invalid')
return
def subscribe_to_task_txn(task_key, subscribe):
task = GCITask.get(task_key)
task.subscribers = list(set(task.subscribers + subscribe))
task.put()
return task
if task.program.key() != program.key():
yield operation.counters.Increment("old_program_task_not_updated")
return
mentors = db.get(task.mentors)
entities = mentors + [task.created_by, task.modified_by]
subscribe = [
ent.key() for ent in entities if ent.automatic_task_subscription
]
result = db.run_in_transaction(subscribe_to_task_txn, task.key(),
subscribe)
if result:
yield operation.counters.Increment("task_updated")
else:
yield operation.counters.Increment("task_not_updated")
def recalculateGCIRanking(self, request, *args, **kwargs):
"""Recalculates student ranking for the entire program.
Args in POST dict:
cursor: Query cursor to figure out where we need to start processing
"""
key_name = '%s/%s' % (kwargs['sponsor'], kwargs['program'])
cursor = request.POST.get('cursor')
program = GCIProgram.get_by_key_name(key_name)
if not program:
logging.warning(
'Enqueued recalculate ranking task for non-existing program: %s',
key_name)
return responses.terminateTask()
# Retrieve the students for the program
q = GCIProfile.all()
q.filter('program', program)
q.filter('is_student', True)
if cursor:
q.with_cursor(cursor)
students = q.fetch(25)
for student in students:
# get all the tasks that the student has completed
task_q = GCITask.all()
task_q.filter('student', student)
task_q.filter('status', 'Closed')
tasks = task_q.fetch(1000)
# calculate score with all the tasks
score_logic.calculateScore(student, tasks, program)
# calculate org score with all the tasks
db.run_in_transaction(org_score_logic.updateOrgScoresTxn(tasks))
if students:
# schedule task to do the rest of the students
params = {
'cursor': q.cursor(),
}
taskqueue.add(queue_name='gci-update',
url=request.path,
params=params)
return responses.terminateTask()
def process(task):
ctx = context.get()
params = ctx.mapreduce_spec.mapper.params
program_key = params['program_key']
program = GCIProgram.get_by_key_name(program_key)
if (task.program.key() == program.key()
and (task.status in task_model.UNAVAILABLE)):
task.status = task_model.OPEN
yield operation.db.Put(task)
yield operation.counters.Increment("task_updated")
yield operation.counters.Increment("task_not_updated")
def process(task):
ctx = context.get()
params = ctx.mapreduce_spec.mapper.params
program_key = params['program_key']
program = GCIProgram.get_by_key_name(program_key)
if (task.program.key() == program.key() and
(task.status == 'Unapproved'or task.status == 'Unpublished')):
task.status = 'Open'
yield operation.db.Put(task)
yield operation.counters.Increment("task_updated")
yield operation.counters.Increment("task_not_updated")
def process(task):
ctx = context.get()
params = ctx.mapreduce_spec.mapper.params
program_key = params['program_key']
program = GCIProgram.get_by_key_name(program_key)
if (task.program.key() == program.key() and
(task.status in task_model.UNAVAILABLE)):
task.status = task_model.OPEN
yield operation.db.Put(task)
yield operation.counters.Increment("task_updated")
yield operation.counters.Increment("task_not_updated")
def recalculateGCIRanking(self, request, *args, **kwargs):
"""Recalculates student ranking for the entire program.
Args in POST dict:
cursor: Query cursor to figure out where we need to start processing
"""
key_name = '%s/%s' % (kwargs['sponsor'], kwargs['program'])
cursor = request.POST.get('cursor')
program = GCIProgram.get_by_key_name(key_name)
if not program:
logging.warning(
'Enqueued recalculate ranking task for non-existing '
'program: %s' %key_name)
return responses.terminateTask()
# Retrieve the students for the program
q = GCIProfile.all()
q.filter('scope', program)
q.filter('is_student', True)
if cursor:
q.with_cursor(cursor)
students = q.fetch(25)
for student in students:
# get all the tasks that the student has completed
task_q = GCITask.all()
task_q.filter('student', student)
task_q.filter('status', 'Closed')
tasks = task_q.fetch(1000)
# calculate ranking with all the tasks
# ranking_logic.calculateRankingForStudent(student, tasks)
ranking_logic.calculateScore(student, tasks, program)
if students:
# schedule task to do the rest of the students
params = {
'cursor': q.cursor(),
}
taskqueue.add(queue_name='gci-update', url=request.path, params=params)
return responses.terminateTask()
def recalculateGCIRanking(self, request, *args, **kwargs):
"""Recalculates student ranking for the entire program.
Args in POST dict:
cursor: Query cursor to figure out where we need to start processing
"""
key_name = "%s/%s" % (kwargs["sponsor"], kwargs["program"])
cursor = request.POST.get("cursor")
program = GCIProgram.get_by_key_name(key_name)
if not program:
logging.warning("Enqueued recalculate ranking task for non-existing program: %s", key_name)
return responses.terminateTask()
# Retrieve the students for the program
q = GCIProfile.all()
q.filter("program", program)
q.filter("is_student", True)
if cursor:
q.with_cursor(cursor)
students = q.fetch(25)
for student in students:
# get all the tasks that the student has completed
task_q = GCITask.all()
task_q.filter("student", student)
task_q.filter("status", "Closed")
tasks = task_q.fetch(1000)
# calculate score with all the tasks
score_logic.calculateScore(student, tasks, program)
# calculate org score with all the tasks
db.run_in_transaction(org_score_logic.updateOrgScoresTxn(tasks))
if students:
# schedule task to do the rest of the students
params = {"cursor": q.cursor()}
taskqueue.add(queue_name="gci-update", url=request.path, params=params)
return responses.terminateTask()
def clearGCIRanking(self, request, *args, **kwargs):
"""Clears student ranking for a program with the specified key_name.
"""
key_name = "%s/%s" % (kwargs["sponsor"], kwargs["program"])
program = GCIProgram.get_by_key_name(key_name)
if not program:
logging.warning("Enqueued recalculate ranking task for non-existing program: %s", key_name)
return responses.terminateTask()
q = GCIScore.all()
q.filter("program", program)
rankings = q.fetch(500)
while rankings:
db.delete(rankings)
rankings = q.fetch(500)
return responses.terminateTask()
def process(org_app):
ctx = context.get()
params = ctx.mapreduce_spec.mapper.params
program_key = params['program_key']
# TODO(SRabbelier): should have been a full url
url = 'gci/profile/organization/%s' % program_key
# TODO(SRabbelier): create a MapReduce/Task RequestData
data = RequestData()
data.program = GCIProgram.get_by_key_name(program_key)
data.site = Site.get_by_key_name('site')
if org_app.status == 'pre-accepted':
org_app_logic.setStatus(data, org_app, 'accepted', url)
yield operation.counters.Increment("proposals_accepted")
elif org_app.status == 'pre-rejected':
org_app_logic.setStatus(data, org_app, 'rejected', url)
yield operation.counters.Increment("proposals_rejected")
else:
yield operation.counters.Increment("proposals_ignored")
def clearGCIRanking(self, request, *args, **kwargs):
"""Clears student ranking for a program with the specified key_name.
"""
key_name = '%s/%s' % (kwargs['sponsor'], kwargs['program'])
program = GCIProgram.get_by_key_name(key_name)
if not program:
logging.warning(
'Enqueued recalculate ranking task for non-existing program: %s',
key_name)
return responses.terminateTask()
q = GCIScore.all()
q.filter('program', program)
rankings = q.fetch(500)
while rankings:
db.delete(rankings)
rankings = q.fetch(500)
return responses.terminateTask()
def downloadStudentForms(options):
from google.appengine.ext import db
from soc.views.helper import lists as list_helper
from soc.modules.gci.models.profile import GCIProfile
from soc.modules.gci.models.profile import GCIStudentInfo
from soc.modules.gci.models.program import GCIProgram
from soc.modules.gci.models.score import GCIScore
if not options.program_path:
print "--program_path or -p option is required"
program = GCIProgram.get_by_key_name(options.program_path)
outputdir = os.path.abspath(options.outputdir)
if not os.path.exists(outputdir):
os.mkdir(outputdir)
if not os.path.isdir(outputdir):
print "Could not create output dir: %s" % outputdir
q = lambda: GCIScore.all().filter("program =", program)
print "Fetching GCIScore..."
scores = list(i for i in interactive.deepFetch(q))
keys = list_helper.collectParentKeys(scores)
keys = list(set(keys))
prefetched = {}
print "Fetching Profile..."
for i in xrange(0, len(keys), 100):
chunk = keys[i:i+100]
entities = db.get(chunk)
prefetched.update(dict((i.key(), i) for i in entities if i))
profiles = prefetched.values()
list_helper.distributeParentKeys(scores, prefetched)
keys = list_helper.collectKeys(GCIProfile.student_info, entities)
keys = list(set(keys))
prefetched = {}
print "Fetching StudentInfo..."
for i in xrange(0, len(keys), 100):
chunk = keys[i:i+100]
entities = db.get(chunk)
prefetched.update(dict((i.key(), i) for i in entities if i))
studentInfos = prefetched.values()
list_helper.distributeKeys(GCIProfile.student_info, profiles, prefetched)
i = 0
while i < len(profiles):
try:
profile = profiles[i]
consent_form = profile.student_info.consent_form
student_id_form = profile.student_info.student_id_form
if not consent_form or not student_id_form:
print "At least one form missing from %s" % profile.link_id
else:
_saveForm(profile, consent_form, 'consent-form', outputdir)
_saveForm(profile, consent_form, 'student-id-form', outputdir)
except Exception, e:
continue
i += 1
def downloadStudentForms(options):
from google.appengine.ext import db
from soc.views.helper import lists as list_helper
from soc.modules.gci.models.profile import GCIProfile
from soc.modules.gci.models.profile import GCIStudentInfo
from soc.modules.gci.models.program import GCIProgram
from soc.modules.gci.models.score import GCIScore
if not options.program_path:
print "--program_path or -p option is required"
program = GCIProgram.get_by_key_name(options.program_path)
outputdir = os.path.abspath(options.outputdir)
if not os.path.exists(outputdir):
os.mkdir(outputdir)
if not os.path.isdir(outputdir):
print "Could not create output dir: %s" % outputdir
q = lambda: GCIScore.all().filter("program =", program)
print "Fetching GCIScore..."
scores = list(i for i in interactive.deepFetch(q))
keys = list_helper.collectParentKeys(scores)
keys = list(set(keys))
prefetched = {}
print "Fetching Profile..."
for i in xrange(0, len(keys), 100):
chunk = keys[i:i + 100]
entities = db.get(chunk)
prefetched.update(dict((i.key(), i) for i in entities if i))
profiles = prefetched.values()
list_helper.distributeParentKeys(scores, prefetched)
keys = list_helper.collectKeys(GCIProfile.student_info, entities)
keys = list(set(keys))
prefetched = {}
print "Fetching StudentInfo..."
for i in xrange(0, len(keys), 100):
chunk = keys[i:i + 100]
entities = db.get(chunk)
prefetched.update(dict((i.key(), i) for i in entities if i))
studentInfos = prefetched.values()
list_helper.distributeKeys(GCIProfile.student_info, profiles, prefetched)
i = 0
while i < len(profiles):
try:
profile = profiles[i]
consent_form = profile.student_info.consent_form
student_id_form = profile.student_info.student_id_form
if not consent_form or not student_id_form:
print "At least one form missing from %s" % profile.link_id
else:
_saveForm(profile, consent_form, 'consent-form', outputdir)
_saveForm(profile, consent_form, 'student-id-form', outputdir)
except Exception, e:
continue
i += 1
