def process(task): ctx = context.get() params = ctx.mapreduce_spec.mapper.params program_key = params["program_key"] try: program = GCIProgram.get_by_key_name(program_key) except db.BadValueError: yield operation.counters.Increment("program_key_is_empty_or_invalid") return def subscribe_to_task_txn(task_key, subscribe): task = GCITask.get(task_key) task.subscribers = list(set(task.subscribers + subscribe)) task.put() return task if task.program.key() != program.key(): yield operation.counters.Increment("old_program_task_not_updated") return mentors = db.get(task.mentors) entities = mentors + [task.created_by, task.modified_by] subscribe = [ent.key() for ent in entities if ent.automatic_task_subscription] result = db.run_in_transaction(subscribe_to_task_txn, task.key(), subscribe) if result: yield operation.counters.Increment("task_updated") else: yield operation.counters.Increment("task_not_updated")
def process(comment): ctx = context.get() params = ctx.mapreduce_spec.mapper.params program_key = params['program_key'] program = GCIProgram.get_by_key_name(program_key) if comment.parent().program.key() != program.key(): yield operation.counters.Increment( "prev_program_comment_not_converted") return if comment.title not in ACTION_TITLES: yield operation.counters.Increment("user_comment_not_converted") return comment_title = ACTION_TITLES[comment.title] changes = ACTION_TITLES[comment_title] # Task reopening is a special case which could have been performed # either by a mentor or by the automated system after the passing of # the deadline. So additional inference of the user has to be made. if comment_title == 'Task Reopened': if comment.created_by: user_info = ugettext('User-Mentor') else: user_info = ugettext('MelangeAutomatic') changes = [user_info] + changes comment.changes = changes yield operation.db.Put(comment) yield operation.counters.Increment("action_comment_converted")
def process(comment): ctx = context.get() params = ctx.mapreduce_spec.mapper.params program_key = params['program_key'] program = GCIProgram.get_by_key_name(program_key) if comment.parent().program.key() != program.key(): yield operation.counters.Increment("prev_program_comment_not_converted") return if comment.title not in ACTION_TITLES: yield operation.counters.Increment("user_comment_not_converted") return comment_title = ACTION_TITLES[comment.title] changes = ACTION_TITLES[comment_title] # Task reopening is a special case which could have been performed # either by a mentor or by the automated system after the passing of # the deadline. So additional inference of the user has to be made. if comment_title == 'Task Reopened': if comment.created_by: user_info = ugettext('User-Mentor') else: user_info = ugettext('MelangeAutomatic') changes = [user_info] + changes comment.changes = changes yield operation.db.Put(comment) yield operation.counters.Increment("action_comment_converted")
def process(org_app): ctx = context.get() params = ctx.mapreduce_spec.mapper.params program_type = params['program_type'] program_key_str = params['program_key'] # now the script is used only for GCI if program_type != 'gci': return program = GCIProgram.get_by_key_name(program_key_str) survey_query = OrgAppSurvey.all(keys_only=True).filter('program', program) survey_key = survey_query.get() # We can skip the survey records not belonging to the given program. if org_app.survey.key() != survey_key: return # TODO(daniel): create a MapReduce/Task RequestData data = MapreduceRequestData(program, Site.get_by_key_name('site')) absolute_url = links.ABSOLUTE_LINKER.program( program, gci_url_names.CREATE_GCI_ORG_PROFILE) if org_app.status == 'pre-accepted': org_app_logic.setStatus(data, org_app, 'accepted', absolute_url) yield operation.counters.Increment("proposals_accepted") elif org_app.status == 'pre-rejected': org_app_logic.setStatus(data, org_app, 'rejected', absolute_url) yield operation.counters.Increment("proposals_rejected") else: yield operation.counters.Increment("proposals_ignored")
def process(task): ctx = context.get() params = ctx.mapreduce_spec.mapper.params program_key = params['program_key'] try: program = GCIProgram.get_by_key_name(program_key) except db.BadValueError: yield operation.counters.Increment('program_key_is_empty_or_invalid') return def subscribe_to_task_txn(task_key, subscribe): task = GCITask.get(task_key) task.subscribers = list(set(task.subscribers + subscribe)) task.put() return task if task.program.key() != program.key(): yield operation.counters.Increment("old_program_task_not_updated") return mentors = db.get(task.mentors) entities = mentors + [task.created_by, task.modified_by] subscribe = [ ent.key() for ent in entities if ent.automatic_task_subscription ] result = db.run_in_transaction(subscribe_to_task_txn, task.key(), subscribe) if result: yield operation.counters.Increment("task_updated") else: yield operation.counters.Increment("task_not_updated")
def recalculateGCIRanking(self, request, *args, **kwargs): """Recalculates student ranking for the entire program. Args in POST dict: cursor: Query cursor to figure out where we need to start processing """ key_name = '%s/%s' % (kwargs['sponsor'], kwargs['program']) cursor = request.POST.get('cursor') program = GCIProgram.get_by_key_name(key_name) if not program: logging.warning( 'Enqueued recalculate ranking task for non-existing program: %s', key_name) return responses.terminateTask() # Retrieve the students for the program q = GCIProfile.all() q.filter('program', program) q.filter('is_student', True) if cursor: q.with_cursor(cursor) students = q.fetch(25) for student in students: # get all the tasks that the student has completed task_q = GCITask.all() task_q.filter('student', student) task_q.filter('status', 'Closed') tasks = task_q.fetch(1000) # calculate score with all the tasks score_logic.calculateScore(student, tasks, program) # calculate org score with all the tasks db.run_in_transaction(org_score_logic.updateOrgScoresTxn(tasks)) if students: # schedule task to do the rest of the students params = { 'cursor': q.cursor(), } taskqueue.add(queue_name='gci-update', url=request.path, params=params) return responses.terminateTask()
def process(task): ctx = context.get() params = ctx.mapreduce_spec.mapper.params program_key = params['program_key'] program = GCIProgram.get_by_key_name(program_key) if (task.program.key() == program.key() and (task.status in task_model.UNAVAILABLE)): task.status = task_model.OPEN yield operation.db.Put(task) yield operation.counters.Increment("task_updated") yield operation.counters.Increment("task_not_updated")
def process(task): ctx = context.get() params = ctx.mapreduce_spec.mapper.params program_key = params['program_key'] program = GCIProgram.get_by_key_name(program_key) if (task.program.key() == program.key() and (task.status == 'Unapproved'or task.status == 'Unpublished')): task.status = 'Open' yield operation.db.Put(task) yield operation.counters.Increment("task_updated") yield operation.counters.Increment("task_not_updated")
def process(task): ctx = context.get() params = ctx.mapreduce_spec.mapper.params program_key = params['program_key'] program = GCIProgram.get_by_key_name(program_key) if (task.program.key() == program.key() and (task.status in task_model.UNAVAILABLE)): task.status = task_model.OPEN yield operation.db.Put(task) yield operation.counters.Increment("task_updated") yield operation.counters.Increment("task_not_updated")
def recalculateGCIRanking(self, request, *args, **kwargs): """Recalculates student ranking for the entire program. Args in POST dict: cursor: Query cursor to figure out where we need to start processing """ key_name = '%s/%s' % (kwargs['sponsor'], kwargs['program']) cursor = request.POST.get('cursor') program = GCIProgram.get_by_key_name(key_name) if not program: logging.warning( 'Enqueued recalculate ranking task for non-existing ' 'program: %s' %key_name) return responses.terminateTask() # Retrieve the students for the program q = GCIProfile.all() q.filter('scope', program) q.filter('is_student', True) if cursor: q.with_cursor(cursor) students = q.fetch(25) for student in students: # get all the tasks that the student has completed task_q = GCITask.all() task_q.filter('student', student) task_q.filter('status', 'Closed') tasks = task_q.fetch(1000) # calculate ranking with all the tasks # ranking_logic.calculateRankingForStudent(student, tasks) ranking_logic.calculateScore(student, tasks, program) if students: # schedule task to do the rest of the students params = { 'cursor': q.cursor(), } taskqueue.add(queue_name='gci-update', url=request.path, params=params) return responses.terminateTask()
def recalculateGCIRanking(self, request, *args, **kwargs): """Recalculates student ranking for the entire program. Args in POST dict: cursor: Query cursor to figure out where we need to start processing """ key_name = "%s/%s" % (kwargs["sponsor"], kwargs["program"]) cursor = request.POST.get("cursor") program = GCIProgram.get_by_key_name(key_name) if not program: logging.warning("Enqueued recalculate ranking task for non-existing program: %s", key_name) return responses.terminateTask() # Retrieve the students for the program q = GCIProfile.all() q.filter("program", program) q.filter("is_student", True) if cursor: q.with_cursor(cursor) students = q.fetch(25) for student in students: # get all the tasks that the student has completed task_q = GCITask.all() task_q.filter("student", student) task_q.filter("status", "Closed") tasks = task_q.fetch(1000) # calculate score with all the tasks score_logic.calculateScore(student, tasks, program) # calculate org score with all the tasks db.run_in_transaction(org_score_logic.updateOrgScoresTxn(tasks)) if students: # schedule task to do the rest of the students params = {"cursor": q.cursor()} taskqueue.add(queue_name="gci-update", url=request.path, params=params) return responses.terminateTask()
def clearGCIRanking(self, request, *args, **kwargs): """Clears student ranking for a program with the specified key_name. """ key_name = "%s/%s" % (kwargs["sponsor"], kwargs["program"]) program = GCIProgram.get_by_key_name(key_name) if not program: logging.warning("Enqueued recalculate ranking task for non-existing program: %s", key_name) return responses.terminateTask() q = GCIScore.all() q.filter("program", program) rankings = q.fetch(500) while rankings: db.delete(rankings) rankings = q.fetch(500) return responses.terminateTask()
def process(org_app): ctx = context.get() params = ctx.mapreduce_spec.mapper.params program_key = params['program_key'] # TODO(SRabbelier): should have been a full url url = 'gci/profile/organization/%s' % program_key # TODO(SRabbelier): create a MapReduce/Task RequestData data = RequestData() data.program = GCIProgram.get_by_key_name(program_key) data.site = Site.get_by_key_name('site') if org_app.status == 'pre-accepted': org_app_logic.setStatus(data, org_app, 'accepted', url) yield operation.counters.Increment("proposals_accepted") elif org_app.status == 'pre-rejected': org_app_logic.setStatus(data, org_app, 'rejected', url) yield operation.counters.Increment("proposals_rejected") else: yield operation.counters.Increment("proposals_ignored")
def clearGCIRanking(self, request, *args, **kwargs): """Clears student ranking for a program with the specified key_name. """ key_name = '%s/%s' % (kwargs['sponsor'], kwargs['program']) program = GCIProgram.get_by_key_name(key_name) if not program: logging.warning( 'Enqueued recalculate ranking task for non-existing program: %s', key_name) return responses.terminateTask() q = GCIScore.all() q.filter('program', program) rankings = q.fetch(500) while rankings: db.delete(rankings) rankings = q.fetch(500) return responses.terminateTask()
def downloadStudentForms(options): from google.appengine.ext import db from soc.views.helper import lists as list_helper from soc.modules.gci.models.profile import GCIProfile from soc.modules.gci.models.profile import GCIStudentInfo from soc.modules.gci.models.program import GCIProgram from soc.modules.gci.models.score import GCIScore if not options.program_path: print "--program_path or -p option is required" program = GCIProgram.get_by_key_name(options.program_path) outputdir = os.path.abspath(options.outputdir) if not os.path.exists(outputdir): os.mkdir(outputdir) if not os.path.isdir(outputdir): print "Could not create output dir: %s" % outputdir q = lambda: GCIScore.all().filter("program =", program) print "Fetching GCIScore..." scores = list(i for i in interactive.deepFetch(q)) keys = list_helper.collectParentKeys(scores) keys = list(set(keys)) prefetched = {} print "Fetching Profile..." for i in xrange(0, len(keys), 100): chunk = keys[i:i+100] entities = db.get(chunk) prefetched.update(dict((i.key(), i) for i in entities if i)) profiles = prefetched.values() list_helper.distributeParentKeys(scores, prefetched) keys = list_helper.collectKeys(GCIProfile.student_info, entities) keys = list(set(keys)) prefetched = {} print "Fetching StudentInfo..." for i in xrange(0, len(keys), 100): chunk = keys[i:i+100] entities = db.get(chunk) prefetched.update(dict((i.key(), i) for i in entities if i)) studentInfos = prefetched.values() list_helper.distributeKeys(GCIProfile.student_info, profiles, prefetched) i = 0 while i < len(profiles): try: profile = profiles[i] consent_form = profile.student_info.consent_form student_id_form = profile.student_info.student_id_form if not consent_form or not student_id_form: print "At least one form missing from %s" % profile.link_id else: _saveForm(profile, consent_form, 'consent-form', outputdir) _saveForm(profile, consent_form, 'student-id-form', outputdir) except Exception, e: continue i += 1
def downloadStudentForms(options): from google.appengine.ext import db from soc.views.helper import lists as list_helper from soc.modules.gci.models.profile import GCIProfile from soc.modules.gci.models.profile import GCIStudentInfo from soc.modules.gci.models.program import GCIProgram from soc.modules.gci.models.score import GCIScore if not options.program_path: print "--program_path or -p option is required" program = GCIProgram.get_by_key_name(options.program_path) outputdir = os.path.abspath(options.outputdir) if not os.path.exists(outputdir): os.mkdir(outputdir) if not os.path.isdir(outputdir): print "Could not create output dir: %s" % outputdir q = lambda: GCIScore.all().filter("program =", program) print "Fetching GCIScore..." scores = list(i for i in interactive.deepFetch(q)) keys = list_helper.collectParentKeys(scores) keys = list(set(keys)) prefetched = {} print "Fetching Profile..." for i in xrange(0, len(keys), 100): chunk = keys[i:i + 100] entities = db.get(chunk) prefetched.update(dict((i.key(), i) for i in entities if i)) profiles = prefetched.values() list_helper.distributeParentKeys(scores, prefetched) keys = list_helper.collectKeys(GCIProfile.student_info, entities) keys = list(set(keys)) prefetched = {} print "Fetching StudentInfo..." for i in xrange(0, len(keys), 100): chunk = keys[i:i + 100] entities = db.get(chunk) prefetched.update(dict((i.key(), i) for i in entities if i)) studentInfos = prefetched.values() list_helper.distributeKeys(GCIProfile.student_info, profiles, prefetched) i = 0 while i < len(profiles): try: profile = profiles[i] consent_form = profile.student_info.consent_form student_id_form = profile.student_info.student_id_form if not consent_form or not student_id_form: print "At least one form missing from %s" % profile.link_id else: _saveForm(profile, consent_form, 'consent-form', outputdir) _saveForm(profile, consent_form, 'student-id-form', outputdir) except Exception, e: continue i += 1