def convertGCIProfileDBEntityGroup(profile_key): """Converts DB based part of entity group associated with the specified profile. Args: profile_key: db.Key of the profile to process. """ to_delete = [] do_put = True org_scores = GCIOrgScore.all().ancestor(profile_key).fetch(1000) for org_score in org_scores: new_org_score = _convertParent(org_score) if do_put: new_org_score.put() to_delete.append(org_score) scores = GCIScore.all().ancestor(profile_key).fetch(1000) for score in scores: new_score = _convertParent(score) if do_put: new_score.put() to_delete.append(score) db.delete(to_delete)
def allScoresForProgramQuery(program): """Returns the query to fetch all the scores for the specified program. Args: program: GCIProgram entity for which the query should filter the program """ return GCIScore.all().filter('program =', program)
def getListData(self): idx = lists.getListIndex(self.request) if idx != self.idx: return None q = GCIScore.all() q.filter('program', self.data.program) starter = lists.keyStarter def prefetcher(entities): keys = [] for entity in entities: key = entity.parent_key() if key: keys.append(key) entities = db.get(keys) sp = dict((i.key(), i) for i in entities if i) return ([sp], {}) response_builder = lists.RawQueryContentResponseBuilder( self.request, self._list_config, q, starter, prefetcher=prefetcher) return response_builder.build()
def allScoresForProgramQuery(program): """Returns the query to fetch all the scores for the specified program. Args: program: GCIProgram entity for which the query should filter the program """ return GCIScore.all().filter('program =', program)
def update_ranking_txn(): logging.info("updateScore txn starts for task %s" % task_key.id()) query = GCIScore.all().ancestor(student) score = query.get() # create a new GCIStore entity if one does not exist yet if not score: logging.info("score entity is being created") score = GCIScore(parent=student, program=program) # check if the task has been included in the score if task_key not in score.tasks: score.points += POINTS[task.difficulty_level] score.tasks.append(task_key) # TODO(dhans): optimize it; sometimes, put may not be needed logging.info("score put") score.put() logging.info("score put returned") query = GCIStudentInfo.all().ancestor(student) student_info = query.get() # set in student info that the student has completed a task if not student_info.task_closed: student_info.task_closed = True student_info.put()
def convertGCIProfileDBEntityGroup(profile_key): """Converts DB based part of entity group associated with the specified profile. Args: profile_key: db.Key of the profile to process. """ to_delete = [] do_put = True org_scores = GCIOrgScore.all().ancestor(profile_key).fetch(1000) for org_score in org_scores: new_org_score = _convertParent(org_score) if do_put: new_org_score.put() to_delete.append(org_score) scores = GCIScore.all().ancestor(profile_key).fetch(1000) for score in scores: new_score = _convertParent(score) if do_put: new_score.put() to_delete.append(score) db.delete(to_delete)
def get(profile): """Gets the score entity associated with the specified profile. Args: profile: GCIProfile entity to retrieve a score for """ query = GCIScore.all().ancestor(profile) return query.get()
def get(profile): """Gets the score entity associated with the specified profile. Args: profile: GCIProfile entity to retrieve a score for """ query = GCIScore.all().ancestor(profile) return query.get()
def getListData(self): idx = lists.getListIndex(self.data.request) if idx == self.LEADERBOARD_LIST_IDX: q = GCIScore.all() q.filter('program', self.data.program) skipper = lambda entity, start: entity.points <= 0 prefetcher = lists.ModelPrefetcher(GCIScore, [], True) response_builder = lists.RawQueryContentResponseBuilder( self.data.request, self._list_config, q, lists.keyStarter, skipper=skipper, prefetcher=prefetcher) return response_builder.build() else: return None
def calculate_score_txn(): query = GCIScore.all().ancestor(student) score = query.get() # create a new GCIStore entity if one does not exist yet if not score: score = GCIScore(parent=student, program=program) score.points = points score.tasks = [task.key() for task in tasks] score.put() # set that the student closed a task in GCIStudentInfo query = GCIStudentInfo.all().ancestor(student) student_info = query.get() student_info.task_closed = True student_info.put()
def calculate_score_txn(): query = GCIScore.all().ancestor(student) score = query.get() # create a new GCIStore entity if one does not exist yet if not score: score = GCIScore(parent=student, program=program) score.points = points score.tasks = [task.key() for task in tasks] score.put() # set that the student closed a task in GCIStudentInfo query = GCIStudentInfo.all().ancestor(student) student_info = query.get() student_info.task_closed = True student_info.put()
def clear(*args, **kwargs): """Removes all entities from the datastore. """ # TODO(dbentley): If there are more than 1000 instances of any model, # this method will not clear all instances. Instead, it should continually # call .all(), delete all those, and loop until .all() is empty. entities = itertools.chain(*[ Survey.all(), SurveyRecord.all(), GCIOrganization.all(), GSoCTimeline.all(), GCITimeline.all(), GSoCProgram.all(), GSoCProject.all(), GSoCProposal.all(), GCIProgram.all(), GCIScore.all(), GSoCStudentInfo.all(), GCIStudentInfo.all(), GCITask.all(), Sponsor.all(), Site.all(), Document.all(), # The below models are all subclasses of ndb.Model and therefore must # use .query() to return all instances instead of .all(). soc_org_model.SOCOrganization.query(), profile_model.Profile.query(), soc_profile.SOCStudentData.query(), user.User.query(), address.Address.query(), contact.Contact.query() ]) try: for entity in entities: if isinstance(entity, ndb.Model): entity.key.delete() else: entity.delete() except db.Timeout: return http.HttpResponseRedirect('#') memcache.flush_all() return http.HttpResponse('Done')
def clear(*args, **kwargs): """Removes all entities from the datastore. """ # TODO(dbentley): If there are more than 1000 instances of any model, # this method will not clear all instances. Instead, it should continually # call .all(), delete all those, and loop until .all() is empty. entities = itertools.chain(*[ Survey.all(), SurveyRecord.all(), GCIOrganization.all(), GSoCTimeline.all(), GCITimeline.all(), GSoCProgram.all(), GSoCProject.all(), GSoCProposal.all(), GCIProgram.all(), GCIScore.all(), GSoCStudentInfo.all(), GCIStudentInfo.all(), GCITask.all(), Sponsor.all(), Site.all(), Document.all(), # The below models are all subclasses of ndb.Model and therefore must # use .query() to return all instances instead of .all(). soc_org_model.SOCOrganization.query(), profile_model.Profile.query(), soc_profile.SOCStudentData.query(), user.User.query(), address.Address.query(), contact.Contact.query() ]) try: for entity in entities: if isinstance(entity, ndb.Model): entity.key.delete() else: entity.delete() except db.Timeout: return http.HttpResponseRedirect('#') memcache.flush_all() return http.HttpResponse('Done')
def winnersForProgram(data): """Returns the winners for the program. The number of winners chosen is configurable from the program edit page. The return datastructure is a dictionary with profile keys of winners as keys with values as dictionaries containing the name, number of task and the points scored. Args: data: The RequestData object. """ program = data.program q = GCIScore.all() q.filter('program', program) q.filter('points >', 0) q.order('-points') scores = q.fetch(program.nr_winners) profile_keys = [s.parent_key() for s in scores] profiles = db.get(profile_keys) winners = SortedDict() for score in scores: winners[score.parent_key()] = { 'score': score, } for profile in profiles: winner = winners[profile.key()] winner['profile'] = profile winner['completed_tasks_link'] = data.redirect.profile( profile.link_id).urlOf(url_names.GCI_STUDENT_TASKS) if profile.avatar: avatar_groups = re.findall(forms.RE_AVATAR_COLOR, profile.avatar) # Being a bit pessimistic if avatar_groups: # We only want the first match, so pick group[0] name, prefix = avatar_groups[0] winner['avatar_name'] = '%s-%s.jpg' % (name, prefix) winner['avatar_prefix'] = prefix return winners
def winnersForProgram(data): """Returns the winners for the program. The number of winners chosen is configurable from the program edit page. The return datastructure is a dictionary with profile keys of winners as keys with values as dictionaries containing the name, number of task and the points scored. Args: data: The RequestData object. """ program = data.program q = GCIScore.all() q.filter('program', program) q.filter('points >', 0) q.order('-points') scores = q.fetch(program.nr_winners) profile_keys = [s.parent_key() for s in scores] profiles = db.get(profile_keys) winners = SortedDict() for score in scores: winners[score.parent_key()] = { 'score': score, } for profile in profiles: winner = winners[profile.key()] winner['profile'] = profile winner['completed_tasks_link'] = data.redirect.profile( profile.link_id).urlOf(url_names.GCI_STUDENT_TASKS) if profile.avatar: avatar_groups = re.findall(forms.RE_AVATAR_COLOR, profile.avatar) # Being a bit pessimistic if avatar_groups: # We only want the first match, so pick group[0] name, prefix = avatar_groups[0] winner['avatar_name'] = '%s-%s.jpg' % (name, prefix) winner['avatar_prefix'] = prefix return winners
def clearGCIRanking(self, request, *args, **kwargs): """Clears student ranking for a program with the specified key_name. """ key_name = "%s/%s" % (kwargs["sponsor"], kwargs["program"]) program = GCIProgram.get_by_key_name(key_name) if not program: logging.warning("Enqueued recalculate ranking task for non-existing program: %s", key_name) return responses.terminateTask() q = GCIScore.all() q.filter("program", program) rankings = q.fetch(500) while rankings: db.delete(rankings) rankings = q.fetch(500) return responses.terminateTask()
def clear(*args, **kwargs): """Removes all entities from the datastore. """ # TODO(dbentley): If there are more than 1000 instances of any model, # this method will not clear all instances. Instead, it should continually # call .all(), delete all those, and loop until .all() is empty. entities = itertools.chain(*[ Notification.all(), GCIStudent.all(), Survey.all(), SurveyRecord.all(), StudentProposal.all(), GSoCOrganization.all(), GCIOrganization.all(), GSoCTimeline.all(), GCITimeline.all(), GSoCProgram.all(), GSoCProfile.all(), GCIProfile.all(), GSoCProposal.all(), GCIProgram.all(), GCIScore.all(), GSoCStudentInfo.all(), GCIStudentInfo.all(), GCITask.all(), Host.all(), Sponsor.all(), User.all(), Site.all(), Document.all(), ]) try: for entity in entities: entity.delete() except db.Timeout: return http.HttpResponseRedirect('#') # pylint: disable=E1101 memcache.flush_all() return http.HttpResponse('Done')
def getListData(self): idx = lists.getListIndex(self.data.request) if idx == self.LEADERBOARD_LIST_IDX: q = GCIScore.all() q.filter('program', self.data.program) skipper = lambda entity, start: entity.points <= 0 prefetcher = lists.ModelPrefetcher(GCIScore, [], True) response_builder = lists.RawQueryContentResponseBuilder( self.data.request, self._list_config, q, lists.keyStarter, skipper=skipper, prefetcher=prefetcher) return response_builder.build() else: return None
def clearGCIRanking(self, request, *args, **kwargs): """Clears student ranking for a program with the specified key_name. """ key_name = '%s/%s' % (kwargs['sponsor'], kwargs['program']) program = GCIProgram.get_by_key_name(key_name) if not program: logging.warning( 'Enqueued recalculate ranking task for non-existing program: %s', key_name) return responses.terminateTask() q = GCIScore.all() q.filter('program', program) rankings = q.fetch(500) while rankings: db.delete(rankings) rankings = q.fetch(500) return responses.terminateTask()
def update_ranking_txn(): logging.info("updateScore txn starts for task %s", task_key.id()) query = GCIScore.all().ancestor(student) score = query.get() # create a new GCIStore entity if one does not exist yet if not score: score = GCIScore(parent=student, program=program) # check if the task has been included in the score if task_key not in score.tasks: if not task.points_invalidated: score.points += POINTS[task.difficulty_level] score.tasks.append(task_key) score.put() query = GCIStudentInfo.all().ancestor(student) student_info = query.get() # set in student info that the student has completed a task if not student_info.task_closed: student_info.task_closed = True student_info.put()
def update_ranking_txn(): logging.info("updateScore txn starts for task %s", task_key.id()) query = GCIScore.all().ancestor(student) score = query.get() # create a new GCIStore entity if one does not exist yet if not score: score = GCIScore(parent=student, program=program) # check if the task has been included in the score if task_key not in score.tasks: if not task.points_invalidated: score.points += POINTS[task.difficulty_level] score.tasks.append(task_key) score.put() query = GCIStudentInfo.all().ancestor(student) student_info = query.get() # set in student info that the student has completed a task if not student_info.task_closed: student_info.task_closed = True student_info.put()
def downloadStudentForms(options): from google.appengine.ext import db from soc.views.helper import lists as list_helper from soc.modules.gci.models.profile import GCIProfile from soc.modules.gci.models.profile import GCIStudentInfo from soc.modules.gci.models.program import GCIProgram from soc.modules.gci.models.score import GCIScore if not options.program_path: print "--program_path or -p option is required" program = GCIProgram.get_by_key_name(options.program_path) outputdir = os.path.abspath(options.outputdir) if not os.path.exists(outputdir): os.mkdir(outputdir) if not os.path.isdir(outputdir): print "Could not create output dir: %s" % outputdir q = lambda: GCIScore.all().filter("program =", program) print "Fetching GCIScore..." scores = list(i for i in interactive.deepFetch(q)) keys = list_helper.collectParentKeys(scores) keys = list(set(keys)) prefetched = {} print "Fetching Profile..." for i in xrange(0, len(keys), 100): chunk = keys[i:i+100] entities = db.get(chunk) prefetched.update(dict((i.key(), i) for i in entities if i)) profiles = prefetched.values() list_helper.distributeParentKeys(scores, prefetched) keys = list_helper.collectKeys(GCIProfile.student_info, entities) keys = list(set(keys)) prefetched = {} print "Fetching StudentInfo..." for i in xrange(0, len(keys), 100): chunk = keys[i:i+100] entities = db.get(chunk) prefetched.update(dict((i.key(), i) for i in entities if i)) studentInfos = prefetched.values() list_helper.distributeKeys(GCIProfile.student_info, profiles, prefetched) i = 0 while i < len(profiles): try: profile = profiles[i] consent_form = profile.student_info.consent_form student_id_form = profile.student_info.student_id_form if not consent_form or not student_id_form: print "At least one form missing from %s" % profile.link_id else: _saveForm(profile, consent_form, 'consent-form', outputdir) _saveForm(profile, consent_form, 'student-id-form', outputdir) except Exception, e: continue i += 1
def downloadStudentForms(options): from google.appengine.ext import db from soc.views.helper import lists as list_helper from soc.modules.gci.models.profile import GCIProfile from soc.modules.gci.models.profile import GCIStudentInfo from soc.modules.gci.models.program import GCIProgram from soc.modules.gci.models.score import GCIScore if not options.program_path: print "--program_path or -p option is required" program = GCIProgram.get_by_key_name(options.program_path) outputdir = os.path.abspath(options.outputdir) if not os.path.exists(outputdir): os.mkdir(outputdir) if not os.path.isdir(outputdir): print "Could not create output dir: %s" % outputdir q = lambda: GCIScore.all().filter("program =", program) print "Fetching GCIScore..." scores = list(i for i in interactive.deepFetch(q)) keys = list_helper.collectParentKeys(scores) keys = list(set(keys)) prefetched = {} print "Fetching Profile..." for i in xrange(0, len(keys), 100): chunk = keys[i:i + 100] entities = db.get(chunk) prefetched.update(dict((i.key(), i) for i in entities if i)) profiles = prefetched.values() list_helper.distributeParentKeys(scores, prefetched) keys = list_helper.collectKeys(GCIProfile.student_info, entities) keys = list(set(keys)) prefetched = {} print "Fetching StudentInfo..." for i in xrange(0, len(keys), 100): chunk = keys[i:i + 100] entities = db.get(chunk) prefetched.update(dict((i.key(), i) for i in entities if i)) studentInfos = prefetched.values() list_helper.distributeKeys(GCIProfile.student_info, profiles, prefetched) i = 0 while i < len(profiles): try: profile = profiles[i] consent_form = profile.student_info.consent_form student_id_form = profile.student_info.student_id_form if not consent_form or not student_id_form: print "At least one form missing from %s" % profile.link_id else: _saveForm(profile, consent_form, 'consent-form', outputdir) _saveForm(profile, consent_form, 'student-id-form', outputdir) except Exception, e: continue i += 1