def convertGCIProfileDBEntityGroup(profile_key):
"""Converts DB based part of entity group associated with the specified
profile.
Args:
profile_key: db.Key of the profile to process.
"""
to_delete = []
do_put = True
org_scores = GCIOrgScore.all().ancestor(profile_key).fetch(1000)
for org_score in org_scores:
new_org_score = _convertParent(org_score)
if do_put:
new_org_score.put()
to_delete.append(org_score)
scores = GCIScore.all().ancestor(profile_key).fetch(1000)
for score in scores:
new_score = _convertParent(score)
if do_put:
new_score.put()
to_delete.append(score)
db.delete(to_delete)
def allScoresForProgramQuery(program):
"""Returns the query to fetch all the scores for the specified program.
Args:
program: GCIProgram entity for which the query should filter the program
"""
return GCIScore.all().filter('program =', program)
def getListData(self):
idx = lists.getListIndex(self.request)
if idx != self.idx:
return None
q = GCIScore.all()
q.filter('program', self.data.program)
starter = lists.keyStarter
def prefetcher(entities):
keys = []
for entity in entities:
key = entity.parent_key()
if key:
keys.append(key)
entities = db.get(keys)
sp = dict((i.key(), i) for i in entities if i)
return ([sp], {})
response_builder = lists.RawQueryContentResponseBuilder(
self.request, self._list_config, q, starter, prefetcher=prefetcher)
return response_builder.build()
def allScoresForProgramQuery(program):
"""Returns the query to fetch all the scores for the specified program.
Args:
program: GCIProgram entity for which the query should filter the program
"""
return GCIScore.all().filter('program =', program)
def update_ranking_txn():
logging.info("updateScore txn starts for task %s" % task_key.id())
query = GCIScore.all().ancestor(student)
score = query.get()
# create a new GCIStore entity if one does not exist yet
if not score:
logging.info("score entity is being created")
score = GCIScore(parent=student, program=program)
# check if the task has been included in the score
if task_key not in score.tasks:
score.points += POINTS[task.difficulty_level]
score.tasks.append(task_key)
# TODO(dhans): optimize it; sometimes, put may not be needed
logging.info("score put")
score.put()
logging.info("score put returned")
query = GCIStudentInfo.all().ancestor(student)
student_info = query.get()
# set in student info that the student has completed a task
if not student_info.task_closed:
student_info.task_closed = True
student_info.put()
def convertGCIProfileDBEntityGroup(profile_key):
"""Converts DB based part of entity group associated with the specified
profile.
Args:
profile_key: db.Key of the profile to process.
"""
to_delete = []
do_put = True
org_scores = GCIOrgScore.all().ancestor(profile_key).fetch(1000)
for org_score in org_scores:
new_org_score = _convertParent(org_score)
if do_put:
new_org_score.put()
to_delete.append(org_score)
scores = GCIScore.all().ancestor(profile_key).fetch(1000)
for score in scores:
new_score = _convertParent(score)
if do_put:
new_score.put()
to_delete.append(score)
db.delete(to_delete)
def get(profile):
"""Gets the score entity associated with the specified profile.
Args:
profile: GCIProfile entity to retrieve a score for
"""
query = GCIScore.all().ancestor(profile)
return query.get()
def get(profile):
"""Gets the score entity associated with the specified profile.
Args:
profile: GCIProfile entity to retrieve a score for
"""
query = GCIScore.all().ancestor(profile)
return query.get()
def getListData(self):
idx = lists.getListIndex(self.data.request)
if idx == self.LEADERBOARD_LIST_IDX:
q = GCIScore.all()
q.filter('program', self.data.program)
skipper = lambda entity, start: entity.points <= 0
prefetcher = lists.ModelPrefetcher(GCIScore, [], True)
response_builder = lists.RawQueryContentResponseBuilder(
self.data.request, self._list_config, q, lists.keyStarter,
skipper=skipper, prefetcher=prefetcher)
return response_builder.build()
else:
return None
def calculate_score_txn():
query = GCIScore.all().ancestor(student)
score = query.get()
# create a new GCIStore entity if one does not exist yet
if not score:
score = GCIScore(parent=student, program=program)
score.points = points
score.tasks = [task.key() for task in tasks]
score.put()
# set that the student closed a task in GCIStudentInfo
query = GCIStudentInfo.all().ancestor(student)
student_info = query.get()
student_info.task_closed = True
student_info.put()
def calculate_score_txn():
query = GCIScore.all().ancestor(student)
score = query.get()
# create a new GCIStore entity if one does not exist yet
if not score:
score = GCIScore(parent=student, program=program)
score.points = points
score.tasks = [task.key() for task in tasks]
score.put()
# set that the student closed a task in GCIStudentInfo
query = GCIStudentInfo.all().ancestor(student)
student_info = query.get()
student_info.task_closed = True
student_info.put()
def clear(*args, **kwargs):
"""Removes all entities from the datastore.
"""
# TODO(dbentley): If there are more than 1000 instances of any model,
# this method will not clear all instances. Instead, it should continually
# call .all(), delete all those, and loop until .all() is empty.
entities = itertools.chain(*[
Survey.all(),
SurveyRecord.all(),
GCIOrganization.all(),
GSoCTimeline.all(),
GCITimeline.all(),
GSoCProgram.all(),
GSoCProject.all(),
GSoCProposal.all(),
GCIProgram.all(),
GCIScore.all(),
GSoCStudentInfo.all(),
GCIStudentInfo.all(),
GCITask.all(),
Sponsor.all(),
Site.all(),
Document.all(),
# The below models are all subclasses of ndb.Model and therefore must
# use .query() to return all instances instead of .all().
soc_org_model.SOCOrganization.query(),
profile_model.Profile.query(),
soc_profile.SOCStudentData.query(),
user.User.query(),
address.Address.query(),
contact.Contact.query()
])
try:
for entity in entities:
if isinstance(entity, ndb.Model):
entity.key.delete()
else:
entity.delete()
except db.Timeout:
return http.HttpResponseRedirect('#')
memcache.flush_all()
return http.HttpResponse('Done')
def clear(*args, **kwargs):
"""Removes all entities from the datastore.
"""
# TODO(dbentley): If there are more than 1000 instances of any model,
# this method will not clear all instances. Instead, it should continually
# call .all(), delete all those, and loop until .all() is empty.
entities = itertools.chain(*[
Survey.all(),
SurveyRecord.all(),
GCIOrganization.all(),
GSoCTimeline.all(),
GCITimeline.all(),
GSoCProgram.all(),
GSoCProject.all(),
GSoCProposal.all(),
GCIProgram.all(),
GCIScore.all(),
GSoCStudentInfo.all(),
GCIStudentInfo.all(),
GCITask.all(),
Sponsor.all(),
Site.all(),
Document.all(),
# The below models are all subclasses of ndb.Model and therefore must
# use .query() to return all instances instead of .all().
soc_org_model.SOCOrganization.query(),
profile_model.Profile.query(),
soc_profile.SOCStudentData.query(),
user.User.query(),
address.Address.query(),
contact.Contact.query()
])
try:
for entity in entities:
if isinstance(entity, ndb.Model):
entity.key.delete()
else:
entity.delete()
except db.Timeout:
return http.HttpResponseRedirect('#')
memcache.flush_all()
return http.HttpResponse('Done')
def winnersForProgram(data):
"""Returns the winners for the program.
The number of winners chosen is configurable from the program edit page. The
return datastructure is a dictionary with profile keys of winners as keys
with values as dictionaries containing the name, number of task and the
points scored.
Args:
data: The RequestData object.
"""
program = data.program
q = GCIScore.all()
q.filter('program', program)
q.filter('points >', 0)
q.order('-points')
scores = q.fetch(program.nr_winners)
profile_keys = [s.parent_key() for s in scores]
profiles = db.get(profile_keys)
winners = SortedDict()
for score in scores:
winners[score.parent_key()] = {
'score': score,
}
for profile in profiles:
winner = winners[profile.key()]
winner['profile'] = profile
winner['completed_tasks_link'] = data.redirect.profile(
profile.link_id).urlOf(url_names.GCI_STUDENT_TASKS)
if profile.avatar:
avatar_groups = re.findall(forms.RE_AVATAR_COLOR, profile.avatar)
# Being a bit pessimistic
if avatar_groups:
# We only want the first match, so pick group[0]
name, prefix = avatar_groups[0]
winner['avatar_name'] = '%s-%s.jpg' % (name, prefix)
winner['avatar_prefix'] = prefix
return winners
def winnersForProgram(data):
"""Returns the winners for the program.
The number of winners chosen is configurable from the program edit page. The
return datastructure is a dictionary with profile keys of winners as keys
with values as dictionaries containing the name, number of task and the
points scored.
Args:
data: The RequestData object.
"""
program = data.program
q = GCIScore.all()
q.filter('program', program)
q.filter('points >', 0)
q.order('-points')
scores = q.fetch(program.nr_winners)
profile_keys = [s.parent_key() for s in scores]
profiles = db.get(profile_keys)
winners = SortedDict()
for score in scores:
winners[score.parent_key()] = {
'score': score,
}
for profile in profiles:
winner = winners[profile.key()]
winner['profile'] = profile
winner['completed_tasks_link'] = data.redirect.profile(
profile.link_id).urlOf(url_names.GCI_STUDENT_TASKS)
if profile.avatar:
avatar_groups = re.findall(forms.RE_AVATAR_COLOR, profile.avatar)
# Being a bit pessimistic
if avatar_groups:
# We only want the first match, so pick group[0]
name, prefix = avatar_groups[0]
winner['avatar_name'] = '%s-%s.jpg' % (name, prefix)
winner['avatar_prefix'] = prefix
return winners
def clearGCIRanking(self, request, *args, **kwargs):
"""Clears student ranking for a program with the specified key_name.
"""
key_name = "%s/%s" % (kwargs["sponsor"], kwargs["program"])
program = GCIProgram.get_by_key_name(key_name)
if not program:
logging.warning("Enqueued recalculate ranking task for non-existing program: %s", key_name)
return responses.terminateTask()
q = GCIScore.all()
q.filter("program", program)
rankings = q.fetch(500)
while rankings:
db.delete(rankings)
rankings = q.fetch(500)
return responses.terminateTask()
def clear(*args, **kwargs):
"""Removes all entities from the datastore.
"""
# TODO(dbentley): If there are more than 1000 instances of any model,
# this method will not clear all instances. Instead, it should continually
# call .all(), delete all those, and loop until .all() is empty.
entities = itertools.chain(*[
Notification.all(),
GCIStudent.all(),
Survey.all(),
SurveyRecord.all(),
StudentProposal.all(),
GSoCOrganization.all(),
GCIOrganization.all(),
GSoCTimeline.all(),
GCITimeline.all(),
GSoCProgram.all(),
GSoCProfile.all(),
GCIProfile.all(),
GSoCProposal.all(),
GCIProgram.all(),
GCIScore.all(),
GSoCStudentInfo.all(),
GCIStudentInfo.all(),
GCITask.all(),
Host.all(),
Sponsor.all(),
User.all(),
Site.all(),
Document.all(),
])
try:
for entity in entities:
entity.delete()
except db.Timeout:
return http.HttpResponseRedirect('#')
# pylint: disable=E1101
memcache.flush_all()
return http.HttpResponse('Done')
def getListData(self):
idx = lists.getListIndex(self.data.request)
if idx == self.LEADERBOARD_LIST_IDX:
q = GCIScore.all()
q.filter('program', self.data.program)
skipper = lambda entity, start: entity.points <= 0
prefetcher = lists.ModelPrefetcher(GCIScore, [], True)
response_builder = lists.RawQueryContentResponseBuilder(
self.data.request,
self._list_config,
q,
lists.keyStarter,
skipper=skipper,
prefetcher=prefetcher)
return response_builder.build()
else:
return None
def clearGCIRanking(self, request, *args, **kwargs):
"""Clears student ranking for a program with the specified key_name.
"""
key_name = '%s/%s' % (kwargs['sponsor'], kwargs['program'])
program = GCIProgram.get_by_key_name(key_name)
if not program:
logging.warning(
'Enqueued recalculate ranking task for non-existing program: %s',
key_name)
return responses.terminateTask()
q = GCIScore.all()
q.filter('program', program)
rankings = q.fetch(500)
while rankings:
db.delete(rankings)
rankings = q.fetch(500)
return responses.terminateTask()
def update_ranking_txn():
logging.info("updateScore txn starts for task %s", task_key.id())
query = GCIScore.all().ancestor(student)
score = query.get()
# create a new GCIStore entity if one does not exist yet
if not score:
score = GCIScore(parent=student, program=program)
# check if the task has been included in the score
if task_key not in score.tasks:
if not task.points_invalidated:
score.points += POINTS[task.difficulty_level]
score.tasks.append(task_key)
score.put()
query = GCIStudentInfo.all().ancestor(student)
student_info = query.get()
# set in student info that the student has completed a task
if not student_info.task_closed:
student_info.task_closed = True
student_info.put()
def update_ranking_txn():
logging.info("updateScore txn starts for task %s", task_key.id())
query = GCIScore.all().ancestor(student)
score = query.get()
# create a new GCIStore entity if one does not exist yet
if not score:
score = GCIScore(parent=student, program=program)
# check if the task has been included in the score
if task_key not in score.tasks:
if not task.points_invalidated:
score.points += POINTS[task.difficulty_level]
score.tasks.append(task_key)
score.put()
query = GCIStudentInfo.all().ancestor(student)
student_info = query.get()
# set in student info that the student has completed a task
if not student_info.task_closed:
student_info.task_closed = True
student_info.put()
def downloadStudentForms(options):
from google.appengine.ext import db
from soc.views.helper import lists as list_helper
from soc.modules.gci.models.profile import GCIProfile
from soc.modules.gci.models.profile import GCIStudentInfo
from soc.modules.gci.models.program import GCIProgram
from soc.modules.gci.models.score import GCIScore
if not options.program_path:
print "--program_path or -p option is required"
program = GCIProgram.get_by_key_name(options.program_path)
outputdir = os.path.abspath(options.outputdir)
if not os.path.exists(outputdir):
os.mkdir(outputdir)
if not os.path.isdir(outputdir):
print "Could not create output dir: %s" % outputdir
q = lambda: GCIScore.all().filter("program =", program)
print "Fetching GCIScore..."
scores = list(i for i in interactive.deepFetch(q))
keys = list_helper.collectParentKeys(scores)
keys = list(set(keys))
prefetched = {}
print "Fetching Profile..."
for i in xrange(0, len(keys), 100):
chunk = keys[i:i+100]
entities = db.get(chunk)
prefetched.update(dict((i.key(), i) for i in entities if i))
profiles = prefetched.values()
list_helper.distributeParentKeys(scores, prefetched)
keys = list_helper.collectKeys(GCIProfile.student_info, entities)
keys = list(set(keys))
prefetched = {}
print "Fetching StudentInfo..."
for i in xrange(0, len(keys), 100):
chunk = keys[i:i+100]
entities = db.get(chunk)
prefetched.update(dict((i.key(), i) for i in entities if i))
studentInfos = prefetched.values()
list_helper.distributeKeys(GCIProfile.student_info, profiles, prefetched)
i = 0
while i < len(profiles):
try:
profile = profiles[i]
consent_form = profile.student_info.consent_form
student_id_form = profile.student_info.student_id_form
if not consent_form or not student_id_form:
print "At least one form missing from %s" % profile.link_id
else:
_saveForm(profile, consent_form, 'consent-form', outputdir)
_saveForm(profile, consent_form, 'student-id-form', outputdir)
except Exception, e:
continue
i += 1
def downloadStudentForms(options):
from google.appengine.ext import db
from soc.views.helper import lists as list_helper
from soc.modules.gci.models.profile import GCIProfile
from soc.modules.gci.models.profile import GCIStudentInfo
from soc.modules.gci.models.program import GCIProgram
from soc.modules.gci.models.score import GCIScore
if not options.program_path:
print "--program_path or -p option is required"
program = GCIProgram.get_by_key_name(options.program_path)
outputdir = os.path.abspath(options.outputdir)
if not os.path.exists(outputdir):
os.mkdir(outputdir)
if not os.path.isdir(outputdir):
print "Could not create output dir: %s" % outputdir
q = lambda: GCIScore.all().filter("program =", program)
print "Fetching GCIScore..."
scores = list(i for i in interactive.deepFetch(q))
keys = list_helper.collectParentKeys(scores)
keys = list(set(keys))
prefetched = {}
print "Fetching Profile..."
for i in xrange(0, len(keys), 100):
chunk = keys[i:i + 100]
entities = db.get(chunk)
prefetched.update(dict((i.key(), i) for i in entities if i))
profiles = prefetched.values()
list_helper.distributeParentKeys(scores, prefetched)
keys = list_helper.collectKeys(GCIProfile.student_info, entities)
keys = list(set(keys))
prefetched = {}
print "Fetching StudentInfo..."
for i in xrange(0, len(keys), 100):
chunk = keys[i:i + 100]
entities = db.get(chunk)
prefetched.update(dict((i.key(), i) for i in entities if i))
studentInfos = prefetched.values()
list_helper.distributeKeys(GCIProfile.student_info, profiles, prefetched)
i = 0
while i < len(profiles):
try:
profile = profiles[i]
consent_form = profile.student_info.consent_form
student_id_form = profile.student_info.student_id_form
if not consent_form or not student_id_form:
print "At least one form missing from %s" % profile.link_id
else:
_saveForm(profile, consent_form, 'consent-form', outputdir)
_saveForm(profile, consent_form, 'student-id-form', outputdir)
except Exception, e:
continue
i += 1
