def recalculateGCIStudentRanking(request, *args, **kwargs):
"""Recalculates GCI Student Ranking for the specified student.
"""
student = gci_student_logic.getFromKeyName(kwargs['key_name'])
if not student:
return responses.terminateTask()
# find ranking entity for the student and clear it
filter = {'student': student}
ranking = gci_student_ranking_logic.getForFields(filter=filter,
unique=True)
ranking.points = 0
ranking.tasks = []
ranking.put()
# get all the tasks that the student has completed
filter = {
'student': student,
'status': 'Closed',
}
tasks = gci_task_logic.getForFields(filter=filter)
for task in tasks:
gci_student_ranking_logic.updateRanking(task)
return responses.terminateTask()
def accept_proposals(request, *args, **kwargs):
"""Accept proposals for an organization
"""
params = request.POST
# Setup an artifical request deadline
timelimit = int(params["timelimit"])
timekeeper = Timekeeper(timelimit)
# Query proposals based on status
org = org_logic.getFromKeyName(params["orgkey"])
proposals = student_proposal_logic.getProposalsToBeAcceptedForOrg(org)
# Accept proposals
try:
for remain, proposal in timekeeper.iterate(proposals):
logging.info("accept %s %s %s", remain, org.key(), proposal.key())
accept_proposal(proposal)
accept_proposal_email(proposal)
# Requeue this task for continuation
except DeadlineExceededError:
taskqueue.add(url=request.path, params=params)
return responses.terminateTask()
# Reject remaining proposals
taskqueue.add(url=params["nextpath"], params=params)
return responses.terminateTask()
def recalculateGCIStudentRanking(request, *args, **kwargs):
"""Recalculates GCI Student Ranking for the specified student.
"""
student = gci_student_logic.getFromKeyName(kwargs['key_name'])
if not student:
return responses.terminateTask()
# find ranking entity for the student and clear it
filter = {
'student': student
}
ranking = gci_student_ranking_logic.getForFields(filter=filter,
unique=True)
ranking.points = 0
ranking.tasks = []
ranking.put()
# get all the tasks that the student has completed
filter = {
'student': student,
'status': 'Closed',
}
tasks = gci_task_logic.getForFields(filter=filter)
for task in tasks:
gci_student_ranking_logic.updateRanking(task)
return responses.terminateTask()
def recalculateForStudent(self, request, *args, **kwargs):
"""Recalculates GCI Student Ranking for the specified student.
Args in POST:
key: The string version of the key for the GCIProfile entity
representing the student.
"""
post_dict = request.POST
key = db.Key(post_dict['key'])
student = GCIProfile.get(key)
if not student:
logging.warning('Enqueued task to recalculate ranking for '
'non-existent student %s' %(key))
return responses.terminateTask()
# get all the tasks that the student has completed
q = GCITask.all()
q.filter('student', student)
q.filter('status', 'Closed')
tasks = q.fetch(1000)
ranking_logic.calculateRankingForStudent(student, tasks)
return responses.terminateTask()
def recalculateForStudent(self, request, *args, **kwargs):
"""Recalculates GCI Student Ranking for the specified student.
Args in POST:
key: The string version of the key for the GCIProfile entity
representing the student.
"""
post_dict = request.POST
key = db.Key(post_dict['key'])
student = GCIProfile.get(key)
if not student:
logging.warning(
'Enqueued task to recalculate ranking for '
'non-existent student %s', key)
return responses.terminateTask()
# get all the tasks that the student has completed
q = GCITask.all()
q.filter('student', student)
q.filter('status', 'Closed')
tasks = q.fetch(1000)
score_logic.calculateRankingForStudent(student, tasks)
return responses.terminateTask()
def accept_proposals(request, *args, **kwargs):
"""Accept proposals for an organization
"""
params = request.POST
# Setup an artifical request deadline
timelimit = int(params["timelimit"])
timekeeper = Timekeeper(timelimit)
# Query proposals based on status
org = org_logic.getFromKeyName(params["orgkey"])
proposals = student_proposal_logic.getProposalsToBeAcceptedForOrg(org)
# Accept proposals
try:
for remain, proposal in timekeeper.iterate(proposals):
logging.info("accept %s %s %s", remain, org.key(), proposal.key())
accept_proposal(proposal)
accept_proposal_email(proposal)
# Requeue this task for continuation
except DeadlineExceededError:
taskqueue.add(url=request.path, params=params)
return responses.terminateTask()
# Reject remaining proposals
taskqueue.add(url=params["nextpath"], params=params)
return responses.terminateTask()
def acceptProposals(self, request, *args, **kwargs):
"""Accept proposals for an single organization.
POST Args:
org_key: The key of the organization
"""
params = request.POST
# Setup an artifical request deadline
timekeeper = Timekeeper(20000)
# Query proposals based on status
org = soc_org_model.SOCOrganization.get_by_id(params['org_key'])
proposals = proposal_logic.getProposalsToBeAcceptedForOrg(org)
# Accept proposals
try:
for remain, proposal in timekeeper.iterate(proposals):
logging.info("accept %s %s %s", remain, org.key.id(), proposal.key())
self.acceptProposal(proposal)
# Requeue this task for continuation
except DeadlineExceededError:
taskqueue.add(url=request.path, params=params)
return responses.terminateTask()
# Reject remaining proposals
taskqueue.add(url='/tasks/gsoc/accept_proposals/reject', params=params)
return responses.terminateTask()
def _startShipmentSync(self, request, *args, **kwargs):
"""Start syncing shipment data.
POST Args:
program_key: the key of the program which task is runnig for.
sheet_content: sheet content data in JSON format.
sheet_type: 'usa' or 'intl'
shipment_info_id: id of the shipment info object that task is running
for.
"""
params = dicts.merge(request.POST, request.GET)
redirect = RedirectHelper(None, None)
if 'program_key' not in params:
logging.error("missing program_key in params: '%s'" % params)
return responses.terminateTask()
if 'sheet_content' not in params:
logging.error("missing sheet_content in params: '%s'" % params)
return responses.terminateTask()
if 'sheet_type' not in params:
logging.error("missing sheet_type in params: '%s'" % params)
return responses.terminateTask()
if 'shipment_info_id' not in params:
logging.error("missing shipment_info_id in params: '%s'" % params)
return responses.terminateTask()
self.setProgram(params['program_key'])
self.setShipmentInfo(int(params['shipment_info_id']))
self.__shipment_info.status = 'syncing'
self.__shipment_info.put()
sheet_content = StringIO.StringIO(
simplejson.loads(params['sheet_content']))
sheet_type = params['sheet_type']
sheet_rows = [row for row in csv.reader(sheet_content)]
if sheet_type == 'usa':
column_indexes = self.findColumnIndexes(
sheet_rows[0], self.USA_EXPECTED_COLUMNS)
elif sheet_type == 'intl':
column_indexes = self.findColumnIndexes(
sheet_rows[0], self.INTL_EXPECTED_COLUMNS)
params = {
'program_key': params['program_key'],
'shipment_info_id': params['shipment_info_id'],
'column_indexes': simplejson.dumps(column_indexes),
'sheet_rows': simplejson.dumps(sheet_rows[1:]),
}
taskqueue.add(url=redirect.urlOf('shipment_sync_task_continue'),
params=params)
return responses.terminateTask()
def _startShipmentSync(self, request, *args, **kwargs):
"""Start syncing shipment data.
POST Args:
program_key: the key of the program which task is runnig for.
sheet_content: sheet content data in JSON format.
sheet_type: 'usa' or 'intl'
shipment_info_id: id of the shipment info object that task is running
for.
"""
params = dicts.merge(request.POST, request.GET)
if 'program_key' not in params:
logging.error("missing program_key in params: '%s'", params)
return responses.terminateTask()
if 'sheet_content' not in params:
logging.error("missing sheet_content in params: '%s'", params)
return responses.terminateTask()
if 'sheet_type' not in params:
logging.error("missing sheet_type in params: '%s'", params)
return responses.terminateTask()
if 'shipment_info_id' not in params:
logging.error("missing shipment_info_id in params: '%s'", params)
return responses.terminateTask()
self.setProgram(params['program_key'])
self.setShipmentInfo(int(params['shipment_info_id']))
self.shipment_info.status = 'syncing'
self.shipment_info.put()
sheet_content = StringIO.StringIO(
json.loads(params['sheet_content']))
sheet_type = params['sheet_type']
sheet_rows = [row for row in csv.reader(sheet_content)]
if sheet_type == 'usa':
column_indexes = self.findColumnIndexes(
sheet_rows[0], self.USA_EXPECTED_COLUMNS)
elif sheet_type == 'intl':
column_indexes = self.findColumnIndexes(
sheet_rows[0], self.INTL_EXPECTED_COLUMNS)
params = {
'program_key': params['program_key'],
'shipment_info_id': params['shipment_info_id'],
'column_indexes': json.dumps(column_indexes),
'sheet_rows': json.dumps(sheet_rows[1:]),
}
task_continue_url = links.SOC_LINKER.site(url_names.GSOC_SHIPMENT_TASK_CONTINUE)
taskqueue.add(url=task_continue_url, params=params)
return responses.terminateTask()
def recalculateGCIRanking(self, request, *args, **kwargs):
"""Recalculates student ranking for the entire program.
Args in POST dict:
cursor: Query cursor to figure out where we need to start processing
"""
key_name = '%s/%s' % (kwargs['sponsor'], kwargs['program'])
cursor = request.POST.get('cursor')
program = GCIProgram.get_by_key_name(key_name)
if not program:
logging.warning(
'Enqueued recalculate ranking task for non-existing program: %s',
key_name)
return responses.terminateTask()
# Retrieve the students for the program
q = GCIProfile.all()
q.filter('program', program)
q.filter('is_student', True)
if cursor:
q.with_cursor(cursor)
students = q.fetch(25)
for student in students:
# get all the tasks that the student has completed
task_q = GCITask.all()
task_q.filter('student', student)
task_q.filter('status', 'Closed')
tasks = task_q.fetch(1000)
# calculate score with all the tasks
score_logic.calculateScore(student, tasks, program)
# calculate org score with all the tasks
db.run_in_transaction(org_score_logic.updateOrgScoresTxn(tasks))
if students:
# schedule task to do the rest of the students
params = {
'cursor': q.cursor(),
}
taskqueue.add(queue_name='gci-update',
url=request.path,
params=params)
return responses.terminateTask()
def updateGCIRanking(request, *args, **kwargs):
"""Updates student ranking based on the task passed as post argument.
"""
post_dict = request.POST
task_keyname = post_dict.get('task_keyname')
if not task_keyname:
responses.terminateTask()
task = gci_task_logic.getFromKeyName(str(task_keyname))
gci_student_ranking_logic.updateRanking(task)
return responses.terminateTask()
def updateGCIRanking(request, *args, **kwargs):
"""Updates student ranking based on the task passed as post argument.
"""
post_dict = request.POST
task_keyname = post_dict.get('task_keyname')
if not task_keyname:
responses.terminateTask()
task = gci_task_logic.getFromKeyName(str(task_keyname))
gci_student_ranking_logic.updateRanking(task)
return responses.terminateTask()
def convertProposals(self, request, *args, **kwargs):
"""Start tasks to convert proposals for all organizations.
POST Args:
program_key: the key of the program whose proposals should be converted
org_cursor: the cursor indicating at which org we currently are
"""
params = dicts.merge(request.POST, request.GET)
if 'program_key' not in params:
logging.error("missing program_key in params: '%s'", params)
return responses.terminateTask()
program = GSoCProgram.get_by_key_name(params['program_key'])
if not program:
logging.error("invalid program_key in params: '%s'", params)
return responses.terminateTask()
query = soc_org_model.SOCOrganization.query(
soc_org_model.SOCOrganization.program ==
ndb.Key.from_old_key(program.key()),
soc_org_model.SOCOrganization.status == org_model.Status.ACCEPTED)
org_cursor = params.get('org_cursor')
start_cursor = (
datastore_query.Cursor(urlsafe=urllib.unquote_plus(org_cursor))
if org_cursor else None)
# Add a task for a single organization
organizations, next_cursor, _ = query.fetch_page(
1, start_cursor=start_cursor)
if organizations:
organization = organizations[0]
logging.info(
'Enqueing task to accept proposals for %s.', organization.name)
# Compounded accept/reject taskflow
taskqueue.add(
url='/tasks/gsoc/accept_proposals/accept',
params={
'org_key': organization.key.id(),
})
# Enqueue a new task to do the next organization
params['org_cursor'] = next_cursor.urlsafe()
taskqueue.add(url=request.path, params=params)
# Exit this task successfully
return responses.terminateTask()
def reject_proposals(request, *args, **kwargs):
"""Reject proposals for an org_logic
"""
params = request.POST
# Setup an artifical request deadline
timelimit = int(params["timelimit"])
timekeeper = Timekeeper(timelimit)
# Query proposals
org = org_logic.getFromKeyName(params["orgkey"])
proposals = reject_proposals_query(org)
# Reject proposals
try:
for remain, proposal in timekeeper.iterate(proposals):
logging.info("reject %s %s %s", remain, org.key(), proposal.key())
reject_proposal(proposal)
reject_proposal_email(proposal)
# Requeue this task for continuation
except DeadlineExceededError:
taskqueue.add(url=request.path, params=params)
# Exit this task successfully
return responses.terminateTask()
def recalculateGCIRanking(request, entities, context, *args, **kwargs):
"""Recalculates student ranking for a program with the specified key_name.
"""
program = gci_program_logic.getFromKeyName(kwargs['key_name'])
if not program:
return responses.terminateTask()
# prefetch all task difficulties
all_d = gci_task_model.TaskDifficultyTag.all().fetch(100)
for entity in entities:
# check if the entity refers to the program in scope
if entity.scope.key() != program.key():
continue
# get all the tasks that the student has completed
filter = {
'student': entity,
'status': 'Closed',
}
tasks = gci_task_logic.getForFields(filter=filter)
# calculate ranking with all the tasks
gci_student_ranking_logic.calculateRankingForStudent(entity, tasks, all_d)
# this task should not be repeated after the program is over
timeline = program.timeline
if timeline_helper.isAfterEvent(timeline, 'program_end'):
raise responses.DoNotRepeatException()
def reject_proposals(request, *args, **kwargs):
"""Reject proposals for an org_logic
"""
params = request.POST
# Setup an artifical request deadline
timelimit = int(params["timelimit"])
timekeeper = Timekeeper(timelimit)
# Query proposals
org = org_logic.getFromKeyName(params["orgkey"])
proposals = reject_proposals_query(org)
# Reject proposals
try:
for remain, proposal in timekeeper.iterate(proposals):
logging.info("reject %s %s %s", remain, org.key(), proposal.key())
reject_proposal(proposal)
reject_proposal_email(proposal)
# Requeue this task for continuation
except DeadlineExceededError:
taskqueue.add(url=request.path, params=params)
# Exit this task successfully
return responses.terminateTask()
def rejectProposals(self, request, *args, **kwargs):
"""Reject proposals for an single organization.
"""
params = request.POST
# Setup an artifical request deadline
timekeeper = Timekeeper(20000)
# Query proposals
org = soc_org_model.SOCOrganization.get_by_id(params['org_key'])
q = GSoCProposal.all()
q.filter('org', org.key.to_old_key())
q.filter('status', 'pending')
# Reject proposals
try:
for remain, proposal in timekeeper.iterate(q):
logging.info("reject %s %s %s", remain, org.key.id(), proposal.key())
self.rejectProposal(proposal)
# Requeue this task for continuation
except DeadlineExceededError:
taskqueue.add(url=request.path, params=params)
# Exit this task successfully
return responses.terminateTask()
def recalculateGCIRanking(request, entities, context, *args, **kwargs):
"""Recalculates student ranking for a program with the specified key_name.
"""
program = gci_program_logic.getFromKeyName(kwargs['key_name'])
if not program:
return responses.terminateTask()
# prefetch all task difficulties
all_d = gci_task_model.TaskDifficultyTag.all().fetch(100)
for entity in entities:
# check if the entity refers to the program in scope
if entity.scope.key() != program.key():
continue
# get all the tasks that the student has completed
filter = {
'student': entity,
'status': 'Closed',
}
tasks = gci_task_logic.getForFields(filter=filter)
# calculate ranking with all the tasks
gci_student_ranking_logic.calculateRankingForStudent(
entity, tasks, all_d)
# this task should not be repeated after the program is over
timeline = program.timeline
if timeline_helper.isAfterEvent(timeline, 'program_end'):
raise responses.DoNotRepeatException()
def collectTask(self,
request,
access_type,
page_name=None,
params=None,
**kwargs):
"""Task for collecting statistics that will be executed by Task Queue API.
"""
logic = params['logic']
link_id = kwargs['link_id']
scope_path = kwargs['scope_path']
statistic = self._getStatisticEntity(link_id, scope_path, logic)
if statistic is None:
raise task_responses.FatalTaskError
statistic, completed = logic.collectDispatcher(statistic)
self._updateCache(statistic, link_id, scope_path, logic)
if completed:
self._updateCacheList(statistic, scope_path, logic)
statistic.put()
else:
task_responses.startTask(url=request.path, queue_name='statistic')
return task_responses.terminateTask()
def recalculateGCIRanking(self, request, *args, **kwargs):
"""Recalculates student ranking for the entire program.
Args in POST dict:
cursor: Query cursor to figure out where we need to start processing
"""
key_name = '%s/%s' % (kwargs['sponsor'], kwargs['program'])
cursor = request.POST.get('cursor')
program = GCIProgram.get_by_key_name(key_name)
if not program:
logging.warning(
'Enqueued recalculate ranking task for non-existing '
'program: %s' %key_name)
return responses.terminateTask()
# Retrieve the students for the program
q = GCIProfile.all()
q.filter('scope', program)
q.filter('is_student', True)
if cursor:
q.with_cursor(cursor)
students = q.fetch(25)
for student in students:
# get all the tasks that the student has completed
task_q = GCITask.all()
task_q.filter('student', student)
task_q.filter('status', 'Closed')
tasks = task_q.fetch(1000)
# calculate ranking with all the tasks
# ranking_logic.calculateRankingForStudent(student, tasks)
ranking_logic.calculateScore(student, tasks, program)
if students:
# schedule task to do the rest of the students
params = {
'cursor': q.cursor(),
}
taskqueue.add(queue_name='gci-update', url=request.path, params=params)
return responses.terminateTask()
def recalculateGCIRanking(self, request, *args, **kwargs):
"""Recalculates student ranking for the entire program.
Args in POST dict:
cursor: Query cursor to figure out where we need to start processing
"""
key_name = "%s/%s" % (kwargs["sponsor"], kwargs["program"])
cursor = request.POST.get("cursor")
program = GCIProgram.get_by_key_name(key_name)
if not program:
logging.warning("Enqueued recalculate ranking task for non-existing program: %s", key_name)
return responses.terminateTask()
# Retrieve the students for the program
q = GCIProfile.all()
q.filter("program", program)
q.filter("is_student", True)
if cursor:
q.with_cursor(cursor)
students = q.fetch(25)
for student in students:
# get all the tasks that the student has completed
task_q = GCITask.all()
task_q.filter("student", student)
task_q.filter("status", "Closed")
tasks = task_q.fetch(1000)
# calculate score with all the tasks
score_logic.calculateScore(student, tasks, program)
# calculate org score with all the tasks
db.run_in_transaction(org_score_logic.updateOrgScoresTxn(tasks))
if students:
# schedule task to do the rest of the students
params = {"cursor": q.cursor()}
taskqueue.add(queue_name="gci-update", url=request.path, params=params)
return responses.terminateTask()
def assignSlots(request, *args, **kwargs):
"""Sets the slots attribute for each organization entity
POST Args:
slots: an org_key:num_slots JSON dictionary
"""
# Setup an artifical request deadline
timelimit = int(request.REQUEST.get("timelimit", 20000))
timekeeper = Timekeeper(timelimit)
program_key = request.REQUEST.get("programkey")
last_key = request.REQUEST.get("lastkey", "")
program = program_logic.getFromKeyName(program_key)
# Copy for modification below
params = request.POST.copy()
params["timelimit"] = timelimit
# Parse the JSON org:slots dictionary
slots = simplejson.loads(program.slots_allocation)
org_keys = [i for i in sorted(slots.keys()) if i > last_key]
logging.info(org_keys)
# Assign slots for each organization
try:
for clock, org_key in timekeeper.iterate(org_keys):
logging.info("%s %s %s", request.path, clock, org_key)
org_slots = slots[org_key]
# Get the organization entity
org = org_logic.getFromKeyFields({
'link_id': org_key,
'scope_path': program_key,
})
if not org:
logging.error("no such org '%s'/'%s'" % (program_key, org_key))
continue
# Count proposals and mentors
org.slots = int(org_slots['slots'])
org.nr_applications, org.nr_mentors = countProposals(org)
# Update the organization entity
org.put()
# Mark the organization as done
last_key = org_key
# Requeue this task for continuation
except DeadlineExceededError:
params["lastkey"] = last_key
taskqueue.add(url=request.path, params=params)
# Exit this task successfully
return responses.terminateTask()
def wrapper(request, *args, **kwargs):
"""Decorator wrapper method
"""
try:
return func(request, *args, **kwargs)
except task_responses.FatalTaskError, error:
logging.exception(error)
return task_responses.terminateTask()
def wrapper(request, *args, **kwargs):
"""Decorator wrapper method
"""
try:
return func(request, *args, **kwargs)
except task_responses.FatalTaskError, error:
logging.exception(error)
return task_responses.terminateTask()
def updateRankingWithTask(self, request, *args, **kwargs):
"""Updates student ranking based on the task passed as post argument.
Args in POST dict:
id: The (numeric) id of the task to update the ranking for
"""
post_dict = request.POST
id = int(post_dict.get('id'))
task = GCITask.get_by_id(id)
if not task:
logging.warning('Ranking update queued for non-existing task: %s' %id)
responses.terminateTask()
ranking_logic.updateScore(task)
logging.info("ranking_update updateRankingWithTask ends")
return responses.terminateTask()
def clearGCIRanking(self, request, *args, **kwargs):
"""Clears student ranking for a program with the specified key_name.
"""
key_name = "%s/%s" % (kwargs["sponsor"], kwargs["program"])
program = GCIProgram.get_by_key_name(key_name)
if not program:
logging.warning("Enqueued recalculate ranking task for non-existing program: %s", key_name)
return responses.terminateTask()
q = GCIScore.all()
q.filter("program", program)
rankings = q.fetch(500)
while rankings:
db.delete(rankings)
rankings = q.fetch(500)
return responses.terminateTask()
def updateRankingWithTask(self, request, *args, **kwargs):
"""Updates student ranking based on the task passed as post argument.
Args in POST dict:
id: The (numeric) id of the task to update the ranking for
"""
post_dict = request.POST
id = int(post_dict.get('id'))
task = GCITask.get_by_id(id)
if not task:
logging.warning('Ranking update queued for non-existing task: %s',
id)
responses.terminateTask()
score_logic.updateScore(task)
logging.info("ranking_update updateRankingWithTask ends")
return responses.terminateTask()
def clearGCIRanking(self, request, *args, **kwargs):
"""Clears student ranking for a program with the specified key_name.
"""
key_name = '%s/%s' % (kwargs['sponsor'], kwargs['program'])
program = GCIProgram.get_by_key_name(key_name)
if not program:
logging.warning(
'Enqueued recalculate ranking task for non-existing program: %s',
key_name)
return responses.terminateTask()
q = GCIScore.all()
q.filter('program', program)
rankings = q.fetch(500)
while rankings:
db.delete(rankings)
rankings = q.fetch(500)
return responses.terminateTask()
def studentIterator(student_profile_function, request, **kwargs):
"""Applies a function to every student profile in a program.
Args:
student_profile_function: A function that accepts a single
student profile_model.GCIProfile as an argument.
request: A RequestData object.
**kwargs: Keyword arguments associated with the request.
Returns:
An HttpResponse object.
"""
# TODO(nathaniel): Call a utility function for this key_name.
key_name = '%s/%s' % (kwargs['sponsor'], kwargs['program'])
cursor = request.POST.get('cursor')
program = program_model.GCIProgram.get_by_key_name(key_name)
if not program:
logging.warning('Enqueued task for nonexistant program %s', key_name)
return responses.terminateTask()
# Retrieve the students for the program.
query = profile_model.GCIProfile.all()
# TODO(nathaniel): These string literals should be constants somewhere.
query.filter('program', program)
query.filter('is_student', True)
if cursor:
query.with_cursor(cursor)
student_profiles = query.fetch(25)
for student_profile in student_profiles:
student_profile_function(student_profile)
if student_profiles:
# Schedule task to do the rest of the students.
params = {'cursor': query.cursor()}
taskqueue.add(queue_name='gci-update', url=request.path, params=params)
return responses.terminateTask()
def manageModelsStatus(request, *args, **kwargs):
"""Sets status of the roles queried by the fields given by POST.
"""
post_dict = request.POST
new_status = post_dict.get('new_status')
if not new_status:
if not status_retriever or not callable(status_retriever):
return error_handler.logErrorAndReturnOK(
'No valid status can be set by the manageModelStatus.')
if not 'fields' in post_dict:
error_handler.logErrorAndReturnOK(
'No fields to filter on found for manageModelStatus.')
fields = pickle.loads(str(post_dict['fields']))
entities = entity_logic.getForFields(fields, limit=BATCH_SIZE)
for entity in entities:
if new_status:
status = new_status
else:
status = status_retriever(entity)
entity.status = status
db.put(entities)
if len(entities) == BATCH_SIZE:
# we might not have exhausted all the roles that can be updated,
# so start the same task again
context = post_dict.copy()
responses.startTask(request.path, context=context)
return responses.terminateTask()
# exhausted all the entities the task has been completed
return responses.terminateTask()
def manageModelsStatus(request, *args, **kwargs):
"""Sets status of the roles queried by the fields given by POST.
"""
post_dict = request.POST
new_status = post_dict.get('new_status')
if not new_status:
if not status_retriever or not callable(status_retriever):
return error_handler.logErrorAndReturnOK(
'No valid status can be set by the manageModelStatus.')
if not 'fields' in post_dict:
error_handler.logErrorAndReturnOK(
'No fields to filter on found for manageModelStatus.')
fields = pickle.loads(str(post_dict['fields']))
entities = entity_logic.getForFields(fields, limit=BATCH_SIZE)
for entity in entities:
if new_status:
status = new_status
else:
status = status_retriever(entity)
entity.status = status
db.put(entities)
if len(entities) == BATCH_SIZE:
# we might not have exhausted all the roles that can be updated,
# so start the same task again
context = post_dict.copy()
responses.startTask(request.path, context=context)
return responses.terminateTask()
# exhausted all the entities the task has been completed
return responses.terminateTask()
def sendMail(request):
"""Sends out an email that is stored in the datastore.
The POST request should contain the following entries:
mail_key: Datastore key for an Email entity.
"""
post_dict = request.POST
mail_key = post_dict.get('mail_key', None)
if not mail_key:
return error_handler.logErrorAndReturnOK('No email key specified')
mail_entity = email.Email.get(mail_key)
if not mail_entity:
return error_handler.logErrorAndReturnOK(
'No email entity found for key %s' %mail_key)
# construct the EmailMessage from the given context
loaded_context = simplejson.loads(mail_entity.context)
context = {}
for key, value in loaded_context.iteritems():
# If we don't do this python will complain about kwargs not being
# strings.
context[str(key)] = value
logging.info('Sending %s' %context)
message = mail.EmailMessage(**context)
try:
message.check_initialized()
except:
logging.error('This message was not properly initialized')
mail_entity.delete()
return responses.terminateTask()
def txn():
"""Transaction that ensures the deletion of the Email entity only if
the mail has been successfully sent.
"""
mail_entity.delete()
message.send()
try:
db.RunInTransaction(txn)
except mail.Error, exception:
# shouldn't happen because validate has been called, keeping the Email
# entity for study purposes.
return error_handler.logErrorAndReturnOK(exception)
def wrapper(func):
def iterative_wrapped(request, *args, **kwargs):
"""Decorator wrapper method
Args:
request: Django HTTP Request object
request.POST usage:
fields: a JSON dict for the properties that the entities should have.
This updates values from the task_default entry.
start_key: the key of the next entity to fetch
Returns:
Standard HTTP Django response
"""
post_dict = request.POST
fields = task_default.get('fields', {})
if 'fields' in post_dict:
fields.update(pickle.loads(str(post_dict['fields'])))
start_key = task_default.get('start_key', None)
if 'start_key' in post_dict:
# get the key where to start this iteration
start_key = post_dict['start_key']
if start_key:
start_key = db.Key(start_key)
# get the entities for this iteration
entities, next_start_key = logic.getBatchOfData(
filter=fields, start_key=start_key)
# copy the post_dict so that the wrapped function can edit what it needs
context = post_dict.copy()
# when true, the iterative task will not be repeated when completed
do_not_repeat = False
try:
func(request,
entities=entities,
context=context,
*args,
**kwargs)
except task_responses.DoNotRepeatException, exception:
do_not_repeat = True
except task_responses.FatalTaskError, error:
logging.debug(post_dict)
logging.error(error)
return task_responses.terminateTask()
def assignProgramSlots(request, *args, **kwargs):
"""Assign slots for organizations within a program
Gets the slot assignment data as a JSON string from the program
and enqueues a task to process the slot assignments
POST Args:
programkey: the key of the program to operate upon
"""
program = None
params = request.REQUEST
# Query the program entity
try:
program = program_logic.getFromKeyName(params["programkey"])
except KeyError:
logging.error("programkey not in params")
return responses.terminateTask()
if not program:
logging.error("no such program '%s'" % params["programkey"])
return responses.terminateTask()
if not program.slots_allocation:
logging.error("empty slots_allocation")
return responses.terminateTask()
# Enqueue a task to assign the slots
taskqueue.add(
url = "/gsoc/tasks/assignslots/assign",
params = {
'programkey': params["programkey"],
})
# Return successful
return responses.terminateTask()
def run(self, request, *args, **kwargs):
"""Processes all OrgAppSurveyRecords that are in the pre-accept or
pre-reject state for the given program.
Expects the following to be present in the POST dict:
program_key: Specifies the program key name for which to loop over all
the OrgAppSurveyRecords for.
Args:
request: Django Request object
"""
# set default batch size
batch_size = 10
# retrieve the program_key from POST data
post_dict = request.POST
program_key = post_dict.get('program_key')
if not program_key:
return error_handler.logErrorAndReturnOK(
'Not all required fields are present in POST dict %s' %
post_dict)
program_entity = self.program_logic.getFromKeyName(program_key)
if not program_entity:
return error_handler.logErrorAndReturnOK(
'No Program exists with keyname: %s' % program_key)
org_app = self.org_app_logic.getForProgram(program_entity)
record_logic = self.org_app_logic.getRecordLogic()
fields = {
'survey': org_app,
'status': ['pre-accepted', 'pre-rejected']
}
org_app_records = record_logic.getForFields(fields, limit=batch_size)
for org_app_record in org_app_records:
record_logic.processRecord(org_app_record)
if len(org_app_records) == batch_size:
# start a new task because we might not have exhausted all OrgAppRecords
context = post_dict.copy()
responses.startTask(self.path, context=context)
# return a 200 response that everything has been completed
return responses.terminateTask()
def wrapper(func):
def iterative_wrapped(request, *args, **kwargs):
"""Decorator wrapper method
Args:
request: Django HTTP Request object
request.POST usage:
fields: a JSON dict for the properties that the entities should have.
This updates values from the task_default entry.
start_key: the key of the next entity to fetch
Returns:
Standard HTTP Django response
"""
post_dict = request.POST
fields = task_default.get('fields', {})
if 'fields' in post_dict:
fields.update(pickle.loads(str(post_dict['fields'])))
start_key = task_default.get('start_key', None)
if 'start_key' in post_dict:
# get the key where to start this iteration
start_key = post_dict['start_key']
if start_key:
start_key = db.Key(start_key)
# get the entities for this iteration
entities, next_start_key = logic.getBatchOfData(filter=fields,
start_key=start_key)
# copy the post_dict so that the wrapped function can edit what it needs
context = post_dict.copy()
# when true, the iterative task will not be repeated when completed
do_not_repeat = False
try:
func(request, entities=entities, context=context, *args, **kwargs)
except task_responses.DoNotRepeatException, exception:
do_not_repeat = True
except task_responses.FatalTaskError, error:
logging.debug(post_dict)
logging.error(error)
return task_responses.terminateTask()
def run(self, request, *args, **kwargs):
"""Processes all OrgAppSurveyRecords that are in the pre-accept or
pre-reject state for the given program.
Expects the following to be present in the POST dict:
program_key: Specifies the program key name for which to loop over all
the OrgAppSurveyRecords for.
Args:
request: Django Request object
"""
# set default batch size
batch_size = 10
# retrieve the program_key from POST data
post_dict = request.POST
program_key = post_dict.get('program_key')
if not program_key:
return error_handler.logErrorAndReturnOK(
'Not all required fields are present in POST dict %s' % post_dict)
program_entity = self.program_logic.getFromKeyName(program_key)
if not program_entity:
return error_handler.logErrorAndReturnOK(
'No Program exists with keyname: %s' % program_key)
org_app = self.org_app_logic.getForProgram(program_entity)
record_logic = self.org_app_logic.getRecordLogic()
fields = {'survey': org_app,
'status': ['pre-accepted', 'pre-rejected']}
org_app_records = record_logic.getForFields(fields, limit=batch_size)
for org_app_record in org_app_records:
record_logic.processRecord(org_app_record)
if len(org_app_records) == batch_size:
# start a new task because we might not have exhausted all OrgAppRecords
context = post_dict.copy()
responses.startTask(self.path, context=context)
# return a 200 response that everything has been completed
return responses.terminateTask()
def clearGCIRanking(request, entities, context, *args, **kwargs):
"""Clears student ranking for a program with the specified key_name.
"""
program = gci_program_logic.getFromKeyName(kwargs['key_name'])
if not program:
return responses.terminateTask()
for entity in entities:
# check if the entity refers to the program in scope
if entity.scope.key() != program.key():
continue
entity.points = 0
entity.tasks = []
entity.put()
def clearGCIRanking(request, entities, context, *args, **kwargs):
"""Clears student ranking for a program with the specified key_name.
"""
program = gci_program_logic.getFromKeyName(kwargs['key_name'])
if not program:
return responses.terminateTask()
for entity in entities:
# check if the entity refers to the program in scope
if entity.scope.key() != program.key():
continue
entity.points = 0
entity.tasks = []
entity.put()
def sendMail(self, request):
"""Sends out an email that is stored in the datastore.
The POST request should contain the following entries:
mail_key: Datastore key for an Email entity.
"""
post_dict = request.POST
mail_key = post_dict.get('mail_key', None)
if not mail_key:
return error_handler.logErrorAndReturnOK('No email key specified')
mail_entity = db.get(mail_key)
if not mail_entity:
return error_handler.logErrorAndReturnOK(
'No email entity found for key %s' % mail_key)
# construct the EmailMessage from the given context
loaded_context = simplejson.loads(mail_entity.context)
context = {}
for key, value in loaded_context.iteritems():
# If we don't do this python will complain about kwargs not being
# strings.
context[str(key)] = value
logging.info('Sending %s' %context)
message = mail.EmailMessage(**context)
try:
message.check_initialized()
except Exception, e:
logging.exception(e)
context['body'] = context.get('body', '')[:10]
logging.error('This message was not properly initialized: "%s"' % context)
mail_entity.delete()
return responses.terminateTask()
def collectTask(self, request, access_type, page_name=None,
params=None, **kwargs):
"""Task for collecting statistics that will be executed by Task Queue API.
"""
logic = params['logic']
link_id = kwargs['link_id']
scope_path = kwargs['scope_path']
statistic = self._getStatisticEntity(link_id, scope_path, logic)
if statistic is None:
raise task_responses.FatalTaskError
statistic, completed = logic.collectDispatcher(statistic)
self._updateCache(statistic, link_id, scope_path, logic)
if completed:
self._updateCacheList(statistic, scope_path, logic)
statistic.put()
else:
task_responses.startTask(url=request.path, queue_name='statistic')
return task_responses.terminateTask()
def _continueShipmentSync(self, request, *args, **kwargs):
"""Continue syncing shipment data.
POST Args:
program_key: the key of the program which sync is being done for.
shipment_info_id: id of the shipment info object that task is running
for.
column_indexes: column indexes for specific columns in JSON format.
sheet_rows: spreadsheets CSV chunk data in JSON format.
"""
timekeeper = Timekeeper(20000)
params = dicts.merge(request.POST, request.GET)
redirect = RedirectHelper(None, None)
if 'program_key' not in params:
logging.error("missing program_key in params: '%s'" % params)
return responses.terminateTask()
if 'shipment_info_id' not in params:
logging.error("missing shipment_info_id in params: '%s'" % params)
return responses.terminateTask()
self.setProgram(params['program_key'])
self.setShipmentInfo(int(params['shipment_info_id']))
if 'sheet_rows' not in params:
logging.error("missing sheet_rows data in params: '%s'" % params)
return responses.terminateTask()
if 'column_indexes' not in params:
logging.error("missing column_indexes data in params: '%s'" % params)
return responses.terminateTask()
column_indexes = simplejson.loads(params['column_indexes'])
sheet_rows = simplejson.loads(params['sheet_rows'])
try:
for remain, row in timekeeper.iterate(sheet_rows):
if len(row) < len(column_indexes):
row.extend((len(column_indexes) - len(row)) * [''])
data = self.getRowData(row, column_indexes)
link_id = data['link_id']
q = GSoCProfile.all().filter('scope', self.__program)
q.filter('link_id', link_id)
profile = q.get()
if not profile:
logging.error("Profile link_id '%s' for program '%s' is not found" %
(link_id, self.__program.name))
continue #continue to next row
if not profile.is_student:
logging.error("Profile link_id '%s' is not a student" %
link_id)
continue
tracking = data['tracking']
date_shipped = data['date_shipped']
notes = data['notes']
full_address = " ".join([
data['address_1'], data['address_2'], data['city'],
data.get('state', ''), data.get('zip', ''),
data.get('zippostal_code', ''), data.get('country', '')
])
self.updateShipmentDataForStudent(
profile, tracking, date_shipped, notes, full_address)
except DeadlineExceededError:
if remain:
remaining_rows = sheet_rows[(-1 * remain):]
params = {
'program_key': params.get('program_key'),
'sheet_rows': simplejson.dumps(remaining_rows),
'column_indexes': params.get('column_indexes'),
'shipment_info_id': params.get('shipment_info_id'),
}
taskqueue.add(
url=redirect.urlOf('shipment_sync_task_continue'), params=params)
return responses.terminateTask()
self.finishSync()
return responses.terminateTask()
try:
func(request, entities=entities, context=context, *args, **kwargs)
except task_responses.DoNotRepeatException, exception:
do_not_repeat = True
except task_responses.FatalTaskError, error:
logging.debug(post_dict)
logging.error(error)
return task_responses.terminateTask()
except Exception, exception:
logging.debug(post_dict)
logging.error(exception)
return task_responses.repeatTask()
if next_start_key:
# set the key to use for the next iteration
context.update({'start_key': next_start_key})
task_responses.startTask(url=request.path, context=context)
elif not do_not_repeat and repeat_in is not None:
# the task will be executed again after repeat_in seconds
if 'start_key' in context:
del context['start_key']
task_responses.startTask(url=request.path, countdown=repeat_in,
context=context)
return task_responses.terminateTask()
return iterative_wrapped
return wrapper
except task_responses.DoNotRepeatException, exception:
do_not_repeat = True
except task_responses.FatalTaskError, error:
logging.debug(post_dict)
logging.error(error)
return task_responses.terminateTask()
except Exception, exception:
logging.debug(post_dict)
logging.error(exception)
return task_responses.repeatTask()
if next_start_key:
# set the key to use for the next iteration
context.update({'start_key': next_start_key})
task_responses.startTask(url=request.path, context=context)
elif not do_not_repeat and repeat_in is not None:
# the task will be executed again after repeat_in seconds
if 'start_key' in context:
del context['start_key']
task_responses.startTask(url=request.path,
countdown=repeat_in,
context=context)
return task_responses.terminateTask()
return iterative_wrapped
return wrapper
def convert_proposals(request, *args, **kwargs):
"""Convert proposals for all organizations.
POST Args:
programkey: the key of the program whose proposals should be converted
orgkey: the organization key to start at
"""
# Setup an artifical request deadline
timelimit = 20000
timekeeper = Timekeeper(timelimit)
# Copy for modification below
params = dicts.merge(request.POST, request.GET)
if "programkey" not in params:
logging.error("missing programkey in params: '%s'" % params)
return responses.terminateTask()
program = program_logic.getFromKeyName(params["programkey"])
if not program:
logging.error("invalid programkey in params: '%s'" % params)
return responses.terminateTask()
fields = {
"scope": program,
"status": "active",
}
# Continue from the next organization
if "orgkey" in params:
org = org_logic.getFromKeyName(params["orgkey"])
if not org:
logging.error("invalid orgkey in params: '%s'" % params)
return responses.terminateTask()
fields["__key__ >="] = org
# Add a task for each organization
org = None
try:
orgs = org_logic.getQueryForFields(filter=fields)
for remain, org in timekeeper.iterate(orgs):
logging.info("convert %s %s", remain, org.key())
# Compound accept/reject taskflow
taskqueue.add(url="/tasks/accept_proposals/accept",
params={
"orgkey": org.key().id_or_name(),
"timelimit": timelimit,
"nextpath": "/tasks/accept_proposals/reject"
})
# Requeue this task for continuation
except DeadlineExceededError:
if org:
params["orgkey"] = org.key().id_or_name()
taskqueue.add(url=request.path, params=params)
# Exit this task successfully
return responses.terminateTask()
context[str(key)] = value
logging.info('Sending %s' %context)
message = mail.EmailMessage(**context)
try:
message.check_initialized()
except:
logging.error('This message was not properly initialized')
mail_entity.delete()
return responses.terminateTask()
def txn():
"""Transaction that ensures the deletion of the Email entity only if
the mail has been successfully sent.
"""
mail_entity.delete()
message.send()
try:
db.RunInTransaction(txn)
except mail.Error, exception:
# shouldn't happen because validate has been called, keeping the Email
# entity for study purposes.
return error_handler.logErrorAndReturnOK(exception)
except OverQuotaError:
return responses.repeatTask()
# mail successfully sent
return responses.terminateTask()
def _continueShipmentSync(self, request, *args, **kwargs):
"""Continue syncing shipment data.
POST Args:
program_key: the key of the program which sync is being done for.
shipment_info_id: id of the shipment info object that task is running
for.
column_indexes: column indexes for specific columns in JSON format.
sheet_rows: spreadsheets CSV chunk data in JSON format.
"""
timekeeper = Timekeeper(20000)
params = dicts.merge(request.POST, request.GET)
if 'program_key' not in params:
logging.error("missing program_key in params: '%s'", params)
return responses.terminateTask()
if 'shipment_info_id' not in params:
logging.error("missing shipment_info_id in params: '%s'", params)
return responses.terminateTask()
self.setProgram(params['program_key'])
self.setShipmentInfo(int(params['shipment_info_id']))
if 'sheet_rows' not in params:
logging.error("missing sheet_rows data in params: '%s'", params)
return responses.terminateTask()
if 'column_indexes' not in params:
logging.error("missing column_indexes data in params: '%s'", params)
return responses.terminateTask()
column_indexes = json.loads(params['column_indexes'])
sheet_rows = json.loads(params['sheet_rows'])
try:
for remain, row in timekeeper.iterate(sheet_rows):
if len(row) < len(column_indexes):
row.extend((len(column_indexes) - len(row)) * [''])
data = self.getRowData(row, column_indexes)
username = data['username']
profile = profile_logic.getProfileForUsername(username, self.program_key)
if not profile:
logging.error("Profile with username '%s' for program '%s' is not found",
username, self.ndb_program_key.id())
continue #continue to next row
if not profile.is_student:
logging.error("Profile with username '%s' is not a student", username)
continue
tracking = data['tracking']
self.updateShipmentDataForStudent(profile, tracking)
except DeadlineExceededError:
if remain:
remaining_rows = sheet_rows[(-1 * remain):]
params = {
'program_key': params.get('program_key'),
'sheet_rows': json.dumps(remaining_rows),
'column_indexes': params.get('column_indexes'),
'shipment_info_id': params.get('shipment_info_id'),
}
task_continue_url = links.SOC_LINKER.site(url_names.GSOC_SHIPMENT_TASK_CONTINUE)
taskqueue.add(url=task_continue_url, params=params)
return responses.terminateTask()
self.finishSync()
return responses.terminateTask()
def convert_proposals(request, *args, **kwargs):
"""Convert proposals for all organizations.
POST Args:
programkey: the key of the program whose proposals should be converted
orgkey: the organization key to start at
"""
# Setup an artifical request deadline
timelimit = 20000
timekeeper = Timekeeper(timelimit)
# Copy for modification below
params = dicts.merge(request.POST, request.GET)
if "programkey" not in params:
logging.error("missing programkey in params: '%s'" % params)
return responses.terminateTask()
program = program_logic.getFromKeyName(params["programkey"])
if not program:
logging.error("invalid programkey in params: '%s'" % params)
return responses.terminateTask()
fields = {
"scope": program,
"status": "active",
}
# Continue from the next organization
if "orgkey" in params:
org = org_logic.getFromKeyName(params["orgkey"])
if not org:
logging.error("invalid orgkey in params: '%s'" % params)
return responses.terminateTask()
fields["__key__ >="] = org
# Add a task for each organization
org = None
try:
orgs = org_logic.getQueryForFields(filter=fields)
for remain, org in timekeeper.iterate(orgs):
logging.info("convert %s %s", remain, org.key())
# Compound accept/reject taskflow
taskqueue.add(
url = "/tasks/accept_proposals/accept",
params = {
"orgkey": org.key().id_or_name(),
"timelimit": timelimit,
"nextpath": "/tasks/accept_proposals/reject"
})
# Requeue this task for continuation
except DeadlineExceededError:
if org:
params["orgkey"] = org.key().id_or_name()
taskqueue.add(url=request.path, params=params)
# Exit this task successfully
return responses.terminateTask()
def iterative_task(logic, **task_default):
"""Iterative wrapper method
Args:
logic: the Logic instance to get entities for
task_default: keyword arguments which can contain the following options:
fields: dictionary to filter the entities on
start_key: the default key where to start this iterative task
"""
def wrapper(func):
def iterative_wrapped(request, *args, **kwargs):
"""Decorator wrapper method
Args:
request: Django HTTP Request object
request.POST usage:
fields: a JSON dict for the properties that the entities should have.
This updates values from the task_default entry.
start_key: the key of the next entity to fetch
Returns:
Standard HTTP Django response
"""
post_dict = request.POST
fields = task_default.get('fields', {})
if 'fields' in post_dict:
fields.update(pickle.loads(str(post_dict['fields'])))
start_key = task_default.get('start_key', None)
if 'start_key' in post_dict:
# get the key where to start this iteration
start_key = post_dict['start_key']
if start_key:
start_key = db.Key(start_key)
# get the entities for this iteration
entities, next_start_key = logic.getBatchOfData(
filter=fields, start_key=start_key)
# copy the post_dict so that the wrapped function can edit what it needs
context = post_dict.copy()
try:
func(request,
entities=entities,
context=context,
*args,
**kwargs)
except task_responses.FatalTaskError, error:
logging.debug(post_dict)
logging.error(error)
return task_responses.terminateTask()
except Exception, exception:
logging.debug(post_dict)
logging.error(exception)
return task_responses.repeatTask()
if next_start_key:
# set the key to use for the next iteration
context.update({'start_key': next_start_key})
task_responses.startTask(url=request.path, context=context)
return task_responses.terminateTask()
def sendMail(self, request):
"""Sends out an email that is stored in the datastore.
The POST request should contain the following entries:
mail_key: Datastore key for an Email entity.
"""
post_dict = request.POST
mail_key = post_dict.get('mail_key', None)
if not mail_key:
return error_handler.logErrorAndReturnOK('No email key specified')
# TODO(daniel): so ugly...
try:
mail_entity = db.get(mail_key)
except datastore_errors.BadKeyError:
mail_entity = ndb.Key(urlsafe=mail_key).get()
if not mail_entity:
return error_handler.logErrorAndReturnOK(
'No email entity found for key %s' % mail_key)
# construct the EmailMessage from the given context
loaded_context = json.loads(mail_entity.context)
context = {}
for key, value in loaded_context.iteritems():
# If we don't do this python will complain about kwargs not being
# strings.
context[str(key)] = value
logging.info('Sending %s', context)
message = mail.EmailMessage(**context)
try:
message.check_initialized()
except Exception as e:
logging.exception(e)
context['body'] = context.get('body', '')[:10]
logging.error('This message was not properly initialized: "%s"',
context)
mail_entity.delete()
return responses.terminateTask()
def txn():
"""Transaction that ensures the deletion of the Email entity only if
the mail has been successfully sent.
"""
mail_entity.delete()
message.send()
try:
db.RunInTransaction(txn)
except mail.Error as exception:
# shouldn't happen because validate has been called, keeping the Email
# entity for study purposes.
return error_handler.logErrorAndReturnOK(exception)
except (OverQuotaError, DeadlineExceededError) as e:
return responses.repeatTask()
# mail successfully sent
return responses.terminateTask()
