def updateConversations(self, request, *args, **kwargs): """Handler for task. The POST dict should have keys: user_key: Key string for User. program_key: Key string for GCIProgram. """ post_dict = request.POST user_key_str = post_dict.get('user_key') if not user_key_str: return error_handler.logErrorAndReturnOK( 'user_key missing from POST data.') program_key_str = post_dict.get('program_key') if not program_key_str: return error_handler.logErrorAndReturnOK( 'program_key missing from POST data.') user_key = ndb.Key(urlsafe=user_key_str) program_key = ndb.Key(urlsafe=program_key_str) gciconversation_logic.refreshConversationsForUserAndProgram( user_key, program_key) return http.HttpResponse('OK')
def updateGCITask(request, *args, **kwargs): """Method executed by Task Queue API to update a GCI Task to relevant state. Expects the gci_task_key entry to be present in the POST dict. Args: request: the standard Django HTTP request object """ post_dict = request.POST key_name = post_dict.get('gci_task_key') if not key_name: # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid updateGCITask data: %s' % post_dict) entity = gci_task_logic.logic.getFromKeyName(key_name) if not entity: # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid updateGCITask gci_task_key: %s' % post_dict) entity, comment_entity = gci_task_logic.logic.updateTaskStatus(entity) if entity: # TODO(madhusudan): does this really mean an unsuccessful update? # return OK return http.HttpResponse()
def update_ranker(request, *args, **kwargs): """Update the ranker for the specified proposal. POST Args: student_proposal_key: the key of the student proposal for which the ranker should be updated value: the value of the new score given to the proposal; type str(int) or '' """ # Copy for modification below params = request.POST if 'student_proposal_key' not in params: return error_handler.logErrorAndReturnOK( 'missing student_proposal_key in params: "%s"' % params) student_proposal = student_proposal_logic.getFromKeyName( params['student_proposal_key']) if not student_proposal: return error_handler.logErrorAndReturnOK( 'invalid student_proposal_key in params: "%s"' % params) value = params.get('value', '') value = [int(value)] if value else None # update the ranker ranker = student_proposal_logic.getRankerFor(student_proposal) ranker.SetScore(student_proposal.key().id_or_name(), value) # return OK return http.HttpResponse()
def updateRecordsForSurveyGroup(self, request, *args, **kwargs): """Updates or creates GradingRecords for the given GradingSurveyGroup. Expects the following to be present in the POST dict: group_key: Specifies the GradingSurveyGroup key name. cursor: optional to specify where the query should continue from. Args: request: Django Request object """ post_dict = request.POST group_key = post_dict.get('group_key') if not group_key: # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid updateRecordForSurveyGroup data: %s' % post_dict) # get the GradingSurveyGroup for the given key survey_group = GSoCGradingSurveyGroup.get_by_id(int(group_key)) if not survey_group: # invalid GradingSurveyGroup specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid GradingSurveyGroup specified: %s' % group_key) q = GSoCProject.all() q.filter('program', survey_group.program) q.filter('status', 'accepted') if 'cursor' in post_dict: q.with_cursor(post_dict['cursor']) # get the first batch_size number of StudentProjects projects = q.fetch(self.DEF_BATCH_SIZE) if not projects: # task completed, update timestamp for last update complete survey_group.last_update_complete = datetime.datetime.now() survey_group.put() return http.HttpResponse() # update/create and batch put the new GradingRecords grading_record.updateOrCreateRecordsFor(survey_group, projects) # pass along these params as POST to the new task task_params = {'group_key': group_key, 'cursor': q.cursor()} new_task = taskqueue.Task(params=task_params, url=request.path) new_task.add() # task completed, return OK return http.HttpResponse('OK')
def sendMail(request): """Sends out an email that is stored in the datastore. The POST request should contain the following entries: mail_key: Datastore key for an Email entity. """ post_dict = request.POST mail_key = post_dict.get('mail_key', None) if not mail_key: return error_handler.logErrorAndReturnOK('No email key specified') mail_entity = email.Email.get(mail_key) if not mail_entity: return error_handler.logErrorAndReturnOK( 'No email entity found for key %s' %mail_key) # construct the EmailMessage from the given context loaded_context = simplejson.loads(mail_entity.context) context = {} for key, value in loaded_context.iteritems(): # If we don't do this python will complain about kwargs not being # strings. context[str(key)] = value logging.info('Sending %s' %context) message = mail.EmailMessage(**context) try: message.check_initialized() except: logging.error('This message was not properly initialized') mail_entity.delete() return responses.terminateTask() def txn(): """Transaction that ensures the deletion of the Email entity only if the mail has been successfully sent. """ mail_entity.delete() message.send() try: db.RunInTransaction(txn) except mail.Error, exception: # shouldn't happen because validate has been called, keeping the Email # entity for study purposes. return error_handler.logErrorAndReturnOK(exception)
def run(self, request, *args, **kwargs): """Processes all OrgAppSurveyRecords that are in the pre-accept or pre-reject state for the given program. Expects the following to be present in the POST dict: program_key: Specifies the program key name for which to loop over all the OrgAppSurveyRecords for. Args: request: Django Request object """ # set default batch size batch_size = 10 # retrieve the program_key from POST data post_dict = request.POST program_key = post_dict.get('program_key') if not program_key: return error_handler.logErrorAndReturnOK( 'Not all required fields are present in POST dict %s' % post_dict) program_entity = self.program_logic.getFromKeyName(program_key) if not program_entity: return error_handler.logErrorAndReturnOK( 'No Program exists with keyname: %s' % program_key) org_app = self.org_app_logic.getForProgram(program_entity) record_logic = self.org_app_logic.getRecordLogic() fields = { 'survey': org_app, 'status': ['pre-accepted', 'pre-rejected'] } org_app_records = record_logic.getForFields(fields, limit=batch_size) for org_app_record in org_app_records: record_logic.processRecord(org_app_record) if len(org_app_records) == batch_size: # start a new task because we might not have exhausted all OrgAppRecords context = post_dict.copy() responses.startTask(self.path, context=context) # return a 200 response that everything has been completed return responses.terminateTask()
def sendMail(request, *args, **kwargs): """Sends out an email using context to supply the needed information. Args: memcache_key (via request.POST): memcache key used to fetch context for the email message Raises: Error that corresponds with the first problem it finds if the message is not properly initialized. List of all possible errors: http://code.google.com/appengine/docs/mail/exceptions.html """ memcache_key = request.POST.get('memcache_key') # fetch context from the memcache context = memcache.get(memcache_key, namespace='mail') if not context: # since key isn't available, do not retry the task return error_handler.logErrorAndReturnOK( 'unable to fetch mail params using memcache key %s' % memcache_key) # construct the EmailMessage from the given context message = mail.EmailMessage(**context) message.check_initialized() try: # send the message message.send() logging.info('sent mail with context %s' % context) except mail.Error, exception: logging.info(context) logging.exception(exception) # do not return HttpResponse - this will prompt a retry of the task return
def run(self, request, *args, **kwargs): """Processes all OrgAppSurveyRecords that are in the pre-accept or pre-reject state for the given program. Expects the following to be present in the POST dict: program_key: Specifies the program key name for which to loop over all the OrgAppSurveyRecords for. Args: request: Django Request object """ # set default batch size batch_size = 10 # retrieve the program_key from POST data post_dict = request.POST program_key = post_dict.get('program_key') if not program_key: return error_handler.logErrorAndReturnOK( 'Not all required fields are present in POST dict %s' % post_dict) program_entity = self.program_logic.getFromKeyName(program_key) if not program_entity: return error_handler.logErrorAndReturnOK( 'No Program exists with keyname: %s' % program_key) org_app = self.org_app_logic.getForProgram(program_entity) record_logic = self.org_app_logic.getRecordLogic() fields = {'survey': org_app, 'status': ['pre-accepted', 'pre-rejected']} org_app_records = record_logic.getForFields(fields, limit=batch_size) for org_app_record in org_app_records: record_logic.processRecord(org_app_record) if len(org_app_records) == batch_size: # start a new task because we might not have exhausted all OrgAppRecords context = post_dict.copy() responses.startTask(self.path, context=context) # return a 200 response that everything has been completed return responses.terminateTask()
def status(self, request, *args, **kwargs): """Update the status of proposals conversion. Expects the following to be present in the POST dict: program_key: Specifies the program key name for which to update the conversion status Args: request: Django Request object """ params = request.POST # retrieve the program_key from POST data program_key = params.get('program_key') if not program_key: # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( '"Missing program_key in params: "%s"' % params) # get the program for the given keyname program_entity = GSoCProgram.get_by_key_name(program_key) if not program_entity: # invalid program specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid program key specified: "%s"' % program_key) # obtain the accept proposals status aps_entity = conversion_logic.getOrCreateStatusForProgram(program_entity) # update the accept proposals status aps_entity.status = 'proceeded' # if the first proposal set the started_on converted_projects = aps_entity.nr_converted_projects + 1 if converted_projects == 1: aps_entity.started_on = datetime.datetime.now() # tracks the number of converted projects so far aps_entity.nr_converted_projects = converted_projects db.put(aps_entity) # return OK return http.HttpResponse()
def runSchoolTypeUpdate(request, *args, **kwargs): """Appengine Task that adds school_type as University for existing Student entities in batches. Addition of required school_type property to Student model requires addition of corresponding value to all the existing Student entities in the datastore. Since this property is introduced during GSoC 2009 all students should be University students. This task sets the school_type value to "University" to all the existing entities. Args: request: Django Request object """ from soc.logic.models.student import logic as student_logic fields = {} post_dict = request.POST start_key = post_dict.get('start_key') if start_key: # retrieve the last student entity that was converted start = student_logic.getFromKeyName(start_key) if not start: # invalid starting student key specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid Student Key specified: %s' %(start_key)) fields['__key__ >'] = start.key() # get the first batch_size number of StudentProjects entities = student_logic.getForFields(fields, limit=DEF_BATCH_SIZE) for entity in entities: entity.school_type = 'University' db.put(entities) if len(entities) == DEF_BATCH_SIZE: # spawn new task starting from the last new_start = entities[DEF_BATCH_SIZE-1].key().id_or_name() # pass along these params as POST to the new task task_params = {'start_key': new_start} new_task = taskqueue.Task(params=task_params, url=request.META['PATH_INFO']) new_task.add() # task completed, return OK return HttpResponse('OK')
def manageModelsStatus(request, *args, **kwargs): """Sets status of the roles queried by the fields given by POST. """ post_dict = request.POST new_status = post_dict.get('new_status') if not new_status: if not status_retriever or not callable(status_retriever): return error_handler.logErrorAndReturnOK( 'No valid status can be set by the manageModelStatus.') if not 'fields' in post_dict: error_handler.logErrorAndReturnOK( 'No fields to filter on found for manageModelStatus.') fields = pickle.loads(str(post_dict['fields'])) entities = entity_logic.getForFields(fields, limit=BATCH_SIZE) for entity in entities: if new_status: status = new_status else: status = status_retriever(entity) entity.status = status db.put(entities) if len(entities) == BATCH_SIZE: # we might not have exhausted all the roles that can be updated, # so start the same task again context = post_dict.copy() responses.startTask(request.path, context=context) return responses.terminateTask() # exhausted all the entities the task has been completed return responses.terminateTask()
def sendMail(self, request): """Sends out an email that is stored in the datastore. The POST request should contain the following entries: mail_key: Datastore key for an Email entity. """ post_dict = request.POST mail_key = post_dict.get('mail_key', None) if not mail_key: return error_handler.logErrorAndReturnOK('No email key specified') mail_entity = db.get(mail_key) if not mail_entity: return error_handler.logErrorAndReturnOK( 'No email entity found for key %s' % mail_key) # construct the EmailMessage from the given context loaded_context = simplejson.loads(mail_entity.context) context = {} for key, value in loaded_context.iteritems(): # If we don't do this python will complain about kwargs not being # strings. context[str(key)] = value logging.info('Sending %s' %context) message = mail.EmailMessage(**context) try: message.check_initialized() except Exception, e: logging.exception(e) context['body'] = context.get('body', '')[:10] logging.error('This message was not properly initialized: "%s"' % context) mail_entity.delete() return responses.terminateTask()
def create_review_for(request, *args, **kwargs): """Create a review for the specified proposal. POST Args: student_proposal_key: the key name of the student proposal for which the review should be created user_key: the key name of the user who is reviewing the proposal comment: the comment left by the reviewer; type str score: the score given by the reviewer; type str(int) is_public: is this review available to public; type str(bool) """ # Copy for modification below params = request.POST if 'student_proposal_key' not in params: return error_handler.logErrorAndReturnOK( 'missing student_proposal_key in params: "%s"' % params) student_proposal = student_proposal_logic.getFromKeyName( params['student_proposal_key']) if not student_proposal: return error_handler.logErrorAndReturnOK( 'invalid student_proposal_key in params: "%s"' % params) if 'user_key' not in params: return error_handler.logErrorAndReturnOK( 'missing user_key in params: "%s"' % params) user = user_logic.logic.getFromKeyName(params['user_key']) if not user: return error_handler.logErrorAndReturnOK( 'invalid user_key in params: "%s"' % params) comment = params.get('comment', '') score = int(params.get('score', 0)) is_public = True if params.get('is_public')=='True' else False student_proposal_logic.createReviewFor(student_proposal_view, student_proposal, user, comment, score, is_public) # return OK return http.HttpResponse()
def updateGCITask(self, request, id, *args, **kwargs): """Method executed by Task Queue API to update a GCI Task to relevant state. Args: request: the standard Django HTTP request object """ id = int(id) task = GCITask.get_by_id(id) if not task: # invalid task data, log and return OK return error_handler.logErrorAndReturnOK("No GCITask found for id: %s" % id) task_logic.updateTaskStatus(task) return http.HttpResponse()
def updateGCITask(self, request, id, *args, **kwargs): """Method executed by Task Queue API to update a GCI Task to relevant state. Args: request: the standard Django HTTP request object """ id = int(id) task = GCITask.get_by_id(id) if not task: # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'No GCITask found for id: %s' % id) task_logic.updateTaskStatus(task) return http.HttpResponse()
def spawnParentalFormMailTask(request, *args, **kwargs): """Spawns a task to send remainder email to existing students for sending parental consent forms. """ program = gci_program_logic.getFromKeyFields(kwargs) if not program: return error_handler.logErrorAndReturnOK( 'Invalid program key name: %(scope)s/%(link_id)s' % kwargs) task_params = { 'program_key': program.key().id_or_name() } new_task = taskqueue.Task(params=task_params, url=DEF_SPAWN_TASK_URL) new_task.add('mail') # just to make sure task was spawned return HttpResponse('OK')
def bulkCreateTasks(request, *args, **kwargs): """Task that creates GCI Tasks from bulk data specified in the POST dict. The POST dict should have the following information present: bulk_create_key: the key of the bulk_create entity """ import settings # keep track of our own timelimit (20 seconds) timelimit = 20000 timekeeper = Timekeeper(timelimit) post_dict = request.POST bulk_create_key = post_dict.get('bulk_create_key') if not bulk_create_key: return error_handler.logErrorAndReturnOK( 'Not all POST data specified in: %s' % post_dict) bulk_data = bulk_create_model.GCIBulkCreateData.get(bulk_create_key) if not bulk_data: return error_handler.logErrorAndReturnOK( 'No valid data found for key: %s' % bulk_create_key) # note that we only query for the quota once org_admin = bulk_data.created_by task_quota = org_logic.getRemainingTaskQuota(org_admin.scope) tasks = bulk_data.tasks while len(tasks) > 0: try: # check if we have time timekeeper.ping() if settings.GCI_TASK_QUOTA_LIMIT_ENABLED and task_quota <= 0: return error_handler.logErrorAndReturnOK( 'Task quota reached for %s' %(org_admin.scope.name)) # remove the first task task_as_string = tasks.pop(0) loaded_task = simplejson.loads(task_as_string) task = {} for key, value in loaded_task.iteritems(): # If we don't do this python will complain about kwargs not being # strings when we try to save the new task. task[key.encode('UTF-8')] = value logging.info('Uncleaned task: %s' %task) # clean the data errors = _cleanTask(task, org_admin) if errors: logging.warning( 'Invalid task data uploaded, the following errors occurred: %s' %errors) bulk_data.errors.append(db.Text( 'The task in row %i contains the following errors.\n %s' \ %(bulk_data.tasksRemoved(), '\n'.join(errors)))) # at-most-once semantics for creating tasks bulk_data.put() if errors: # do the next task continue # set other properties task['link_id'] = 't%i' % (int(time.time()*100)) task['scope'] = org_admin.scope task['scope_path'] = org_admin.scope_path task['program'] = org_admin.program task['status'] = 'Unpublished' task['created_by'] = org_admin task['modified_by'] = org_admin # create the new task logging.info('Creating new task with fields: %s' %task) task_logic.updateOrCreateFromFields(task) task_quota = task_quota - 1 except DeadlineExceededError: # time to bail out pass if len(tasks) == 0: # send out a message notifications.sendBulkCreationCompleted(bulk_data) bulk_data.delete() else: # there is still work to be done, do a non 500 response and requeue task_params = { 'bulk_create_key': bulk_data.key().id_or_name() } new_task = taskqueue.Task(params=task_params, url=BULK_CREATE_URL) # add to the gci queue new_task.add(queue_name='gci-update') # we're done here return http.HttpResponse('OK')
def spawnRemindersForProjectSurvey(request, *args, **kwargs): """Spawns tasks for each StudentProject in the given Program. Expects the following to be present in the POST dict: program_key: Specifies the program key name for which to loop over all the StudentProjects for survey_key: specifies the key name for the ProjectSurvey to send reminders for survey_type: either project or grading depending on the type of Survey project_key: optional to specify which project was the last for which a task was spawn Args: request: Django Request object """ from soc.modules.gsoc.logic.models.program import logic as program_logic from soc.modules.gsoc.logic.models.student_project import logic as \ student_project_logic # set default batch size batch_size = 10 post_dict = request.POST # retrieve the program_key and survey_key from POST data program_key = post_dict.get('program_key') survey_key = post_dict.get('survey_key') survey_type = post_dict.get('survey_type') if not (program_key and survey_key and survey_type): # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid sendRemindersForProjectSurvey data: %s' % post_dict) # get the program for the given keyname program_entity = program_logic.getFromKeyName(program_key) if not program_entity: # invalid program specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid program specified: %s' % program_key) # check and retrieve the project_key that has been done last if 'project_key' in post_dict: project_start_key = post_dict['project_key'] else: project_start_key = None # get all valid StudentProjects from starting key fields = {'program': program_entity, 'status': 'accepted'} if project_start_key: # retrieve the last project that was done project_start = student_project_logic.getFromKeyName(project_start_key) if not project_start: # invalid starting project key specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid Student Project Key specified: %s' % (project_start_key)) fields['__key__ >'] = project_start.key() project_entities = student_project_logic.getForFields(fields, limit=batch_size) for project_entity in project_entities: # pass along these params as POST to the new task task_params = { 'survey_key': survey_key, 'survey_type': survey_type, 'project_key': project_entity.key().id_or_name() } task_url = '/tasks/surveys/projects/send_reminder/send' new_task = taskqueue.Task(params=task_params, url=task_url) new_task.add('mail') if len(project_entities) == batch_size: # spawn new task starting from the last new_project_start = project_entities[batch_size - 1].key().id_or_name() # pass along these params as POST to the new task task_params = { 'program_key': program_key, 'survey_key': survey_key, 'survey_type': survey_type, 'project_key': new_project_start } task_url = '/tasks/surveys/projects/send_reminder/spawn' new_task = taskqueue.Task(params=task_params, url=task_url) new_task.add() # return OK return http.HttpResponse()
def updateTasksPostStudentSignUp(request, *args, **kwargs): """Appengine task that updates the GCI Tasks after the student signs up. Expects the following to be present in the POST dict: student_key: Specifies the student key name who registered Args: request: Django Request object """ from soc.modules.gci.logic.models import student as gci_student_logic post_dict = request.POST student_key = post_dict.get('student_key') if not student_key: # invalid student data, log and return OK return error_handler.logErrorAndReturnOK('Invalid Student data: %s' % post_dict) student_entity = gci_student_logic.logic.getFromKeyNameOr404(student_key) # retrieve all tasks currently assigned to the user task_fields = { 'user': student_entity.user, } task_entities = gci_task_logic.logic.getForFields(task_fields) # TODO(madhusudan) move this to the Task Logic # Make sure the tasks store references to the student as well as # closing all tasks that are AwaitingRegistration. for task_entity in task_entities: task_entity.student = student_entity if task_entity.status == 'AwaitingRegistration': task_entities.remove(task_entity) properties = { 'status': 'Closed', 'closed_on': datetime.datetime.utcnow() } changes = [ ugettext('User-MelangeAutomatic'), ugettext('Action-Student registered'), ugettext('Status-%s' % (properties['status'])) ] comment_properties = { 'parent': task_entity, 'scope_path': task_entity.key().name(), 'created_by': None, 'changes': changes, 'content': ugettext( '(The Melange Automated System has detected that the student ' 'has signed up for the program and hence has closed this task.' ), } gci_task_logic.logic.updateEntityPropertiesWithCWS( task_entity, properties, comment_properties) ranking_update.startUpdatingTask(task_entity) db.put(task_entities) # return OK return http.HttpResponse()
def spawnRemindersForProjectSurvey(self, request, *args, **kwargs): """Spawns tasks for each GSoCProject in the given Program. Expects the following to be present in the POST dict: program_key: Specifies the program key name for which to loop over all the GSoCProject entities survey_key: specifies the key name for the ProjectSurvey to send reminders for survey_type: a string which is project or grading depending on the type of Survey. cursor: optional query cursor to indicate how far along we are. Args: request: Django Request object """ post_dict = request.POST # retrieve the program_key and survey_key from POST data program_key = post_dict.get('program_key') survey_key = post_dict.get('survey_key') survey_type = post_dict.get('survey_type') if not (program_key and survey_key and survey_type): # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid spawnRemindersForProjectSurvey data: %s' % post_dict) program_entity = GSoCProgram.get_by_key_name(program_key) if not program_entity: # invalid program specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid program specified: %s' % program_key) q = GSoCProject.all() q.filter('status', 'accepted') q.filter('program', program_entity) if 'cursor' in post_dict: q.with_cursor(post_dict['cursor']) projects = q.fetch(self.BATCH_SIZE) if not projects: # we are done, return OK return http.HttpResponse() for project in projects: task_params = { 'survey_key': survey_key, 'survey_type': survey_type, 'project_key': str(project.key()) } task_url = '/tasks/gsoc/surveys/send_reminder/send' new_task = taskqueue.Task(params=task_params, url=task_url) new_task.add('mail') # pass along these params as POST to the new task task_params = { 'program_key': program_key, 'survey_key': survey_key, 'survey_type': survey_type, 'cursor': q.cursor() } task_url = request.path new_task = taskqueue.Task(params=task_params, url=task_url) new_task.add() # return OK return http.HttpResponse()
def calculate(self, request, *args, **kwargs): """Calculates the duplicate proposals in a given program for a student on a per Organization basis. Expects the following to be present in the POST dict: program_key: Specifies the program key name for which to find the duplicate proposals org_cursor: Specifies the organization datastore cursor from which to start the processing of finding the duplicate proposals Args: request: Django Request object """ post_dict = request.POST program_key = post_dict.get('program_key') if not program_key: # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid program key: %s' % post_dict) program_entity = GSoCProgram.get_by_key_name(program_key) if not program_entity: # invalid program specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid program specified: %s' % program_key) # get the organization and update the cursor if possible query = soc_org_model.SOCOrganization.query( soc_org_model.SOCOrganization.status == org_model.Status.ACCEPTED, soc_org_model.SOCOrganization.program == ndb.Key.from_old_key(program_entity.key()), soc_org_model.SOCOrganization.slot_allocation > 0) # retrieve the org_cursor from POST data org_cursor = post_dict.get('org_cursor') start_cursor = ( datastore_query.Cursor(urlsafe=org_cursor) if org_cursor else None) organizations, next_cursor, _ = query.fetch_page( 1, start_cursor=start_cursor) if organizations: organization = organizations[0] # get all the proposals likely to be accepted in the program accepted_proposals = ( proposal_logic.getProposalsToBeAcceptedForOrg(organization)) for accepted_proposal in accepted_proposals: q = GSoCProposalDuplicate.all() q.filter('student', accepted_proposal.parent_key()) proposal_duplicate = q.get() if (proposal_duplicate and accepted_proposal.key() not in proposal_duplicate.duplicates): # non-counted (to-be) accepted proposal found proposal_duplicate.duplicates = proposal_duplicate.duplicates + \ [accepted_proposal.key()] proposal_duplicate.is_duplicate = \ len(proposal_duplicate.duplicates) >= 2 if organization.key.to_old_key() not in proposal_duplicate.orgs: proposal_duplicate.orgs = ( proposal_duplicate.orgs + [organization.key.to_old_key()]) else: pd_fields = { 'program': program_entity, 'student': accepted_proposal.parent_key(), 'orgs':[organization.key.to_old_key()], 'duplicates': [accepted_proposal.key()], 'is_duplicate': False } proposal_duplicate = GSoCProposalDuplicate(**pd_fields) proposal_duplicate.put() # Adds a new task that performs duplicate calculation for # the next organization. task_params = { 'program_key': program_key, 'org_cursor': next_cursor.urlsafe() } task_url = '/tasks/gsoc/proposal_duplicates/calculate' new_task = taskqueue.Task(params=task_params, url=task_url) new_task.add() else: # There aren't any more organizations to process. So delete # all the proposals for which there are not more than one # proposal for duplicates property. duplicates_logic.deleteAllForProgram(program_entity, non_dupes_only=True) # update the proposal duplicate status and its timestamp pds_entity = duplicates_logic.getOrCreateStatusForProgram(program_entity) pds_entity.status = 'idle' pds_entity.calculated_on = datetime.datetime.now() pds_entity.put() # return OK return http.HttpResponse()
def AddToFeedTask(request, *args, **kwargs): """ Creates a FeedItem entity pairing an event for a sender entity and one receiver entity Params: feed_item_key - key_name used for feed item sender_key - key for the entity sending the event receivers - entities assigned to get item in their feed update_type - create, update, delete, etc. payload - optional payload message for this feed item user_key - key_name of user who made the update """ params = request.POST """ if not params.get('user_key'): return error_handler.logErrorAndReturnOK( 'no user specified for AddToFeedTask') """ feed_item_key = params.get('feed_item_key') user_key = params.get('user_key', None) if user_key: acting_user = user_logic.getFromKeyName(user_key) else: acting_user = None sender_key = params.get('sender_key') if not sender_key: return error_handler.logErrorAndReturnOK( 'no sender_key specified for AddToFeedTask') receivers = [db.Key(key) for key in params.get( 'receivers', '').split(',')] if not receivers: return error_handler.logErrorAndReturnOK( 'no receivers specified for AddToFeedTask') update_type = params.get('update_type') if not update_type: return error_handler.logErrorAndReturnOK( 'no update_type specified for AddToFeedTask') # optional params payload = params.get('payload') # save item to datastore feed_item_properties = dict( sender= db.Key(sender_key), receivers = receivers, update_type = update_type ) if payload: feed_item_properties['payload'] = payload if acting_user: feed_item_properties['user'] = acting_user new_feed_item = news_feed_logic.updateOrCreateFromKeyName( feed_item_properties, feed_item_key) sender = db.get(sender_key) sendFeedItemEmailNotifications(sender, acting_user, update_type, payload, **kwargs) # send update ping for each receiver's feed receiver_entities = db.get(receivers) for receiver in receiver_entities: sendHubNotification(receiver) # task completed, return OK return http.HttpResponse('OK')
def sendSurveyReminderForProject(request, *args, **kwargs): """Sends a reminder mail for a given StudentProject and Survey. A reminder is only send if no record is on file for the given Survey and StudentProject. Expects the following to be present in the POST dict: survey_key: specifies the key name for the ProjectSurvey to send reminders for survey_type: either project or grading depending on the type of Survey project_key: key which specifies the project to send a reminder for Args: request: Django Request object """ from soc.logic import mail_dispatcher from soc.logic.models.org_admin import logic as org_admin_logic from soc.logic.models.site import logic as site_logic from soc.logic.models.student_project import logic as student_project_logic from soc.logic.models.survey import grading_logic from soc.logic.models.survey import project_logic from soc.views.helper import redirects post_dict = request.POST project_key = post_dict.get('project_key') survey_key = post_dict.get('survey_key') survey_type = post_dict.get('survey_type') if not (project_key and survey_key and survey_type): # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid sendSurveyReminderForProject data: %s' % post_dict) # set logic depending on survey type specified in POST if survey_type == 'project': survey_logic = project_logic elif survey_type == 'grading': survey_logic = grading_logic # retrieve the project and survey student_project = student_project_logic.getFromKeyName(project_key) if not student_project: # no existing project found, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid project specified %s:' % project_key) survey = survey_logic.getFromKeyName(survey_key) if not survey: # no existing survey found, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid survey specified %s:' % survey_key) # try to retrieve an existing record record_logic = survey_logic.getRecordLogic() fields = {'project': student_project, 'survey': survey} record_entity = record_logic.getForFields(fields, unique=True) if not record_entity: # send reminder email because we found no record student_entity = student_project.student site_entity = site_logic.getSingleton() if survey_type == 'project': survey_redirect = redirects.getTakeSurveyRedirect( survey,{'url_name': 'project_survey'}) to_role = student_entity mail_template = 'soc/project_survey/mail/reminder_gsoc.html' elif survey_type == 'grading': survey_redirect = redirects.getTakeSurveyRedirect( survey,{'url_name': 'grading_project_survey'}) to_role = student_project.mentor mail_template = 'soc/grading_project_survey/mail/reminder_gsoc.html' survey_url = "http://%(host)s%(redirect)s" % { 'redirect': survey_redirect, 'host': os.environ['HTTP_HOST'], } # set the context for the mail template mail_context = { 'student_name': student_entity.name(), 'project_title': student_project.title, 'survey_url': survey_url, 'survey_end': survey.survey_end, 'to_name': to_role.name(), 'site_name': site_entity.site_name, } # set the sender (sender, sender_address) = mail_dispatcher.getDefaultMailSender() mail_context['sender'] = sender_address # set the receiver and subject mail_context['to'] = to_role.email mail_context['subject'] = 'Evaluation Survey "%s" Reminder' %(survey.title) # find all org admins for the project's organization org_entity = student_project.scope fields = {'scope': org_entity, 'status': 'active'} org_admin_entities = org_admin_logic.getForFields(fields) # collect email addresses for all found org admins org_admin_addresses = [] for org_admin_entity in org_admin_entities: org_admin_addresses.append(org_admin_entity.email) if org_admin_addresses: mail_context['cc'] = org_admin_addresses # send out the email mail_dispatcher.sendMailFromTemplate(mail_template, mail_context) # return OK return http.HttpResponse()
def sendMailAboutGradingRecordResult(request, *args, **kwargs): """Sends out a mail about the result of one GradingRecord. Expects the following to be present in the POST dict: record_key: Specifies the key for the record to process. Args: request: Django Request object """ from soc.logic import mail_dispatcher from soc.logic.models.grading_record import logic as grading_record_logic from soc.logic.models.org_admin import logic as org_admin_logic from soc.logic.models.site import logic as site_logic post_dict = request.POST # check and retrieve the record_key that has been done last if 'record_key' in post_dict and post_dict['record_key'].isdigit(): record_key = int(post_dict['record_key']) else: record_key = None if not record_key: # no GradingRecord key specified, log and return OK error_handler.logErrorAndReturnOK( 'No valid record_key specified in POST data: %s' % request.POST) record_entity = grading_record_logic.getFromID(record_key) if not record_entity: # no valid GradingRecord key specified, log and return OK error_handler.logErrorAndReturnOK( 'No valid GradingRecord key specified: %s' % record_key) survey_group_entity = record_entity.grading_survey_group project_entity = record_entity.project student_entity = project_entity.student mentor_entity = project_entity.mentor org_entity = project_entity.scope site_entity = site_logic.getSingleton() mail_context = { 'survey_group': survey_group_entity, 'grading_record': record_entity, 'project': project_entity, 'organization': org_entity, 'site_name': site_entity.site_name, 'to_name': student_entity.name() } # set the sender (sender, sender_address) = mail_dispatcher.getDefaultMailSender() mail_context['sender'] = sender_address # set the receiver and subject mail_context['to'] = student_entity.email mail_context['cc'] = [mentor_entity.email] mail_context['subject'] = '%s results processed for %s' % ( survey_group_entity.name, project_entity.title) # find all org admins for the project's organization fields = {'scope': org_entity, 'status': 'active'} org_admin_entities = org_admin_logic.getForFields(fields) # collect email addresses for all found org admins org_admin_addresses = [] for org_admin_entity in org_admin_entities: org_admin_addresses.append(org_admin_entity.email) if org_admin_addresses: mail_context['cc'].extend(org_admin_addresses) # send out the email using a template mail_template = 'soc/grading_record/mail/result.html' mail_dispatcher.sendMailFromTemplate(mail_template, mail_context) # return OK return http.HttpResponse()
def calculate(self, request, *args, **kwargs): """Calculates the duplicate proposals in a given program for a student on a per Organization basis. Expects the following to be present in the POST dict: program_key: Specifies the program key name for which to find the duplicate proposals org_cursor: Specifies the organization datastore cursor from which to start the processing of finding the duplicate proposals Args: request: Django Request object """ post_dict = request.POST program_key = post_dict.get('program_key') if not program_key: # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid program key: %s' % post_dict) program_entity = GSoCProgram.get_by_key_name(program_key) if not program_entity: # invalid program specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid program specified: %s' % program_key) # get the organization and update the cursor if possible q = GSoCOrganization.all() q.filter('status', 'active') q.filter('scope', program_entity) q.filter('slots >', 0) # retrieve the org_cursor from POST data org_cursor = post_dict.get('org_cursor') if org_cursor: org_cursor = str(org_cursor) q.with_cursor(org_cursor) org_entity = q.get() # update the cursor org_cursor = q.cursor() if org_entity: # get all the proposals likely to be accepted in the program accepted_proposals = proposal_logic.getProposalsToBeAcceptedForOrg(org_entity) for ap in accepted_proposals: student_entity = ap.parent() q = GSoCProposalDuplicate.all() q.filter('student', student_entity) proposal_duplicate = q.get() if proposal_duplicate and ap.key() not in proposal_duplicate.duplicates: # non-counted (to-be) accepted proposal found proposal_duplicate.duplicates = proposal_duplicate.duplicates + \ [ap.key()] proposal_duplicate.is_duplicate = \ len(proposal_duplicate.duplicates) >= 2 if org_entity.key() not in proposal_duplicate.orgs: proposal_duplicate.orgs = proposal_duplicate.orgs + [org_entity.key()] else: pd_fields = { 'program': program_entity, 'student': student_entity, 'orgs':[org_entity.key()], 'duplicates': [ap.key()], 'is_duplicate': False } proposal_duplicate = GSoCProposalDuplicate(**pd_fields) proposal_duplicate.put() # Adds a new task that performs duplicate calculation for # the next organization. task_params = {'program_key': program_key, 'org_cursor': unicode(org_cursor)} task_url = '/tasks/gsoc/proposal_duplicates/calculate' new_task = taskqueue.Task(params=task_params, url=task_url) new_task.add() else: # There aren't any more organizations to process. So delete # all the proposals for which there are not more than one # proposal for duplicates property. duplicates_logic.deleteAllForProgram(program_entity, non_dupes_only=True) # update the proposal duplicate status and its timestamp pds_entity = duplicates_logic.getOrCreateStatusForProgram(program_entity) pds_entity.status = 'idle' pds_entity.calculated_on = datetime.datetime.now() pds_entity.put() # return OK return http.HttpResponse()
def sendSurveyReminderForProject(request, *args, **kwargs): """Sends a reminder mail for a given StudentProject and Survey. A reminder is only send if no record is on file for the given Survey and StudentProject. Expects the following to be present in the POST dict: survey_key: specifies the key name for the ProjectSurvey to send reminders for survey_type: either project or grading depending on the type of Survey project_key: key which specifies the project to send a reminder for Args: request: Django Request object """ from soc.logic import mail_dispatcher from soc.logic.models.site import logic as site_logic from soc.views.helper import redirects from soc.modules.gsoc.logic.models.org_admin import logic as org_admin_logic from soc.modules.gsoc.logic.models.student_project import logic as \ student_project_logic from soc.modules.gsoc.logic.models.survey import grading_logic from soc.modules.gsoc.logic.models.survey import project_logic post_dict = request.POST project_key = post_dict.get('project_key') survey_key = post_dict.get('survey_key') survey_type = post_dict.get('survey_type') if not (project_key and survey_key and survey_type): # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid sendSurveyReminderForProject data: %s' % post_dict) # set logic depending on survey type specified in POST if survey_type == 'project': survey_logic = project_logic elif survey_type == 'grading': survey_logic = grading_logic # retrieve the project and survey student_project = student_project_logic.getFromKeyName(project_key) if not student_project: # no existing project found, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid project specified %s:' % project_key) survey = survey_logic.getFromKeyName(survey_key) if not survey: # no existing survey found, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid survey specified %s:' % survey_key) # try to retrieve an existing record record_logic = survey_logic.getRecordLogic() fields = {'project': student_project, 'survey': survey} record_entity = record_logic.getForFields(fields, unique=True) if not record_entity: # send reminder email because we found no record student_entity = student_project.student site_entity = site_logic.getSingleton() if survey_type == 'project': survey_redirect = redirects.getTakeSurveyRedirect( survey, {'url_name': 'gsoc/project_survey'}) to_role = student_entity mail_template = 'soc/project_survey/mail/reminder_gsoc.html' elif survey_type == 'grading': survey_redirect = redirects.getTakeSurveyRedirect( survey, {'url_name': 'gsoc/grading_project_survey'}) to_role = student_project.mentor mail_template = 'soc/grading_project_survey/mail/reminder_gsoc.html' survey_url = "http://%(host)s%(redirect)s" % { 'redirect': survey_redirect, 'host': system.getHostname(), } # set the context for the mail template mail_context = { 'student_name': student_entity.name(), 'project_title': student_project.title, 'survey_url': survey_url, 'survey_end': survey.survey_end, 'to_name': to_role.name(), 'site_name': site_entity.site_name, } # set the sender (_, sender_address) = mail_dispatcher.getDefaultMailSender() mail_context['sender'] = sender_address # set the receiver and subject mail_context['to'] = to_role.email mail_context['subject'] = 'Evaluation Survey "%s" Reminder' % ( survey.title) # find all org admins for the project's organization org_entity = student_project.scope fields = {'scope': org_entity, 'status': 'active'} org_admin_entities = org_admin_logic.getForFields(fields) # collect email addresses for all found org admins org_admin_addresses = [] for org_admin_entity in org_admin_entities: org_admin_addresses.append(org_admin_entity.email) if org_admin_addresses: mail_context['cc'] = org_admin_addresses # send out the email mail_dispatcher.sendMailFromTemplate(mail_template, mail_context) # return OK return http.HttpResponse()
def updateOrCreateRecordsForSurveyGroup(request, *args, **kwargs): """Updates or creates GradingRecords for the given GradingSurveyGroup. Expects the following to be present in the POST dict: group_key: Specifies the GradingSurveyGroup key name. project_key: optional to specify which project was the last for which this task was run Args: request: Django Request object """ from soc.modules.gsoc.logic.models.grading_record import logic as grading_record_logic from soc.modules.gsoc.logic.models.grading_survey_group import logic as survey_group_logic from soc.modules.gsoc.logic.models.student_project import logic as student_project_logic post_dict = request.POST group_key = post_dict.get('group_key') if not group_key: # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid updateRecordForSurveyGroup data: %s' % post_dict) # get the GradingSurveyGroup for the given keyname survey_group_entity = survey_group_logic.getFromKeyName(group_key) if not survey_group_entity: # invalid GradingSurveyGroup specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid GradingSurveyGroup specified: %s' % group_key) # check and retrieve the project_key that has been done last if 'project_key' in post_dict: project_start_key = post_dict['project_key'] else: project_start_key = None # get all valid StudentProjects from starting key fields = { 'program': survey_group_entity.scope, 'status': ['accepted', 'failed', 'completed'] } if project_start_key: # retrieve the last project that was done project_start = student_project_logic.getFromKeyName(project_start_key) if not project_start: # invalid starting project key specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid Student Project Key specified: %s' % (project_start_key)) fields['__key__ >'] = project_start.key() # get the first batch_size number of StudentProjects project_entities = student_project_logic.getForFields(fields, limit=DEF_BATCH_SIZE) # update/create and batch put the new GradingRecords grading_record_logic.updateOrCreateRecordsFor(survey_group_entity, project_entities) if len(project_entities) == DEF_BATCH_SIZE: # spawn new task starting from the last new_project_start = project_entities[DEF_BATCH_SIZE - 1].key().id_or_name() # pass along these params as POST to the new task task_params = { 'group_key': group_key, 'project_key': new_project_start } task_url = '/tasks/grading_survey_group/update_records' new_task = taskqueue.Task(params=task_params, url=task_url) new_task.add() else: # task completed, update timestamp for last update complete fields = {'last_update_complete': datetime.datetime.now()} survey_group_logic.updateEntityProperties(survey_group_entity, fields) # task completed, return OK return http.HttpResponse('OK')
def updateTasksPostStudentSignUp(request, *args, **kwargs): """Appengine task that updates the GCI Tasks after the student signs up. Expects the following to be present in the POST dict: student_key: Specifies the student key name who registered Args: request: Django Request object """ from soc.modules.gci.logic.models import student as gci_student_logic post_dict = request.POST student_key = post_dict.get('student_key') if not student_key: # invalid student data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid Student data: %s' % post_dict) student_entity = gci_student_logic.logic.getFromKeyNameOr404(student_key) # retrieve all tasks currently assigned to the user task_fields = { 'user': student_entity.user, } task_entities = gci_task_logic.logic.getForFields(task_fields) # TODO(madhusudan) move this to the Task Logic # Make sure the tasks store references to the student as well as # closing all tasks that are AwaitingRegistration. for task_entity in task_entities: task_entity.student = student_entity if task_entity.status == 'AwaitingRegistration': task_entities.remove(task_entity) properties = { 'status': 'Closed', 'closed_on': datetime.datetime.utcnow() } changes = [ugettext('User-MelangeAutomatic'), ugettext('Action-Student registered'), ugettext('Status-%s' % (properties['status']))] comment_properties = { 'parent': task_entity, 'scope_path': task_entity.key().name(), 'created_by': None, 'changes': changes, 'content': ugettext( '(The Melange Automated System has detected that the student ' 'has signed up for the program and hence has closed this task.'), } gci_task_logic.logic.updateEntityPropertiesWithCWS( task_entity, properties, comment_properties) ranking_update.startUpdatingTask(task_entity) db.put(task_entities) # return OK return http.HttpResponse()
def createNotificationMail(request, *args, **kwargs): """Appengine task that sends mail to the subscribed users. Expects the following to be present in the POST dict: comment_key: Specifies the comment id for which to send the notifications task_key: Specifies the task key name for which the comment belongs to Args: request: Django Request object """ from soc.modules.gci.logic.helper import notifications as gci_notifications from soc.modules.gci.logic.models import comment as gci_comment_logic from soc.modules.gci.logic.models import task_subscription as \ gci_task_subscription_logic # set default batch size batch_size = 10 post_dict = request.POST comment_key = post_dict.get('comment_key') task_key = post_dict.get('task_key') if not (comment_key and task_key): # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid createNotificationMail data: %s' % post_dict) comment_key = long(comment_key) # get the task entity under which the specified comment was made task_entity = gci_task_logic.logic.getFromKeyName(task_key) # get the comment for the given id comment_entity = gci_comment_logic.logic.getFromID( comment_key, task_entity) if not comment_entity: # invalid comment specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid comment specified: %s/%s' % (comment_key, task_key)) # check and retrieve the subscriber_start_key that has been done last idx = post_dict.get('subscriber_start_index', '') subscriber_start_index = int(idx) if idx.isdigit() else 0 # get all subscribers to GCI task fields = { 'task': task_entity, } ts_entity = gci_task_subscription_logic.logic.getForFields( fields, unique=True) subscribers = db.get(ts_entity.subscribers[ subscriber_start_index:subscriber_start_index+batch_size]) task_url = "http://%(host)s%(task)s" % { 'host': system.getHostname(), 'task': redirects.getPublicRedirect( task_entity, {'url_name': 'gci/task'}), } # create the data for the mail to be sent message_properties = { 'task_url': task_url, 'redirect_url': "%(task_url)s#c%(cid)d" % { 'task_url': task_url, 'cid': comment_entity.key().id_or_name() }, 'comment_entity': comment_entity, 'task_entity': task_entity, } subject = DEF_TASK_UPDATE_SUBJECT_FMT % { 'program_name': task_entity.program.short_name, 'title': task_entity.title, } for subscriber in subscribers: gci_notifications.sendTaskUpdateMail(subscriber, subject, message_properties) if len(subscribers) == batch_size: # spawn task for sending out notifications to next set of subscribers next_start = subscriber_start_index + batch_size task_params = { 'comment_key': comment_key, 'task_key': task_key, 'subscriber_start_index': next_start } task_url = '/tasks/gci/task/mail/create' new_task = taskqueue.Task(params=task_params, url=task_url) new_task.add('mail') # return OK return http.HttpResponse()
def calculate(self, request, *args, **kwargs): """Calculates the duplicate proposals in a given program for a student on a per Organization basis. Expects the following to be present in the POST dict: program_key: Specifies the program key name for which to find the duplicate proposals org_cursor: Specifies the organization datastore cursor from which to start the processing of finding the duplicate proposals Args: request: Django Request object """ post_dict = request.POST program_key = post_dict.get('program_key') if not program_key: # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid program key: %s' % post_dict) program_entity = GSoCProgram.get_by_key_name(program_key) if not program_entity: # invalid program specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid program specified: %s' % program_key) # get the organization and update the cursor if possible query = soc_org_model.SOCOrganization.query( soc_org_model.SOCOrganization.status == org_model.Status.ACCEPTED, soc_org_model.SOCOrganization.program == ndb.Key.from_old_key( program_entity.key()), soc_org_model.SOCOrganization.slot_allocation > 0) # retrieve the org_cursor from POST data org_cursor = post_dict.get('org_cursor') start_cursor = (datastore_query.Cursor( urlsafe=org_cursor) if org_cursor else None) organizations, next_cursor, _ = query.fetch_page( 1, start_cursor=start_cursor) if organizations: organization = organizations[0] # get all the proposals likely to be accepted in the program accepted_proposals = ( proposal_logic.getProposalsToBeAcceptedForOrg(organization)) for accepted_proposal in accepted_proposals: q = GSoCProposalDuplicate.all() q.filter('student', accepted_proposal.parent_key()) proposal_duplicate = q.get() if (proposal_duplicate and accepted_proposal.key() not in proposal_duplicate.duplicates): # non-counted (to-be) accepted proposal found proposal_duplicate.duplicates = proposal_duplicate.duplicates + \ [accepted_proposal.key()] proposal_duplicate.is_duplicate = \ len(proposal_duplicate.duplicates) >= 2 if organization.key.to_old_key( ) not in proposal_duplicate.orgs: proposal_duplicate.orgs = ( proposal_duplicate.orgs + [organization.key.to_old_key()]) else: pd_fields = { 'program': program_entity, 'student': accepted_proposal.parent_key(), 'orgs': [organization.key.to_old_key()], 'duplicates': [accepted_proposal.key()], 'is_duplicate': False } proposal_duplicate = GSoCProposalDuplicate(**pd_fields) proposal_duplicate.put() # Adds a new task that performs duplicate calculation for # the next organization. task_params = { 'program_key': program_key, 'org_cursor': next_cursor.urlsafe() } task_url = '/tasks/gsoc/proposal_duplicates/calculate' new_task = taskqueue.Task(params=task_params, url=task_url) new_task.add() else: # There aren't any more organizations to process. So delete # all the proposals for which there are not more than one # proposal for duplicates property. duplicates_logic.deleteAllForProgram(program_entity, non_dupes_only=True) # update the proposal duplicate status and its timestamp pds_entity = duplicates_logic.getOrCreateStatusForProgram( program_entity) pds_entity.status = 'idle' pds_entity.calculated_on = datetime.datetime.now() pds_entity.put() # return OK return http.HttpResponse()
def sendMail(self, request): """Sends out an email that is stored in the datastore. The POST request should contain the following entries: mail_key: Datastore key for an Email entity. """ post_dict = request.POST mail_key = post_dict.get('mail_key', None) if not mail_key: return error_handler.logErrorAndReturnOK('No email key specified') # TODO(daniel): so ugly... try: mail_entity = db.get(mail_key) except datastore_errors.BadKeyError: mail_entity = ndb.Key(urlsafe=mail_key).get() if not mail_entity: return error_handler.logErrorAndReturnOK( 'No email entity found for key %s' % mail_key) # construct the EmailMessage from the given context loaded_context = json.loads(mail_entity.context) context = {} for key, value in loaded_context.iteritems(): # If we don't do this python will complain about kwargs not being # strings. context[str(key)] = value logging.info('Sending %s', context) message = mail.EmailMessage(**context) try: message.check_initialized() except Exception as e: logging.exception(e) context['body'] = context.get('body', '')[:10] logging.error('This message was not properly initialized: "%s"', context) mail_entity.delete() return responses.terminateTask() def txn(): """Transaction that ensures the deletion of the Email entity only if the mail has been successfully sent. """ mail_entity.delete() message.send() try: db.RunInTransaction(txn) except mail.Error as exception: # shouldn't happen because validate has been called, keeping the Email # entity for study purposes. return error_handler.logErrorAndReturnOK(exception) except (OverQuotaError, DeadlineExceededError) as e: return responses.repeatTask() # mail successfully sent return responses.terminateTask()
def sendParentalFormMail(request, *args, **kwargs): """Method executed by Task Queue API to send remainder email to existing students for sending parental consent forms. Optionally accepts the start_key entry to be present in the POST dict. Args: request: the standard Django HTTP request object """ post_dict = request.POST program_key = post_dict.get('program_key') if not program_key: return error_handler.logErrorAndReturnOK( 'The program key is not specified: %s' % post_dict) program = gci_program_logic.getFromKeyName(program_key) if not program: return error_handler.logErrorAndReturnOK( 'Invalid program key specified: %s' % program_key) fields = { 'scope': program, 'parental_form_mail': False, } start_key = post_dict.get('start_key', None) if start_key: # retrieve the last student entity that was converted start = gci_student_logic.getFromKeyName(start_key) if not start: # invalid starting student key specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid Student Key specified: %s' %(start_key)) fields['__key__ >'] = start.key() # get the first batch_size number of StudentProjects entities = gci_student_logic.getForFields(fields, limit=DEF_BATCH_SIZE) for entity in entities: gci_notifications.sendParentalConsentFormRequired( entity.user, entity.scope) gci_student_logic.updateEntityProperties( entity, {'parental_form_mail': True}) if len(entities) == DEF_BATCH_SIZE: # spawn new task starting from the last new_start = entities[DEF_BATCH_SIZE-1].key().id_or_name() # pass along these params as POST to the new task task_params = { 'start_key': new_start, 'program_key': program_key, } new_task = taskqueue.Task(params=task_params, url=DEF_SPAWN_TASK_URL) new_task.add('mail') # return OK return HttpResponse('OK')
def bulkCreateTasks(self, request, *args, **kwargs): """Task that creates GCI Tasks from bulk data specified in the POST dict. The POST dict should have the following information present: bulk_create_key: the key of the bulk_create entity """ import settings # keep track of our own timelimit (20 seconds) timelimit = 20000 timekeeper = Timekeeper(timelimit) post_dict = request.POST bulk_create_key = post_dict.get('bulk_create_key') if not bulk_create_key: return error_handler.logErrorAndReturnOK( 'Not all POST data specified in: %s' % post_dict) bulk_data = GCIBulkCreateData.get(bulk_create_key) if not bulk_data: return error_handler.logErrorAndReturnOK( 'No valid data found for key: %s' % bulk_create_key) # note that we only query for the quota once org_admin = ndb.Key.from_old_key( GCIBulkCreateData.created_by.get_value_for_datastore(bulk_data)).get() org = bulk_data.org task_quota = getRemainingTaskQuota(org) # TODO(ljvderijk): Add transactions tasks = bulk_data.tasks while len(tasks) > 0: try: # check if we have time timekeeper.ping() if settings.GCI_TASK_QUOTA_LIMIT_ENABLED and task_quota <= 0: return error_handler.logErrorAndReturnOK( 'Task quota reached for %s' %(org.name)) # remove the first task task_as_string = tasks.pop(0) loaded_task = json.loads(task_as_string) task = {} for key, value in loaded_task.iteritems(): # If we don't do this python will complain about kwargs not being # strings when we try to save the new task. task[key.encode('UTF-8')] = value logging.info('Uncleaned task: %s', task) # clean the data errors = self._cleanTask(task, org) if errors: logging.warning( 'Invalid task data uploaded, the following errors occurred: %s', errors) bulk_data.errors.append(db.Text( 'The task in row %i contains the following errors.\n %s' \ %(bulk_data.tasksRemoved(), '\n'.join(errors)))) # at-most-once semantics for creating tasks bulk_data.put() if errors: # do the next task continue # set other properties task['org'] = org # TODO(daniel): access program in more efficient way task['program'] = org_admin.program.to_old_key() task['status'] = task_model.UNPUBLISHED task['created_by'] = org_admin.to_old_key() task['modified_by'] = org_admin.to_old_key() # TODO(ljv): Remove difficulty level completely if needed. # Difficulty is hardcoded to easy since GCI2012 has no difficulty. task['difficulty_level'] = DifficultyLevel.EASY subscribers_entities = task['mentor_entities'] + [org_admin] task['subscribers'] = list(set([ent.key() for ent in subscribers_entities if ent.automatic_task_subscription])) # create the new task logging.info('Creating new task with fields: %s', task) task_entity = GCITask(**task) task_entity.put() task_quota = task_quota - 1 except DeadlineExceededError: # time to bail out break if len(tasks) == 0: # send out a message notifications.sendBulkCreationCompleted(bulk_data) bulk_data.delete() else: # there is still work to be done, do a non 500 response and requeue task_params = { 'bulk_create_key': bulk_data.key() } new_task = taskqueue.Task(params=task_params, url=BULK_CREATE_URL) # add to the gci queue new_task.add(queue_name='gci-update') # we're done here return http.HttpResponse('OK')
def updateProjectsForSurveyGroup(request, *args, **kwargs): """Updates each StudentProject for which a GradingRecord is found. Expects the following to be present in the POST dict: group_key: Specifies the GradingSurveyGroup key name. record_key: Optional, specifies the key of the last processed GradingRecord. send_mail: Optional, if this string evaluates to True mail will be send for each GradingRecord that's processed. Args: request: Django Request object """ from soc.logic.models.grading_record import logic as grading_record_logic from soc.logic.models.grading_survey_group import logic as survey_group_logic from soc.logic.models.student_project import logic as student_project_logic post_dict = request.POST group_key = post_dict.get('group_key') if not group_key: # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid updateRecordForSurveyGroup data: %s' % post_dict) # get the GradingSurveyGroup for the given keyname survey_group_entity = survey_group_logic.getFromKeyName(group_key) if not survey_group_entity: # invalid GradingSurveyGroup specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid GradingSurveyGroup specified: %s' % group_key) # check and retrieve the record_key that has been done last if 'record_key' in post_dict and post_dict['record_key'].isdigit(): record_start_key = int(post_dict['record_key']) else: record_start_key = None # get all valid StudentProjects from starting key fields = {'grading_survey_group': survey_group_entity} if record_start_key: # retrieve the last record that was done record_start = grading_record_logic.getFromID(record_start_key) if not record_start: # invalid starting record key specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid GradingRecord Key specified: %s' % (record_start_key)) fields['__key__ >'] = record_start.key() # get the first batch_size number of GradingRecords record_entities = grading_record_logic.getForFields(fields, limit=DEF_BATCH_SIZE) student_project_logic.updateProjectsForGradingRecords(record_entities) # check if we need to send an email for each GradingRecord send_mail = post_dict.get('send_mail', '') if send_mail: # enqueue a task to send mail for each GradingRecord for record_entity in record_entities: # pass along these params as POST to the new task task_params = {'record_key': record_entity.key().id_or_name()} task_url = '/tasks/grading_survey_group/mail_result' mail_task = taskqueue.Task(params=task_params, url=task_url) mail_task.add('mail') if len(record_entities) == DEF_BATCH_SIZE: # spawn new task starting from the last new_record_start = record_entities[DEF_BATCH_SIZE - 1].key().id_or_name() # pass along these params as POST to the new task task_params = { 'group_key': group_key, 'record_key': new_record_start, 'send_mail': send_mail } task_url = '/tasks/grading_survey_group/update_projects' new_task = taskqueue.Task(params=task_params, url=task_url) new_task.add() # task completed, return OK return http.HttpResponse('OK')
def start(request, *args, **kwargs): """Starts the task to find all duplicate proposals which are about to be accepted for a single GSoCProgram. Expects the following to be present in the POST dict: program_key: Specifies the program key name for which to find the duplicate proposals repeat: Specifies if a new task that must be performed again an hour later, with the same POST data Args: request: Django Request object """ from soc.logic.helper import timeline as timeline_helper post_dict = request.POST # retrieve the program_key and survey_key from POST data program_key = post_dict.get('program_key') repeat = post_dict.get('repeat') if not (program_key and repeat): # invalid task data, log and return OK return error_handler.logErrorAndReturnOK('Invalid task data: %s' % post_dict) # get the program for the given keyname program_entity = program_logic.getFromKeyName(program_key) if not program_entity: # invalid program specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid program specified: %s' % program_key) # obtain the proposal duplicate status pds_entity = pds_logic.getOrCreateForProgram(program_entity) if pds_entity.status == 'idle': # delete all old duplicates prop_duplicates = pd_logic.deleteAllForProgram(program_entity) # pass these data along params as POST to the new task task_params = {'program_key': program_key} task_url = '/tasks/gsoc/proposal_duplicates/calculate' new_task = taskqueue.Task(params=task_params, url=task_url) # add a new task that performs duplicate calculation per # organization new_task.add() # update the status of the PDS entity to processing fields = {'status': 'processing'} pds_logic.updateEntityProperties(pds_entity, fields) # Add a new clone of this task that must be performed an hour later because # the current task is part of the task that repeatedly runs but repeat # it before accepted students are announced only. if repeat == 'yes' and timeline_helper.isBeforeEvent( program_entity.timeline, 'accepted_students_announced_deadline'): # pass along these params as POST to the new task task_params = {'program_key': program_key, 'repeat': 'yes'} task_url = '/tasks/gsoc/proposal_duplicates/start' new_task = taskqueue.Task(params=task_params, url=task_url, countdown=3600) new_task.add() # return OK return http.HttpResponse()
def spawnRemindersForProjectSurvey(request, *args, **kwargs): """Spawns tasks for each StudentProject in the given Program. Expects the following to be present in the POST dict: program_key: Specifies the program key name for which to loop over all the StudentProjects for survey_key: specifies the key name for the ProjectSurvey to send reminders for survey_type: either project or grading depending on the type of Survey project_key: optional to specify which project was the last for which a task was spawn Args: request: Django Request object """ from google.appengine.ext import db from soc.logic.models.program import logic as program_logic from soc.logic.models.student_project import logic as student_project_logic # set default batch size batch_size = 10 post_dict = request.POST # retrieve the program_key and survey_key from POST data program_key = post_dict.get('program_key') survey_key = post_dict.get('survey_key') survey_type = post_dict.get('survey_type') if not (program_key and survey_key and survey_type): # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid sendRemindersForProjectSurvey data: %s' % post_dict) # get the program for the given keyname program_entity = program_logic.getFromKeyName(program_key) if not program_entity: # invalid program specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid program specified: %s' % program_key) # check and retrieve the project_key that has been done last if 'project_key' in post_dict: project_start_key = post_dict['project_key'] else: project_start_key = None # get all valid StudentProjects from starting key fields = {'program': program_entity, 'status': 'accepted'} if project_start_key: # retrieve the last project that was done project_start = student_project_logic.getFromKeyName(project_start_key) if not project_start: # invalid starting project key specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid Student Project Key specified: %s' %(project_start_key)) fields['__key__ >'] = project_start.key() project_entities = student_project_logic.getForFields(fields, limit=batch_size) for project_entity in project_entities: # pass along these params as POST to the new task task_params = {'survey_key': survey_key, 'survey_type': survey_type, 'project_key': project_entity.key().id_or_name()} task_url = '/tasks/surveys/projects/send_reminder/send' new_task = taskqueue.Task(params=task_params, url=task_url) new_task.add('mail') if len(project_entities) == batch_size: # spawn new task starting from the last new_project_start = project_entities[batch_size-1].key().id_or_name() # pass along these params as POST to the new task task_params = {'program_key': program_key, 'survey_key': survey_key, 'survey_type': survey_type, 'project_key': new_project_start} task_url = '/tasks/surveys/projects/send_reminder/spawn' new_task = taskqueue.Task(params=task_params, url=task_url) new_task.add() # return OK return http.HttpResponse()
def updateProjectsForSurveyGroup(request, *args, **kwargs): """Updates each StudentProject for which a GradingRecord is found. Expects the following to be present in the POST dict: group_key: Specifies the GradingSurveyGroup key name. record_key: Optional, specifies the key of the last processed GradingRecord. send_mail: Optional, if this string evaluates to True mail will be send for each GradingRecord that's processed. Args: request: Django Request object """ from soc.logic.models.grading_record import logic as grading_record_logic from soc.logic.models.grading_survey_group import logic as survey_group_logic from soc.logic.models.student_project import logic as student_project_logic post_dict = request.POST group_key = post_dict.get('group_key') if not group_key: # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid updateRecordForSurveyGroup data: %s' % post_dict) # get the GradingSurveyGroup for the given keyname survey_group_entity = survey_group_logic.getFromKeyName(group_key) if not survey_group_entity: # invalid GradingSurveyGroup specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid GradingSurveyGroup specified: %s' % group_key) # check and retrieve the record_key that has been done last if 'record_key' in post_dict and post_dict['record_key'].isdigit(): record_start_key = int(post_dict['record_key']) else: record_start_key = None # get all valid StudentProjects from starting key fields = {'grading_survey_group': survey_group_entity} if record_start_key: # retrieve the last record that was done record_start = grading_record_logic.getFromID(record_start_key) if not record_start: # invalid starting record key specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid GradingRecord Key specified: %s' %(record_start_key)) fields['__key__ >'] = record_start.key() # get the first batch_size number of GradingRecords record_entities = grading_record_logic.getForFields(fields, limit=DEF_BATCH_SIZE) student_project_logic.updateProjectsForGradingRecords(record_entities) # check if we need to send an email for each GradingRecord send_mail = post_dict.get('send_mail', '') if send_mail: # enqueue a task to send mail for each GradingRecord for record_entity in record_entities: # pass along these params as POST to the new task task_params = {'record_key': record_entity.key().id_or_name()} task_url = '/tasks/grading_survey_group/mail_result' mail_task = taskqueue.Task(params=task_params, url=task_url) mail_task.add('mail') if len(record_entities) == DEF_BATCH_SIZE: # spawn new task starting from the last new_record_start = record_entities[DEF_BATCH_SIZE-1].key().id_or_name() # pass along these params as POST to the new task task_params = {'group_key': group_key, 'record_key': new_record_start, 'send_mail': send_mail} task_url = '/tasks/grading_survey_group/update_projects' new_task = taskqueue.Task(params=task_params, url=task_url) new_task.add() # task completed, return OK return http.HttpResponse('OK')
def sendSurveyReminderForProject(self, request, *args, **kwargs): """Sends a reminder mail for a given GSoCProject and Survey. A reminder is only send if no record is on file for the given Survey and GSoCProject. Expects the following to be present in the POST dict: survey_key: specifies the key name for the ProjectSurvey to send reminders for survey_type: either project or grading depending on the type of Survey project_key: encoded Key which specifies the project to send a reminder for Args: request: Django Request object """ post_dict = request.POST project_key = post_dict.get('project_key') survey_key = post_dict.get('survey_key') survey_type = post_dict.get('survey_type') if not (project_key and survey_key and survey_type): # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid sendSurveyReminderForProject data: %s' % post_dict) # set model depending on survey type specified in POST if survey_type == 'project': survey_model = ProjectSurvey record_model = GSoCProjectSurveyRecord elif survey_type == 'grading': survey_model = GradingProjectSurvey record_model = GSoCGradingProjectSurveyRecord else: return error_handler.logErrorAndReturnOK( '%s is an invalid survey_type' %survey_type) # retrieve the project and survey project_key = db.Key(project_key) project = GSoCProject.get(project_key) if not project: # no existing project found, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid project specified %s:' % project_key) survey = survey_model.get_by_key_name(survey_key) if not survey: # no existing survey found, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid survey specified %s:' % survey_key) # try to retrieve an existing record q = record_model.all() q.filter('project', project) q.filter('survey', survey) record = q.get() if not record: # send reminder email because we found no record student = ndb.Key.from_old_key(project.parent_key()).get() site_entity = site.singleton() if survey_type == 'project': url_name = 'gsoc_take_student_evaluation' to_name = student.public_name to_address = student.contact.email mail_template = 'modules/gsoc/reminder/student_eval_reminder.html' elif survey_type == 'grading': url_name = 'gsoc_take_mentor_evaluation' mentors = ndb.get_multi(map(ndb.Key.from_old_key, project.mentors)) to_address = [mentor.contact.email for mentor in mentors] to_name = 'mentor(s) for project "%s"' % (project.title) mail_template = ( 'modules/gsoc/reminder/mentor_eval_reminder.html') program = project.program hostname = site.getHostname() url_kwargs = { 'sponsor': program_logic.getSponsorKey(program).name(), 'program': program.link_id, 'survey': survey.link_id, 'user': student.profile_id, 'id': str(project.key().id()), } url_path_and_query = reverse(url_name, kwargs=url_kwargs) survey_url = '%s://%s%s' % ('http', hostname, url_path_and_query) # set the context for the mail template mail_context = { 'student_name': student.public_name, 'project_title': project.title, 'survey_url': survey_url, 'survey_end': survey.survey_end, 'to_name': to_name, 'site_name': site_entity.site_name, 'sender_name': "The %s Team" % site_entity.site_name, } # set the sender _, sender_address = mail_dispatcher.getDefaultMailSender() mail_context['sender'] = sender_address # set the receiver and subject mail_context['to'] = to_address mail_context['subject'] = ( 'Evaluation "%s" Reminder' % survey.title) # find all org admins for the project's organization org_key = ndb.Key.from_old_key( GSoCProject.org.get_value_for_datastore(project)) org_admins = profile_logic.getOrgAdmins(org_key) # collect email addresses for all found org admins org_admin_addresses = [] for org_admin in org_admins: org_admin_addresses.append(org_admin.contact.email) if org_admin_addresses: mail_context['cc'] = org_admin_addresses # send out the email mail_dispatcher.sendMailFromTemplate(mail_template, mail_context) # return OK return http.HttpResponse()
def sendMailAboutGradingRecordResult(request, *args, **kwargs): """Sends out a mail about the result of one GradingRecord. Expects the following to be present in the POST dict: record_key: Specifies the key for the record to process. Args: request: Django Request object """ from soc.logic import mail_dispatcher from soc.logic.models.grading_record import logic as grading_record_logic from soc.logic.models.org_admin import logic as org_admin_logic from soc.logic.models.site import logic as site_logic post_dict = request.POST # check and retrieve the record_key that has been done last if 'record_key' in post_dict and post_dict['record_key'].isdigit(): record_key = int(post_dict['record_key']) else: record_key = None if not record_key: # no GradingRecord key specified, log and return OK error_handler.logErrorAndReturnOK( 'No valid record_key specified in POST data: %s' % request.POST) record_entity = grading_record_logic.getFromID(record_key) if not record_entity: # no valid GradingRecord key specified, log and return OK error_handler.logErrorAndReturnOK( 'No valid GradingRecord key specified: %s' % record_key) survey_group_entity = record_entity.grading_survey_group project_entity = record_entity.project student_entity = project_entity.student mentor_entity = project_entity.mentor org_entity = project_entity.scope site_entity = site_logic.getSingleton() mail_context = { 'survey_group': survey_group_entity, 'grading_record': record_entity, 'project': project_entity, 'organization': org_entity, 'site_name': site_entity.site_name, 'to_name': student_entity.name() } # set the sender (sender, sender_address) = mail_dispatcher.getDefaultMailSender() mail_context['sender'] = sender_address # set the receiver and subject mail_context['to'] = student_entity.email mail_context['cc'] = [mentor_entity.email] mail_context['subject'] = '%s results processed for %s' %( survey_group_entity.name, project_entity.title) # find all org admins for the project's organization fields = {'scope': org_entity, 'status': 'active'} org_admin_entities = org_admin_logic.getForFields(fields) # collect email addresses for all found org admins org_admin_addresses = [] for org_admin_entity in org_admin_entities: org_admin_addresses.append(org_admin_entity.email) if org_admin_addresses: mail_context['cc'].extend(org_admin_addresses) # send out the email using a template mail_template = 'soc/grading_record/mail/result.html' mail_dispatcher.sendMailFromTemplate(mail_template, mail_context) # return OK return http.HttpResponse()
def bulkCreateTasks(self, request, *args, **kwargs): """Task that creates GCI Tasks from bulk data specified in the POST dict. The POST dict should have the following information present: bulk_create_key: the key of the bulk_create entity """ import settings # keep track of our own timelimit (20 seconds) timelimit = 20000 timekeeper = Timekeeper(timelimit) post_dict = request.POST bulk_create_key = post_dict.get('bulk_create_key') if not bulk_create_key: return error_handler.logErrorAndReturnOK( 'Not all POST data specified in: %s' % post_dict) bulk_data = GCIBulkCreateData.get(bulk_create_key) if not bulk_data: return error_handler.logErrorAndReturnOK( 'No valid data found for key: %s' % bulk_create_key) # note that we only query for the quota once org_admin = ndb.Key.from_old_key( GCIBulkCreateData.created_by.get_value_for_datastore( bulk_data)).get() org = bulk_data.org task_quota = getRemainingTaskQuota(org) # TODO(ljvderijk): Add transactions tasks = bulk_data.tasks while len(tasks) > 0: try: # check if we have time timekeeper.ping() if settings.GCI_TASK_QUOTA_LIMIT_ENABLED and task_quota <= 0: return error_handler.logErrorAndReturnOK( 'Task quota reached for %s' % (org.name)) # remove the first task task_as_string = tasks.pop(0) loaded_task = json.loads(task_as_string) task = {} for key, value in loaded_task.iteritems(): # If we don't do this python will complain about kwargs not being # strings when we try to save the new task. task[key.encode('UTF-8')] = value logging.info('Uncleaned task: %s', task) # clean the data errors = self._cleanTask(task, org) if errors: logging.warning( 'Invalid task data uploaded, the following errors occurred: %s', errors) bulk_data.errors.append(db.Text( 'The task in row %i contains the following errors.\n %s' \ %(bulk_data.tasksRemoved(), '\n'.join(errors)))) # at-most-once semantics for creating tasks bulk_data.put() if errors: # do the next task continue # set other properties task['org'] = org # TODO(daniel): access program in more efficient way task['program'] = org_admin.program.to_old_key() task['status'] = task_model.UNPUBLISHED task['created_by'] = org_admin.to_old_key() task['modified_by'] = org_admin.to_old_key() # TODO(ljv): Remove difficulty level completely if needed. # Difficulty is hardcoded to easy since GCI2012 has no difficulty. task['difficulty_level'] = DifficultyLevel.EASY subscribers_entities = task['mentor_entities'] + [org_admin] task['subscribers'] = list( set([ ent.key() for ent in subscribers_entities if ent.automatic_task_subscription ])) # create the new task logging.info('Creating new task with fields: %s', task) task_entity = GCITask(**task) task_entity.put() task_quota = task_quota - 1 except DeadlineExceededError: # time to bail out break if len(tasks) == 0: # send out a message notifications.sendBulkCreationCompleted(bulk_data) bulk_data.delete() else: # there is still work to be done, do a non 500 response and requeue task_params = {'bulk_create_key': bulk_data.key()} new_task = taskqueue.Task(params=task_params, url=BULK_CREATE_URL) # add to the gci queue new_task.add(queue_name='gci-update') # we're done here return http.HttpResponse('OK')
def updateOrCreateRecordsForSurveyGroup(request, *args, **kwargs): """Updates or creates GradingRecords for the given GradingSurveyGroup. Expects the following to be present in the POST dict: group_key: Specifies the GradingSurveyGroup key name. project_key: optional to specify which project was the last for which this task was run Args: request: Django Request object """ from soc.logic.models.grading_record import logic as grading_record_logic from soc.logic.models.grading_survey_group import logic as survey_group_logic from soc.logic.models.student_project import logic as student_project_logic post_dict = request.POST group_key = post_dict.get('group_key') if not group_key: # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid updateRecordForSurveyGroup data: %s' % post_dict) # get the GradingSurveyGroup for the given keyname survey_group_entity = survey_group_logic.getFromKeyName(group_key) if not survey_group_entity: # invalid GradingSurveyGroup specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid GradingSurveyGroup specified: %s' % group_key) # check and retrieve the project_key that has been done last if 'project_key' in post_dict: project_start_key = post_dict['project_key'] else: project_start_key = None # get all valid StudentProjects from starting key fields = {'program': survey_group_entity.scope, 'status': ['accepted', 'failed', 'completed']} if project_start_key: # retrieve the last project that was done project_start = student_project_logic.getFromKeyName(project_start_key) if not project_start: # invalid starting project key specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid Student Project Key specified: %s' %(project_start_key)) fields['__key__ >'] = project_start.key() # get the first batch_size number of StudentProjects project_entities = student_project_logic.getForFields(fields, limit=DEF_BATCH_SIZE) # update/create and batch put the new GradingRecords grading_record_logic.updateOrCreateRecordsFor(survey_group_entity, project_entities) if len(project_entities) == DEF_BATCH_SIZE: # spawn new task starting from the last new_project_start = project_entities[DEF_BATCH_SIZE-1].key().id_or_name() # pass along these params as POST to the new task task_params = {'group_key': group_key, 'project_key': new_project_start} task_url = '/tasks/grading_survey_group/update_records' new_task = taskqueue.Task(params=task_params, url=task_url) new_task.add() else: # task completed, update timestamp for last update complete fields = {'last_update_complete': datetime.datetime.now()} survey_group_logic.updateEntityProperties(survey_group_entity, fields) # task completed, return OK return http.HttpResponse('OK')
def start(self, request, *args, **kwargs): """Starts the task to find all duplicate proposals which are about to be accepted for a single GSoCProgram. Expects the following to be present in the POST dict: program_key: Specifies the program key name for which to find the duplicate proposals repeat: Specifies if a new task that must be performed again an hour later, with the same POST data Args: request: Django Request object """ from soc.logic.helper import timeline as timeline_helper post_dict = request.POST # retrieve the program_key and repeat option from POST data program_key = post_dict.get('program_key') repeat = post_dict.get('repeat') if not (program_key and repeat): # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid task data: %s' % post_dict) # get the program for the given keyname program_entity = GSoCProgram.get_by_key_name(program_key) if not program_entity: # invalid program specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid program specified: %s' % program_key) # obtain the proposal duplicate status pds_entity = duplicates_logic.getOrCreateStatusForProgram(program_entity) if pds_entity.status == 'idle': # delete all old duplicates duplicates_logic.deleteAllForProgram(program_entity) # pass these data along params as POST to the new task task_params = {'program_key': program_key} task_url = '/tasks/gsoc/proposal_duplicates/calculate' new_task = taskqueue.Task(params=task_params, url=task_url) def txn(): # add a new task that performs duplicate calculation per # organization new_task.add(transactional=True) # update the status of the PDS entity to processing pds_entity.status = 'processing' pds_entity.put() db.RunInTransaction(txn) # Add a new clone of this task that must be performed an hour later because # the current task is part of the task that repeatedly runs but repeat # it before accepted students are announced only. if repeat == 'yes' and timeline_helper.isBeforeEvent( program_entity.timeline, 'accepted_students_announced_deadline'): # pass along these params as POST to the new task task_params = {'program_key': program_key, 'repeat': 'yes'} task_url = '/tasks/gsoc/proposal_duplicates/start' new_task = taskqueue.Task(params=task_params, url=task_url, countdown=3600) new_task.add() # return OK return http.HttpResponse()
def createNotificationMail(request, *args, **kwargs): """Appengine task that sends mail to the subscribed users. Expects the following to be present in the POST dict: comment_key: Specifies the comment id for which to send the notifications task_key: Specifies the task key name for which the comment belongs to Args: request: Django Request object """ from soc.modules.gci.logic.helper import notifications as gci_notifications from soc.modules.gci.logic.models import comment as gci_comment_logic from soc.modules.gci.logic.models import task_subscription as \ gci_task_subscription_logic # set default batch size batch_size = 10 post_dict = request.POST comment_key = post_dict.get('comment_key') task_key = post_dict.get('task_key') if not (comment_key and task_key): # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid createNotificationMail data: %s' % post_dict) comment_key = long(comment_key) # get the task entity under which the specified comment was made task_entity = gci_task_logic.logic.getFromKeyName(task_key) # get the comment for the given id comment_entity = gci_comment_logic.logic.getFromID(comment_key, task_entity) if not comment_entity: # invalid comment specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid comment specified: %s/%s' % (comment_key, task_key)) # check and retrieve the subscriber_start_key that has been done last idx = post_dict.get('subscriber_start_index', '') subscriber_start_index = int(idx) if idx.isdigit() else 0 # get all subscribers to GCI task fields = { 'task': task_entity, } ts_entity = gci_task_subscription_logic.logic.getForFields(fields, unique=True) subscribers = db.get( ts_entity.subscribers[subscriber_start_index:subscriber_start_index + batch_size]) task_url = "http://%(host)s%(task)s" % { 'host': system.getHostname(), 'task': redirects.getPublicRedirect(task_entity, {'url_name': 'gci/task'}), } # create the data for the mail to be sent message_properties = { 'task_url': task_url, 'redirect_url': "%(task_url)s#c%(cid)d" % { 'task_url': task_url, 'cid': comment_entity.key().id_or_name() }, 'comment_entity': comment_entity, 'task_entity': task_entity, } subject = DEF_TASK_UPDATE_SUBJECT_FMT % { 'program_name': task_entity.program.short_name, 'title': task_entity.title, } for subscriber in subscribers: gci_notifications.sendTaskUpdateMail(subscriber, subject, message_properties) if len(subscribers) == batch_size: # spawn task for sending out notifications to next set of subscribers next_start = subscriber_start_index + batch_size task_params = { 'comment_key': comment_key, 'task_key': task_key, 'subscriber_start_index': next_start } task_url = '/tasks/gci/task/mail/create' new_task = taskqueue.Task(params=task_params, url=task_url) new_task.add('mail') # return OK return http.HttpResponse()
def calculate(request, *args, **kwargs): """Calculates the duplicate proposals in a given program for a student on a per Organization basis. Expects the following to be present in the POST dict: program_key: Specifies the program key name for which to find the duplicate proposals org_cursor: Specifies the organization datastore cursor from which to start the processing of finding the duplicate proposals Args: request: Django Request object """ from soc.modules.gsoc.logic.models.student_proposal import logic \ as sp_logic post_dict = request.POST program_key = post_dict.get('program_key') if not program_key: # invalid task data, log and return OK return error_handler.logErrorAndReturnOK('Invalid program key: %s' % post_dict) program_entity = program_logic.getFromKeyName(program_key) if not program_entity: # invalid program specified, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid program specified: %s' % program_key) fields = {'scope': program_entity, 'slots >': 0, 'status': 'active'} # get the organization and update the cursor if possible q = org_logic.getQueryForFields(fields) # retrieve the org_cursor from POST data org_cursor = post_dict.get('org_cursor') if org_cursor: org_cursor = str(org_cursor) q.with_cursor(org_cursor) result = q.fetch(1) # update the cursor org_cursor = q.cursor() if result: org_entity = result[0] # get all the proposals likely to be accepted in the program accepted_proposals = sp_logic.getProposalsToBeAcceptedForOrg( org_entity) for ap in accepted_proposals: student_entity = ap.scope proposal_duplicate = pd_logic.getForFields( {'student': student_entity}, unique=True) if proposal_duplicate and ap.key( ) not in proposal_duplicate.duplicates: # non-counted (to-be) accepted proposal found pd_fields = { 'duplicates': proposal_duplicate.duplicates + [ap.key()], } pd_fields['is_duplicate'] = len(pd_fields['duplicates']) >= 2 if org_entity.key() not in proposal_duplicate.orgs: pd_fields['orgs'] = proposal_duplicate.orgs + [ org_entity.key() ] proposal_duplicate = pd_logic.updateEntityProperties( proposal_duplicate, pd_fields) else: pd_fields = { 'program': program_entity, 'student': student_entity, 'orgs': [org_entity.key()], 'duplicates': [ap.key()], 'is_duplicate': False } proposal_duplicate = pd_logic.updateOrCreateFromFields( pd_fields) # Adds a new task that performs duplicate calculation for # the next organization. task_params = { 'program_key': program_key, 'org_cursor': unicode(org_cursor) } task_url = '/tasks/gsoc/proposal_duplicates/calculate' new_task = taskqueue.Task(params=task_params, url=task_url) new_task.add() else: # There aren't any more organizations to process. So delete # all the proposals for which there are not more than one # proposal for duplicates property. pd_logic.deleteAllForProgram(program_entity, non_dupes_only=True) # update the proposal duplicate status and its timestamp pds_entity = pds_logic.getOrCreateForProgram(program_entity) new_fields = { 'status': 'idle', 'calculated_on': datetime.datetime.now() } pds_logic.updateEntityProperties(pds_entity, new_fields) # return OK return http.HttpResponse()
def sendSurveyReminderForProject(self, request, *args, **kwargs): """Sends a reminder mail for a given GSoCProject and Survey. A reminder is only send if no record is on file for the given Survey and GSoCProject. Expects the following to be present in the POST dict: survey_key: specifies the key name for the ProjectSurvey to send reminders for survey_type: either project or grading depending on the type of Survey project_key: encoded Key which specifies the project to send a reminder for Args: request: Django Request object """ post_dict = request.POST project_key = post_dict.get('project_key') survey_key = post_dict.get('survey_key') survey_type = post_dict.get('survey_type') if not (project_key and survey_key and survey_type): # invalid task data, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid sendSurveyReminderForProject data: %s' % post_dict) # set model depending on survey type specified in POST if survey_type == 'project': survey_model = ProjectSurvey record_model = GSoCProjectSurveyRecord elif survey_type == 'grading': survey_model = GradingProjectSurvey record_model = GSoCGradingProjectSurveyRecord else: return error_handler.logErrorAndReturnOK( '%s is an invalid survey_type' % survey_type) # retrieve the project and survey project_key = db.Key(project_key) project = GSoCProject.get(project_key) if not project: # no existing project found, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid project specified %s:' % project_key) survey = survey_model.get_by_key_name(survey_key) if not survey: # no existing survey found, log and return OK return error_handler.logErrorAndReturnOK( 'Invalid survey specified %s:' % survey_key) # try to retrieve an existing record q = record_model.all() q.filter('project', project) q.filter('survey', survey) record = q.get() if not record: # send reminder email because we found no record student = ndb.Key.from_old_key(project.parent_key()).get() site_entity = site.singleton() if survey_type == 'project': url_name = 'gsoc_take_student_evaluation' to_name = student.public_name to_address = student.contact.email mail_template = 'modules/gsoc/reminder/student_eval_reminder.html' elif survey_type == 'grading': url_name = 'gsoc_take_mentor_evaluation' mentors = ndb.get_multi( map(ndb.Key.from_old_key, project.mentors)) to_address = [mentor.contact.email for mentor in mentors] to_name = 'mentor(s) for project "%s"' % (project.title) mail_template = ( 'modules/gsoc/reminder/mentor_eval_reminder.html') program = project.program hostname = site.getHostname() url_kwargs = { 'sponsor': program_logic.getSponsorKey(program).name(), 'program': program.link_id, 'survey': survey.link_id, 'user': student.profile_id, 'id': str(project.key().id()), } url_path_and_query = reverse(url_name, kwargs=url_kwargs) survey_url = '%s://%s%s' % ('http', hostname, url_path_and_query) # set the context for the mail template mail_context = { 'student_name': student.public_name, 'project_title': project.title, 'survey_url': survey_url, 'survey_end': survey.survey_end, 'to_name': to_name, 'site_name': site_entity.site_name, 'sender_name': "The %s Team" % site_entity.site_name, } # set the sender _, sender_address = mail_dispatcher.getDefaultMailSender() mail_context['sender'] = sender_address # set the receiver and subject mail_context['to'] = to_address mail_context['subject'] = ('Evaluation "%s" Reminder' % survey.title) # find all org admins for the project's organization org_key = ndb.Key.from_old_key( GSoCProject.org.get_value_for_datastore(project)) org_admins = profile_logic.getOrgAdmins(org_key) # collect email addresses for all found org admins org_admin_addresses = [] for org_admin in org_admins: org_admin_addresses.append(org_admin.contact.email) if org_admin_addresses: mail_context['cc'] = org_admin_addresses # send out the email mail_dispatcher.sendMailFromTemplate(mail_template, mail_context) # return OK return http.HttpResponse()