def accept_proposals(request, *args, **kwargs): """Accept proposals for an organization """ params = request.POST # Setup an artifical request deadline timelimit = int(params["timelimit"]) timekeeper = Timekeeper(timelimit) # Query proposals based on status org = org_logic.getFromKeyName(params["orgkey"]) proposals = student_proposal_logic.getProposalsToBeAcceptedForOrg(org) # Accept proposals try: for remain, proposal in timekeeper.iterate(proposals): logging.info("accept %s %s %s", remain, org.key(), proposal.key()) accept_proposal(proposal) accept_proposal_email(proposal) # Requeue this task for continuation except DeadlineExceededError: taskqueue.add(url=request.path, params=params) return responses.terminateTask() # Reject remaining proposals taskqueue.add(url=params["nextpath"], params=params) return responses.terminateTask()
def reject_proposals(request, *args, **kwargs): """Reject proposals for an org_logic """ params = request.POST # Setup an artifical request deadline timelimit = int(params["timelimit"]) timekeeper = Timekeeper(timelimit) # Query proposals org = org_logic.getFromKeyName(params["orgkey"]) proposals = reject_proposals_query(org) # Reject proposals try: for remain, proposal in timekeeper.iterate(proposals): logging.info("reject %s %s %s", remain, org.key(), proposal.key()) reject_proposal(proposal) reject_proposal_email(proposal) # Requeue this task for continuation except DeadlineExceededError: taskqueue.add(url=request.path, params=params) # Exit this task successfully return responses.terminateTask()
def rejectProposals(self, request, *args, **kwargs): """Reject proposals for an single organization. """ params = request.POST # Setup an artifical request deadline timekeeper = Timekeeper(20000) # Query proposals org = soc_org_model.SOCOrganization.get_by_id(params['org_key']) q = GSoCProposal.all() q.filter('org', org.key.to_old_key()) q.filter('status', 'pending') # Reject proposals try: for remain, proposal in timekeeper.iterate(q): logging.info("reject %s %s %s", remain, org.key.id(), proposal.key()) self.rejectProposal(proposal) # Requeue this task for continuation except DeadlineExceededError: taskqueue.add(url=request.path, params=params) # Exit this task successfully return responses.terminateTask()
def acceptProposals(self, request, *args, **kwargs): """Accept proposals for an single organization. POST Args: org_key: The key of the organization """ params = request.POST # Setup an artifical request deadline timekeeper = Timekeeper(20000) # Query proposals based on status org = soc_org_model.SOCOrganization.get_by_id(params['org_key']) proposals = proposal_logic.getProposalsToBeAcceptedForOrg(org) # Accept proposals try: for remain, proposal in timekeeper.iterate(proposals): logging.info("accept %s %s %s", remain, org.key.id(), proposal.key()) self.acceptProposal(proposal) # Requeue this task for continuation except DeadlineExceededError: taskqueue.add(url=request.path, params=params) return responses.terminateTask() # Reject remaining proposals taskqueue.add(url='/tasks/gsoc/accept_proposals/reject', params=params) return responses.terminateTask()
def assignSlots(request, *args, **kwargs): """Sets the slots attribute for each organization entity POST Args: slots: an org_key:num_slots JSON dictionary """ # Setup an artifical request deadline timelimit = int(request.REQUEST.get("timelimit", 20000)) timekeeper = Timekeeper(timelimit) program_key = request.REQUEST.get("programkey") last_key = request.REQUEST.get("lastkey", "") program = program_logic.getFromKeyName(program_key) # Copy for modification below params = request.POST.copy() params["timelimit"] = timelimit # Parse the JSON org:slots dictionary slots = simplejson.loads(program.slots_allocation) org_keys = [i for i in sorted(slots.keys()) if i > last_key] logging.info(org_keys) # Assign slots for each organization try: for clock, org_key in timekeeper.iterate(org_keys): logging.info("%s %s %s", request.path, clock, org_key) org_slots = slots[org_key] # Get the organization entity org = org_logic.getFromKeyFields({ 'link_id': org_key, 'scope_path': program_key, }) if not org: logging.error("no such org '%s'/'%s'" % (program_key, org_key)) continue # Count proposals and mentors org.slots = int(org_slots['slots']) org.nr_applications, org.nr_mentors = countProposals(org) # Update the organization entity org.put() # Mark the organization as done last_key = org_key # Requeue this task for continuation except DeadlineExceededError: params["lastkey"] = last_key taskqueue.add(url=request.path, params=params) # Exit this task successfully return responses.terminateTask()
def bulkCreateTasks(self, request, *args, **kwargs): """Task that creates GCI Tasks from bulk data specified in the POST dict. The POST dict should have the following information present: bulk_create_key: the key of the bulk_create entity """ import settings # keep track of our own timelimit (20 seconds) timelimit = 20000 timekeeper = Timekeeper(timelimit) post_dict = request.POST bulk_create_key = post_dict.get('bulk_create_key') if not bulk_create_key: return error_handler.logErrorAndReturnOK( 'Not all POST data specified in: %s' % post_dict) bulk_data = GCIBulkCreateData.get(bulk_create_key) if not bulk_data: return error_handler.logErrorAndReturnOK( 'No valid data found for key: %s' % bulk_create_key) # note that we only query for the quota once org_admin = ndb.Key.from_old_key( GCIBulkCreateData.created_by.get_value_for_datastore( bulk_data)).get() org = bulk_data.org task_quota = getRemainingTaskQuota(org) # TODO(ljvderijk): Add transactions tasks = bulk_data.tasks while len(tasks) > 0: try: # check if we have time timekeeper.ping() if settings.GCI_TASK_QUOTA_LIMIT_ENABLED and task_quota <= 0: return error_handler.logErrorAndReturnOK( 'Task quota reached for %s' % (org.name)) # remove the first task task_as_string = tasks.pop(0) loaded_task = json.loads(task_as_string) task = {} for key, value in loaded_task.iteritems(): # If we don't do this python will complain about kwargs not being # strings when we try to save the new task. task[key.encode('UTF-8')] = value logging.info('Uncleaned task: %s', task) # clean the data errors = self._cleanTask(task, org) if errors: logging.warning( 'Invalid task data uploaded, the following errors occurred: %s', errors) bulk_data.errors.append(db.Text( 'The task in row %i contains the following errors.\n %s' \ %(bulk_data.tasksRemoved(), '\n'.join(errors)))) # at-most-once semantics for creating tasks bulk_data.put() if errors: # do the next task continue # set other properties task['org'] = org # TODO(daniel): access program in more efficient way task['program'] = org_admin.program.to_old_key() task['status'] = task_model.UNPUBLISHED task['created_by'] = org_admin.to_old_key() task['modified_by'] = org_admin.to_old_key() # TODO(ljv): Remove difficulty level completely if needed. # Difficulty is hardcoded to easy since GCI2012 has no difficulty. task['difficulty_level'] = DifficultyLevel.EASY subscribers_entities = task['mentor_entities'] + [org_admin] task['subscribers'] = list( set([ ent.key() for ent in subscribers_entities if ent.automatic_task_subscription ])) # create the new task logging.info('Creating new task with fields: %s', task) task_entity = GCITask(**task) task_entity.put() task_quota = task_quota - 1 except DeadlineExceededError: # time to bail out break if len(tasks) == 0: # send out a message notifications.sendBulkCreationCompleted(bulk_data) bulk_data.delete() else: # there is still work to be done, do a non 500 response and requeue task_params = {'bulk_create_key': bulk_data.key()} new_task = taskqueue.Task(params=task_params, url=BULK_CREATE_URL) # add to the gci queue new_task.add(queue_name='gci-update') # we're done here return http.HttpResponse('OK')
def _continueShipmentSync(self, request, *args, **kwargs): """Continue syncing shipment data. POST Args: program_key: the key of the program which sync is being done for. shipment_info_id: id of the shipment info object that task is running for. column_indexes: column indexes for specific columns in JSON format. sheet_rows: spreadsheets CSV chunk data in JSON format. """ timekeeper = Timekeeper(20000) params = dicts.merge(request.POST, request.GET) if 'program_key' not in params: logging.error("missing program_key in params: '%s'", params) return responses.terminateTask() if 'shipment_info_id' not in params: logging.error("missing shipment_info_id in params: '%s'", params) return responses.terminateTask() self.setProgram(params['program_key']) self.setShipmentInfo(int(params['shipment_info_id'])) if 'sheet_rows' not in params: logging.error("missing sheet_rows data in params: '%s'", params) return responses.terminateTask() if 'column_indexes' not in params: logging.error("missing column_indexes data in params: '%s'", params) return responses.terminateTask() column_indexes = json.loads(params['column_indexes']) sheet_rows = json.loads(params['sheet_rows']) try: for remain, row in timekeeper.iterate(sheet_rows): if len(row) < len(column_indexes): row.extend((len(column_indexes) - len(row)) * ['']) data = self.getRowData(row, column_indexes) username = data['username'] profile = profile_logic.getProfileForUsername(username, self.program_key) if not profile: logging.error("Profile with username '%s' for program '%s' is not found", username, self.ndb_program_key.id()) continue #continue to next row if not profile.is_student: logging.error("Profile with username '%s' is not a student", username) continue tracking = data['tracking'] self.updateShipmentDataForStudent(profile, tracking) except DeadlineExceededError: if remain: remaining_rows = sheet_rows[(-1 * remain):] params = { 'program_key': params.get('program_key'), 'sheet_rows': json.dumps(remaining_rows), 'column_indexes': params.get('column_indexes'), 'shipment_info_id': params.get('shipment_info_id'), } task_continue_url = links.SOC_LINKER.site(url_names.GSOC_SHIPMENT_TASK_CONTINUE) taskqueue.add(url=task_continue_url, params=params) return responses.terminateTask() self.finishSync() return responses.terminateTask()
def bulkCreateTasks(self, request, *args, **kwargs): """Task that creates GCI Tasks from bulk data specified in the POST dict. The POST dict should have the following information present: bulk_create_key: the key of the bulk_create entity """ import settings # keep track of our own timelimit (20 seconds) timelimit = 20000 timekeeper = Timekeeper(timelimit) post_dict = request.POST bulk_create_key = post_dict.get('bulk_create_key') if not bulk_create_key: return error_handler.logErrorAndReturnOK( 'Not all POST data specified in: %s' % post_dict) bulk_data = GCIBulkCreateData.get(bulk_create_key) if not bulk_data: return error_handler.logErrorAndReturnOK( 'No valid data found for key: %s' % bulk_create_key) # note that we only query for the quota once org_admin = ndb.Key.from_old_key( GCIBulkCreateData.created_by.get_value_for_datastore(bulk_data)).get() org = bulk_data.org task_quota = getRemainingTaskQuota(org) # TODO(ljvderijk): Add transactions tasks = bulk_data.tasks while len(tasks) > 0: try: # check if we have time timekeeper.ping() if settings.GCI_TASK_QUOTA_LIMIT_ENABLED and task_quota <= 0: return error_handler.logErrorAndReturnOK( 'Task quota reached for %s' %(org.name)) # remove the first task task_as_string = tasks.pop(0) loaded_task = json.loads(task_as_string) task = {} for key, value in loaded_task.iteritems(): # If we don't do this python will complain about kwargs not being # strings when we try to save the new task. task[key.encode('UTF-8')] = value logging.info('Uncleaned task: %s', task) # clean the data errors = self._cleanTask(task, org) if errors: logging.warning( 'Invalid task data uploaded, the following errors occurred: %s', errors) bulk_data.errors.append(db.Text( 'The task in row %i contains the following errors.\n %s' \ %(bulk_data.tasksRemoved(), '\n'.join(errors)))) # at-most-once semantics for creating tasks bulk_data.put() if errors: # do the next task continue # set other properties task['org'] = org # TODO(daniel): access program in more efficient way task['program'] = org_admin.program.to_old_key() task['status'] = task_model.UNPUBLISHED task['created_by'] = org_admin.to_old_key() task['modified_by'] = org_admin.to_old_key() # TODO(ljv): Remove difficulty level completely if needed. # Difficulty is hardcoded to easy since GCI2012 has no difficulty. task['difficulty_level'] = DifficultyLevel.EASY subscribers_entities = task['mentor_entities'] + [org_admin] task['subscribers'] = list(set([ent.key() for ent in subscribers_entities if ent.automatic_task_subscription])) # create the new task logging.info('Creating new task with fields: %s', task) task_entity = GCITask(**task) task_entity.put() task_quota = task_quota - 1 except DeadlineExceededError: # time to bail out break if len(tasks) == 0: # send out a message notifications.sendBulkCreationCompleted(bulk_data) bulk_data.delete() else: # there is still work to be done, do a non 500 response and requeue task_params = { 'bulk_create_key': bulk_data.key() } new_task = taskqueue.Task(params=task_params, url=BULK_CREATE_URL) # add to the gci queue new_task.add(queue_name='gci-update') # we're done here return http.HttpResponse('OK')
def bulkCreateTasks(request, *args, **kwargs): """Task that creates GCI Tasks from bulk data specified in the POST dict. The POST dict should have the following information present: bulk_create_key: the key of the bulk_create entity """ import settings # keep track of our own timelimit (20 seconds) timelimit = 20000 timekeeper = Timekeeper(timelimit) post_dict = request.POST bulk_create_key = post_dict.get('bulk_create_key') if not bulk_create_key: return error_handler.logErrorAndReturnOK( 'Not all POST data specified in: %s' % post_dict) bulk_data = bulk_create_model.GCIBulkCreateData.get(bulk_create_key) if not bulk_data: return error_handler.logErrorAndReturnOK( 'No valid data found for key: %s' % bulk_create_key) # note that we only query for the quota once org_admin = bulk_data.created_by task_quota = org_logic.getRemainingTaskQuota(org_admin.scope) tasks = bulk_data.tasks while len(tasks) > 0: try: # check if we have time timekeeper.ping() if settings.GCI_TASK_QUOTA_LIMIT_ENABLED and task_quota <= 0: return error_handler.logErrorAndReturnOK( 'Task quota reached for %s' %(org_admin.scope.name)) # remove the first task task_as_string = tasks.pop(0) loaded_task = simplejson.loads(task_as_string) task = {} for key, value in loaded_task.iteritems(): # If we don't do this python will complain about kwargs not being # strings when we try to save the new task. task[key.encode('UTF-8')] = value logging.info('Uncleaned task: %s' %task) # clean the data errors = _cleanTask(task, org_admin) if errors: logging.warning( 'Invalid task data uploaded, the following errors occurred: %s' %errors) bulk_data.errors.append(db.Text( 'The task in row %i contains the following errors.\n %s' \ %(bulk_data.tasksRemoved(), '\n'.join(errors)))) # at-most-once semantics for creating tasks bulk_data.put() if errors: # do the next task continue # set other properties task['link_id'] = 't%i' % (int(time.time()*100)) task['scope'] = org_admin.scope task['scope_path'] = org_admin.scope_path task['program'] = org_admin.program task['status'] = 'Unpublished' task['created_by'] = org_admin task['modified_by'] = org_admin # create the new task logging.info('Creating new task with fields: %s' %task) task_logic.updateOrCreateFromFields(task) task_quota = task_quota - 1 except DeadlineExceededError: # time to bail out pass if len(tasks) == 0: # send out a message notifications.sendBulkCreationCompleted(bulk_data) bulk_data.delete() else: # there is still work to be done, do a non 500 response and requeue task_params = { 'bulk_create_key': bulk_data.key().id_or_name() } new_task = taskqueue.Task(params=task_params, url=BULK_CREATE_URL) # add to the gci queue new_task.add(queue_name='gci-update') # we're done here return http.HttpResponse('OK')
def _continueShipmentSync(self, request, *args, **kwargs): """Continue syncing shipment data. POST Args: program_key: the key of the program which sync is being done for. shipment_info_id: id of the shipment info object that task is running for. column_indexes: column indexes for specific columns in JSON format. sheet_rows: spreadsheets CSV chunk data in JSON format. """ timekeeper = Timekeeper(20000) params = dicts.merge(request.POST, request.GET) redirect = RedirectHelper(None, None) if 'program_key' not in params: logging.error("missing program_key in params: '%s'" % params) return responses.terminateTask() if 'shipment_info_id' not in params: logging.error("missing shipment_info_id in params: '%s'" % params) return responses.terminateTask() self.setProgram(params['program_key']) self.setShipmentInfo(int(params['shipment_info_id'])) if 'sheet_rows' not in params: logging.error("missing sheet_rows data in params: '%s'" % params) return responses.terminateTask() if 'column_indexes' not in params: logging.error("missing column_indexes data in params: '%s'" % params) return responses.terminateTask() column_indexes = simplejson.loads(params['column_indexes']) sheet_rows = simplejson.loads(params['sheet_rows']) try: for remain, row in timekeeper.iterate(sheet_rows): if len(row) < len(column_indexes): row.extend((len(column_indexes) - len(row)) * ['']) data = self.getRowData(row, column_indexes) link_id = data['link_id'] q = GSoCProfile.all().filter('scope', self.__program) q.filter('link_id', link_id) profile = q.get() if not profile: logging.error("Profile link_id '%s' for program '%s' is not found" % (link_id, self.__program.name)) continue #continue to next row if not profile.is_student: logging.error("Profile link_id '%s' is not a student" % link_id) continue tracking = data['tracking'] date_shipped = data['date_shipped'] notes = data['notes'] full_address = " ".join([ data['address_1'], data['address_2'], data['city'], data.get('state', ''), data.get('zip', ''), data.get('zippostal_code', ''), data.get('country', '') ]) self.updateShipmentDataForStudent( profile, tracking, date_shipped, notes, full_address) except DeadlineExceededError: if remain: remaining_rows = sheet_rows[(-1 * remain):] params = { 'program_key': params.get('program_key'), 'sheet_rows': simplejson.dumps(remaining_rows), 'column_indexes': params.get('column_indexes'), 'shipment_info_id': params.get('shipment_info_id'), } taskqueue.add( url=redirect.urlOf('shipment_sync_task_continue'), params=params) return responses.terminateTask() self.finishSync() return responses.terminateTask()
def convert_proposals(request, *args, **kwargs): """Convert proposals for all organizations. POST Args: programkey: the key of the program whose proposals should be converted orgkey: the organization key to start at """ # Setup an artifical request deadline timelimit = 20000 timekeeper = Timekeeper(timelimit) # Copy for modification below params = dicts.merge(request.POST, request.GET) if "programkey" not in params: logging.error("missing programkey in params: '%s'" % params) return responses.terminateTask() program = program_logic.getFromKeyName(params["programkey"]) if not program: logging.error("invalid programkey in params: '%s'" % params) return responses.terminateTask() fields = { "scope": program, "status": "active", } # Continue from the next organization if "orgkey" in params: org = org_logic.getFromKeyName(params["orgkey"]) if not org: logging.error("invalid orgkey in params: '%s'" % params) return responses.terminateTask() fields["__key__ >="] = org # Add a task for each organization org = None try: orgs = org_logic.getQueryForFields(filter=fields) for remain, org in timekeeper.iterate(orgs): logging.info("convert %s %s", remain, org.key()) # Compound accept/reject taskflow taskqueue.add( url = "/tasks/accept_proposals/accept", params = { "orgkey": org.key().id_or_name(), "timelimit": timelimit, "nextpath": "/tasks/accept_proposals/reject" }) # Requeue this task for continuation except DeadlineExceededError: if org: params["orgkey"] = org.key().id_or_name() taskqueue.add(url=request.path, params=params) # Exit this task successfully return responses.terminateTask()
def convert_proposals(request, *args, **kwargs): """Convert proposals for all organizations. POST Args: programkey: the key of the program whose proposals should be converted orgkey: the organization key to start at """ # Setup an artifical request deadline timelimit = 20000 timekeeper = Timekeeper(timelimit) # Copy for modification below params = dicts.merge(request.POST, request.GET) if "programkey" not in params: logging.error("missing programkey in params: '%s'" % params) return responses.terminateTask() program = program_logic.getFromKeyName(params["programkey"]) if not program: logging.error("invalid programkey in params: '%s'" % params) return responses.terminateTask() fields = { "scope": program, "status": "active", } # Continue from the next organization if "orgkey" in params: org = org_logic.getFromKeyName(params["orgkey"]) if not org: logging.error("invalid orgkey in params: '%s'" % params) return responses.terminateTask() fields["__key__ >="] = org # Add a task for each organization org = None try: orgs = org_logic.getQueryForFields(filter=fields) for remain, org in timekeeper.iterate(orgs): logging.info("convert %s %s", remain, org.key()) # Compound accept/reject taskflow taskqueue.add(url="/tasks/accept_proposals/accept", params={ "orgkey": org.key().id_or_name(), "timelimit": timelimit, "nextpath": "/tasks/accept_proposals/reject" }) # Requeue this task for continuation except DeadlineExceededError: if org: params["orgkey"] = org.key().id_or_name() taskqueue.add(url=request.path, params=params) # Exit this task successfully return responses.terminateTask()