def testCalculateFinishedThroughPostWithCorrectXsrfToken(self):
"""Tests the result after calculating duplicate proposals is finished.
Through HTTP POST with correct XSRF token, when finished
(there is no org to calculate), all pb records with no duplicate proposals
are deleted and there is no taskqueue spawned.
"""
pd_fields = {
'program': self.program,
'student': self.student,
'orgs':[self.organization.key()],
'duplicates': [self.student_proposal.key()],
'is_duplicate': False
}
pd_logic.updateOrCreateFromFields(pd_fields)
pds = pd_logic.getForFields({'program': self.program})
self.assertEqual(len(pds), 1)
fields = {'scope': self.program,
'slots >': 0,
'status': 'active'}
# Get all organizations and save the cursor
q = gsoc_organization_logic.getQueryForFields(fields)
q.fetch(3)
org_cursor = q.cursor()
url = '/tasks/gsoc/proposal_duplicates/calculate'
postdata = {'program_key': self.program.key().name(),
'org_cursor': unicode(org_cursor)}
xsrf_token = self.getXsrfToken(url, data=postdata)
postdata.update(xsrf_token=xsrf_token)
response = self.client.post(url, postdata)
self.assertEqual(response.status_code, httplib.OK)
pds = pd_logic.getForFields({'program': self.program})
self.assertEqual(len(pds), 0)
task_url = '/tasks/gsoc/proposal_duplicates/calculate'
self.assertTasksInQueue(n=0, url=task_url)
def testCalculateThroughPostWithCorrectXsrfToken(self):
"""Tests that duplicate proposals can be calculated with correct XSRF token.
Through HTTP POST with correct XSRF token, the duplicate
proposals in a given program for a student can be calculated on a per
organization basis and a new taskqueue is spawned if there are more than
one organization after the org_cursor.
"""
pds = pd_logic.getForFields({'program': self.program})
self.assertEqual(len(pds), 0)
fields = {'scope': self.program, 'slots >': 0, 'status': 'active'}
# Get an organization and save the cursor
q = gsoc_organization_logic.getQueryForFields(fields)
orgs = q.fetch(1)
org_cursor = q.cursor()
url = '/tasks/gsoc/proposal_duplicates/calculate'
postdata = {
'program_key': self.program.key().name(),
'org_cursor': unicode(org_cursor)
}
xsrf_token = self.getXsrfToken(url, data=postdata)
postdata.update(xsrf_token=xsrf_token)
response = self.client.post(url, postdata)
self.assertEqual(response.status_code, httplib.OK)
pds = pd_logic.getForFields({'program': self.program})
self.assertEqual(len(pds), 1)
self.assertTrue(pds[0].is_duplicate)
task_url = '/tasks/gsoc/proposal_duplicates/calculate'
self.assertTasksInQueue(n=1, url=task_url)
def testCalculateThroughPostWithCorrectXsrfToken(self):
"""Tests that duplicate proposals can be calculated with correct XSRF token.
Through HTTP POST with correct XSRF token, the duplicate
proposals in a given program for a student can be calculated on a per
organization basis and a new taskqueue is spawned if there are more than
one organization after the org_cursor.
"""
pds = pd_logic.getForFields({'program': self.program})
self.assertEqual(len(pds), 0)
fields = {'scope': self.program,
'slots >': 0,
'status': 'active'}
# Get an organization and save the cursor
q = gsoc_organization_logic.getQueryForFields(fields)
orgs = q.fetch(1)
org_cursor = q.cursor()
url = '/tasks/gsoc/proposal_duplicates/calculate'
postdata = {'program_key': self.program.key().name(),
'org_cursor': unicode(org_cursor)}
xsrf_token = self.getXsrfToken(url, data=postdata)
postdata.update(xsrf_token=xsrf_token)
response = self.client.post(url, postdata)
self.assertEqual(response.status_code, httplib.OK)
pds = pd_logic.getForFields({'program': self.program})
self.assertEqual(len(pds), 1)
self.assertTrue(pds[0].is_duplicate)
task_url = '/tasks/gsoc/proposal_duplicates/calculate'
self.assertTasksInQueue(n=1, url=task_url)
def testCalculateFinishedThroughPostWithCorrectXsrfToken(self):
"""Tests the result after calculating duplicate proposals is finished.
Through HTTP POST with correct XSRF token, when finished
(there is no org to calculate), all pb records with no duplicate proposals
are deleted and there is no taskqueue spawned.
"""
pd_fields = {
'program': self.program,
'student': self.student,
'orgs': [self.organization.key()],
'duplicates': [self.student_proposal.key()],
'is_duplicate': False
}
pd_logic.updateOrCreateFromFields(pd_fields)
pds = pd_logic.getForFields({'program': self.program})
self.assertEqual(len(pds), 1)
fields = {'scope': self.program, 'slots >': 0, 'status': 'active'}
# Get all organizations and save the cursor
q = gsoc_organization_logic.getQueryForFields(fields)
q.fetch(3)
org_cursor = q.cursor()
url = '/tasks/gsoc/proposal_duplicates/calculate'
postdata = {
'program_key': self.program.key().name(),
'org_cursor': unicode(org_cursor)
}
xsrf_token = self.getXsrfToken(url, data=postdata)
postdata.update(xsrf_token=xsrf_token)
response = self.client.post(url, postdata)
self.assertEqual(response.status_code, httplib.OK)
pds = pd_logic.getForFields({'program': self.program})
self.assertEqual(len(pds), 0)
task_url = '/tasks/gsoc/proposal_duplicates/calculate'
self.assertTasksInQueue(n=0, url=task_url)
def testCalculateThroughPostWithoutCorrectXsrfToken(self):
"""Tests that calculating duplicate is forbidden without correct XSRF token.
Through HTTP POST without correct XSRF token, the task of
calculating the duplicate proposals in a given program for a student on
a per organization basis is forbidden.
"""
fields = {'scope': self.program,
'slots >': 0,
'status': 'active'}
# Get an organization and save the cursor
q = gsoc_organization_logic.getQueryForFields(fields)
orgs = q.fetch(1)
org_cursor = q.cursor()
url = '/tasks/gsoc/proposal_duplicates/calculate'
postdata = {'program_key': self.program.key().name(),
'org_cursor': unicode(org_cursor)}
response = self.client.post(url, postdata)
self.assertEqual(response.status_code, httplib.FORBIDDEN)
def testCalculateThroughPostWithoutCorrectXsrfToken(self):
"""Tests that calculating duplicate is forbidden without correct XSRF token.
Through HTTP POST without correct XSRF token, the task of
calculating the duplicate proposals in a given program for a student on
a per organization basis is forbidden.
"""
fields = {'scope': self.program, 'slots >': 0, 'status': 'active'}
# Get an organization and save the cursor
q = gsoc_organization_logic.getQueryForFields(fields)
orgs = q.fetch(1)
org_cursor = q.cursor()
url = '/tasks/gsoc/proposal_duplicates/calculate'
postdata = {
'program_key': self.program.key().name(),
'org_cursor': unicode(org_cursor)
}
response = self.client.post(url, postdata)
self.assertEqual(response.status_code, httplib.FORBIDDEN)
def calculate(request, *args, **kwargs):
"""Calculates the duplicate proposals in a given program for
a student on a per Organization basis.
Expects the following to be present in the POST dict:
program_key: Specifies the program key name for which to find the
duplicate proposals
org_cursor: Specifies the organization datastore cursor from which to
start the processing of finding the duplicate proposals
Args:
request: Django Request object
"""
from soc.modules.gsoc.logic.models.student_proposal import logic \
as sp_logic
post_dict = request.POST
program_key = post_dict.get('program_key')
if not program_key:
# invalid task data, log and return OK
return error_handler.logErrorAndReturnOK('Invalid program key: %s' %
post_dict)
program_entity = program_logic.getFromKeyName(program_key)
if not program_entity:
# invalid program specified, log and return OK
return error_handler.logErrorAndReturnOK(
'Invalid program specified: %s' % program_key)
fields = {'scope': program_entity, 'slots >': 0, 'status': 'active'}
# get the organization and update the cursor if possible
q = org_logic.getQueryForFields(fields)
# retrieve the org_cursor from POST data
org_cursor = post_dict.get('org_cursor')
if org_cursor:
org_cursor = str(org_cursor)
q.with_cursor(org_cursor)
result = q.fetch(1)
# update the cursor
org_cursor = q.cursor()
if result:
org_entity = result[0]
# get all the proposals likely to be accepted in the program
accepted_proposals = sp_logic.getProposalsToBeAcceptedForOrg(
org_entity)
for ap in accepted_proposals:
student_entity = ap.scope
proposal_duplicate = pd_logic.getForFields(
{'student': student_entity}, unique=True)
if proposal_duplicate and ap.key(
) not in proposal_duplicate.duplicates:
# non-counted (to-be) accepted proposal found
pd_fields = {
'duplicates': proposal_duplicate.duplicates + [ap.key()],
}
pd_fields['is_duplicate'] = len(pd_fields['duplicates']) >= 2
if org_entity.key() not in proposal_duplicate.orgs:
pd_fields['orgs'] = proposal_duplicate.orgs + [
org_entity.key()
]
proposal_duplicate = pd_logic.updateEntityProperties(
proposal_duplicate, pd_fields)
else:
pd_fields = {
'program': program_entity,
'student': student_entity,
'orgs': [org_entity.key()],
'duplicates': [ap.key()],
'is_duplicate': False
}
proposal_duplicate = pd_logic.updateOrCreateFromFields(
pd_fields)
# Adds a new task that performs duplicate calculation for
# the next organization.
task_params = {
'program_key': program_key,
'org_cursor': unicode(org_cursor)
}
task_url = '/tasks/gsoc/proposal_duplicates/calculate'
new_task = taskqueue.Task(params=task_params, url=task_url)
new_task.add()
else:
# There aren't any more organizations to process. So delete
# all the proposals for which there are not more than one
# proposal for duplicates property.
pd_logic.deleteAllForProgram(program_entity, non_dupes_only=True)
# update the proposal duplicate status and its timestamp
pds_entity = pds_logic.getOrCreateForProgram(program_entity)
new_fields = {
'status': 'idle',
'calculated_on': datetime.datetime.now()
}
pds_logic.updateEntityProperties(pds_entity, new_fields)
# return OK
return http.HttpResponse()
def convert_proposals(request, *args, **kwargs):
"""Convert proposals for all organizations.
POST Args:
programkey: the key of the program whose proposals should be converted
orgkey: the organization key to start at
"""
# Setup an artifical request deadline
timelimit = 20000
timekeeper = Timekeeper(timelimit)
# Copy for modification below
params = dicts.merge(request.POST, request.GET)
if "programkey" not in params:
logging.error("missing programkey in params: '%s'" % params)
return responses.terminateTask()
program = program_logic.getFromKeyName(params["programkey"])
if not program:
logging.error("invalid programkey in params: '%s'" % params)
return responses.terminateTask()
fields = {
"scope": program,
"status": "active",
}
# Continue from the next organization
if "orgkey" in params:
org = org_logic.getFromKeyName(params["orgkey"])
if not org:
logging.error("invalid orgkey in params: '%s'" % params)
return responses.terminateTask()
fields["__key__ >="] = org
# Add a task for each organization
org = None
try:
orgs = org_logic.getQueryForFields(filter=fields)
for remain, org in timekeeper.iterate(orgs):
logging.info("convert %s %s", remain, org.key())
# Compound accept/reject taskflow
taskqueue.add(
url = "/tasks/accept_proposals/accept",
params = {
"orgkey": org.key().id_or_name(),
"timelimit": timelimit,
"nextpath": "/tasks/accept_proposals/reject"
})
# Requeue this task for continuation
except DeadlineExceededError:
if org:
params["orgkey"] = org.key().id_or_name()
taskqueue.add(url=request.path, params=params)
# Exit this task successfully
return responses.terminateTask()
def convert_proposals(request, *args, **kwargs):
"""Convert proposals for all organizations.
POST Args:
programkey: the key of the program whose proposals should be converted
orgkey: the organization key to start at
"""
# Setup an artifical request deadline
timelimit = 20000
timekeeper = Timekeeper(timelimit)
# Copy for modification below
params = dicts.merge(request.POST, request.GET)
if "programkey" not in params:
logging.error("missing programkey in params: '%s'" % params)
return responses.terminateTask()
program = program_logic.getFromKeyName(params["programkey"])
if not program:
logging.error("invalid programkey in params: '%s'" % params)
return responses.terminateTask()
fields = {
"scope": program,
"status": "active",
}
# Continue from the next organization
if "orgkey" in params:
org = org_logic.getFromKeyName(params["orgkey"])
if not org:
logging.error("invalid orgkey in params: '%s'" % params)
return responses.terminateTask()
fields["__key__ >="] = org
# Add a task for each organization
org = None
try:
orgs = org_logic.getQueryForFields(filter=fields)
for remain, org in timekeeper.iterate(orgs):
logging.info("convert %s %s", remain, org.key())
# Compound accept/reject taskflow
taskqueue.add(url="/tasks/accept_proposals/accept",
params={
"orgkey": org.key().id_or_name(),
"timelimit": timelimit,
"nextpath": "/tasks/accept_proposals/reject"
})
# Requeue this task for continuation
except DeadlineExceededError:
if org:
params["orgkey"] = org.key().id_or_name()
taskqueue.add(url=request.path, params=params)
# Exit this task successfully
return responses.terminateTask()
def slots(self,
request,
acces_type,
page_name=None,
params=None,
**kwargs):
"""Returns a JSON object with all orgs allocation.
Args:
request: the standard Django HTTP request object
access_type : the name of the access type which should be checked
page_name: the page name displayed in templates as page and header title
params: a dict with params for this View, not used
"""
program_entity = program_logic.getFromKeyFieldsOr404(kwargs)
program_slots = program_entity.slots
filter = {
'scope': program_entity,
'status': ['new', 'active', 'inactive'],
}
query = org_logic.getQueryForFields(filter=filter)
organizations = org_logic.getAll(query)
if request.method == 'POST' and 'result' in request.POST:
result = request.POST['result']
submit = request.GET.get('submit')
load = request.GET.get('load')
stored = program_entity.slots_allocation
if load and stored:
result = stored
if submit:
program_entity.slots_allocation = result
program_entity.put()
orgs = {}
applications = {}
max = {}
for org in organizations:
orgs[org.link_id] = org
applications[org.link_id] = org.nr_applications
max[org.link_id] = org.slots_desired
max_slots_per_org = program_entity.max_slots
min_slots_per_org = program_entity.min_slots
algorithm = 2
allocator = allocations.Allocator(orgs.keys(), applications, max,
program_slots, max_slots_per_org,
min_slots_per_org, algorithm)
from_json = simplejson.loads(program_entity.slots_allocation)
locked_slots = dicts.groupDictBy(from_json, 'locked', 'slots')
result = allocator.allocate(locked_slots)
data = []
for link_id in orgs.keys():
data.append({
'link_id': link_id,
'slots': result.get(link_id, 0),
'locked': bool(locked_slots.get(link_id))
})
return self.json(request, data)
def slots(self, request, acces_type, page_name=None, params=None, **kwargs):
"""Returns a JSON object with all orgs allocation.
Args:
request: the standard Django HTTP request object
access_type : the name of the access type which should be checked
page_name: the page name displayed in templates as page and header title
params: a dict with params for this View, not used
"""
program_entity = program_logic.getFromKeyFieldsOr404(kwargs)
program_slots = program_entity.slots
filter = {
'scope': program_entity,
'status': ['new', 'active', 'inactive'],
}
query = org_logic.getQueryForFields(filter=filter)
organizations = org_logic.getAll(query)
if request.method == 'POST' and 'result' in request.POST:
result = request.POST['result']
submit = request.GET.get('submit')
load = request.GET.get('load')
stored = program_entity.slots_allocation
if load and stored:
result = stored
if submit:
program_entity.slots_allocation = result
program_entity.put()
orgs = {}
applications = {}
max = {}
for org in organizations:
orgs[org.link_id] = org
applications[org.link_id] = org.nr_applications
max[org.link_id] = org.slots_desired
max_slots_per_org = program_entity.max_slots
min_slots_per_org = program_entity.min_slots
algorithm = 2
allocator = allocations.Allocator(orgs.keys(), applications, max,
program_slots, max_slots_per_org,
min_slots_per_org, algorithm)
from_json = simplejson.loads(program_entity.slots_allocation)
locked_slots = dicts.groupDictBy(from_json, 'locked', 'slots')
result = allocator.allocate(locked_slots)
data = []
for link_id in orgs.keys():
data.append({
'link_id': link_id,
'slots': result.get(link_id,0),
'locked': bool(locked_slots.get(link_id))
})
return self.json(request, data)
def calculate(request, *args, **kwargs):
"""Calculates the duplicate proposals in a given program for
a student on a per Organization basis.
Expects the following to be present in the POST dict:
program_key: Specifies the program key name for which to find the
duplicate proposals
org_cursor: Specifies the organization datastore cursor from which to
start the processing of finding the duplicate proposals
Args:
request: Django Request object
"""
from soc.modules.gsoc.logic.models.student_proposal import logic \
as sp_logic
post_dict = request.POST
program_key = post_dict.get('program_key')
if not program_key:
# invalid task data, log and return OK
return error_handler.logErrorAndReturnOK(
'Invalid program key: %s' % post_dict)
program_entity = program_logic.getFromKeyName(program_key)
if not program_entity:
# invalid program specified, log and return OK
return error_handler.logErrorAndReturnOK(
'Invalid program specified: %s' % program_key)
fields = {'scope': program_entity,
'slots >': 0,
'status': 'active'}
# get the organization and update the cursor if possible
q = org_logic.getQueryForFields(fields)
# retrieve the org_cursor from POST data
org_cursor = post_dict.get('org_cursor')
if org_cursor:
org_cursor = str(org_cursor)
q.with_cursor(org_cursor)
result = q.fetch(1)
# update the cursor
org_cursor = q.cursor()
if result:
org_entity = result[0]
# get all the proposals likely to be accepted in the program
accepted_proposals = sp_logic.getProposalsToBeAcceptedForOrg(org_entity)
for ap in accepted_proposals:
student_entity = ap.scope
proposal_duplicate = pd_logic.getForFields({'student': student_entity},
unique=True)
if proposal_duplicate and ap.key() not in proposal_duplicate.duplicates:
# non-counted (to-be) accepted proposal found
pd_fields = {
'duplicates': proposal_duplicate.duplicates + [ap.key()],
}
pd_fields['is_duplicate'] = len(pd_fields['duplicates']) >= 2
if org_entity.key() not in proposal_duplicate.orgs:
pd_fields['orgs'] = proposal_duplicate.orgs + [org_entity.key()]
proposal_duplicate = pd_logic.updateEntityProperties(
proposal_duplicate, pd_fields)
else:
pd_fields = {
'program': program_entity,
'student': student_entity,
'orgs':[org_entity.key()],
'duplicates': [ap.key()],
'is_duplicate': False
}
proposal_duplicate = pd_logic.updateOrCreateFromFields(pd_fields)
# Adds a new task that performs duplicate calculation for
# the next organization.
task_params = {'program_key': program_key,
'org_cursor': unicode(org_cursor)}
task_url = '/tasks/gsoc/proposal_duplicates/calculate'
new_task = taskqueue.Task(params=task_params, url=task_url)
new_task.add()
else:
# There aren't any more organizations to process. So delete
# all the proposals for which there are not more than one
# proposal for duplicates property.
pd_logic.deleteAllForProgram(program_entity, non_dupes_only=True)
# update the proposal duplicate status and its timestamp
pds_entity = pds_logic.getOrCreateForProgram(program_entity)
new_fields = {'status': 'idle',
'calculated_on': datetime.datetime.now()}
pds_logic.updateEntityProperties(pds_entity, new_fields)
# return OK
return http.HttpResponse()
