def collectTask(self,
request,
access_type,
page_name=None,
params=None,
**kwargs):
"""Task for collecting statistics that will be executed by Task Queue API.
"""
logic = params['logic']
link_id = kwargs['link_id']
scope_path = kwargs['scope_path']
statistic = self._getStatisticEntity(link_id, scope_path, logic)
if statistic is None:
raise task_responses.FatalTaskError
statistic, completed = logic.collectDispatcher(statistic)
self._updateCache(statistic, link_id, scope_path, logic)
if completed:
self._updateCacheList(statistic, scope_path, logic)
statistic.put()
else:
task_responses.startTask(url=request.path, queue_name='statistic')
return task_responses.terminateTask()
def startClearingTask(key_name): """Starts a new task which clears all ranking entities for the program specified by the given key_name. """ url = '/tasks/gci/ranking/clear/%s' % key_name queue_name = 'gci-update' responses.startTask(url, queue_name)
def startClearingTask(key_name):
"""Starts a new task which clears all ranking entities for the program
specified by the given key_name.
"""
url = '/tasks/gci/ranking/clear/%s' % key_name
queue_name = 'gci-update'
responses.startTask(url, queue_name)
def startUpdatingTask(task):
"""Starts a new task which updates ranking entity for the specified task.
"""
url = '/tasks/gci/ranking/update'
queue_name = 'gci-update'
context = {'task_keyname': task.key().id_or_name()}
responses.startTask(url, queue_name, context)
def startUpdatingTask(task):
"""Starts a new task which updates ranking entity for the specified task.
"""
url = '/tasks/gci/ranking/update'
queue_name = 'gci-update'
context = {
'task_keyname': task.key().id_or_name()
}
responses.startTask(url, queue_name, context)
def run(self, request, *args, **kwargs):
"""Processes all OrgAppSurveyRecords that are in the pre-accept or
pre-reject state for the given program.
Expects the following to be present in the POST dict:
program_key: Specifies the program key name for which to loop over all
the OrgAppSurveyRecords for.
Args:
request: Django Request object
"""
# set default batch size
batch_size = 10
# retrieve the program_key from POST data
post_dict = request.POST
program_key = post_dict.get('program_key')
if not program_key:
return error_handler.logErrorAndReturnOK(
'Not all required fields are present in POST dict %s' %
post_dict)
program_entity = self.program_logic.getFromKeyName(program_key)
if not program_entity:
return error_handler.logErrorAndReturnOK(
'No Program exists with keyname: %s' % program_key)
org_app = self.org_app_logic.getForProgram(program_entity)
record_logic = self.org_app_logic.getRecordLogic()
fields = {
'survey': org_app,
'status': ['pre-accepted', 'pre-rejected']
}
org_app_records = record_logic.getForFields(fields, limit=batch_size)
for org_app_record in org_app_records:
record_logic.processRecord(org_app_record)
if len(org_app_records) == batch_size:
# start a new task because we might not have exhausted all OrgAppRecords
context = post_dict.copy()
responses.startTask(self.path, context=context)
# return a 200 response that everything has been completed
return responses.terminateTask()
def setCollectTask(self,
request,
access_type,
page_name=None,
params=None,
**kwargs):
"""Starts a statistic collecting task.
"""
logic = params['logic']
link_id = kwargs['link_id']
scope_path = kwargs['scope_path']
fields = {
'url_name': params['url_name'],
'scope_path': scope_path,
'link_id': link_id
}
task_url = '/%(url_name)s/collect_task/%(scope_path)s/%(link_id)s' % fields
task = task_responses.startTask(task_url, queue_name='statistic')
if task is not None:
return self.json(request, {'response': 'success'})
else:
return self.json(request, {'response': 'failure'})
def run(self, request, *args, **kwargs):
"""Processes all OrgAppSurveyRecords that are in the pre-accept or
pre-reject state for the given program.
Expects the following to be present in the POST dict:
program_key: Specifies the program key name for which to loop over all
the OrgAppSurveyRecords for.
Args:
request: Django Request object
"""
# set default batch size
batch_size = 10
# retrieve the program_key from POST data
post_dict = request.POST
program_key = post_dict.get('program_key')
if not program_key:
return error_handler.logErrorAndReturnOK(
'Not all required fields are present in POST dict %s' % post_dict)
program_entity = self.program_logic.getFromKeyName(program_key)
if not program_entity:
return error_handler.logErrorAndReturnOK(
'No Program exists with keyname: %s' % program_key)
org_app = self.org_app_logic.getForProgram(program_entity)
record_logic = self.org_app_logic.getRecordLogic()
fields = {'survey': org_app,
'status': ['pre-accepted', 'pre-rejected']}
org_app_records = record_logic.getForFields(fields, limit=batch_size)
for org_app_record in org_app_records:
record_logic.processRecord(org_app_record)
if len(org_app_records) == batch_size:
# start a new task because we might not have exhausted all OrgAppRecords
context = post_dict.copy()
responses.startTask(self.path, context=context)
# return a 200 response that everything has been completed
return responses.terminateTask()
def manageModelsStatus(request, *args, **kwargs):
"""Sets status of the roles queried by the fields given by POST.
"""
post_dict = request.POST
new_status = post_dict.get('new_status')
if not new_status:
if not status_retriever or not callable(status_retriever):
return error_handler.logErrorAndReturnOK(
'No valid status can be set by the manageModelStatus.')
if not 'fields' in post_dict:
error_handler.logErrorAndReturnOK(
'No fields to filter on found for manageModelStatus.')
fields = pickle.loads(str(post_dict['fields']))
entities = entity_logic.getForFields(fields, limit=BATCH_SIZE)
for entity in entities:
if new_status:
status = new_status
else:
status = status_retriever(entity)
entity.status = status
db.put(entities)
if len(entities) == BATCH_SIZE:
# we might not have exhausted all the roles that can be updated,
# so start the same task again
context = post_dict.copy()
responses.startTask(request.path, context=context)
return responses.terminateTask()
# exhausted all the entities the task has been completed
return responses.terminateTask()
def manageModelsStatus(request, *args, **kwargs):
"""Sets status of the roles queried by the fields given by POST.
"""
post_dict = request.POST
new_status = post_dict.get('new_status')
if not new_status:
if not status_retriever or not callable(status_retriever):
return error_handler.logErrorAndReturnOK(
'No valid status can be set by the manageModelStatus.')
if not 'fields' in post_dict:
error_handler.logErrorAndReturnOK(
'No fields to filter on found for manageModelStatus.')
fields = pickle.loads(str(post_dict['fields']))
entities = entity_logic.getForFields(fields, limit=BATCH_SIZE)
for entity in entities:
if new_status:
status = new_status
else:
status = status_retriever(entity)
entity.status = status
db.put(entities)
if len(entities) == BATCH_SIZE:
# we might not have exhausted all the roles that can be updated,
# so start the same task again
context = post_dict.copy()
responses.startTask(request.path, context=context)
return responses.terminateTask()
# exhausted all the entities the task has been completed
return responses.terminateTask()
def collectTask(self, request, access_type, page_name=None,
params=None, **kwargs):
"""Task for collecting statistics that will be executed by Task Queue API.
"""
logic = params['logic']
link_id = kwargs['link_id']
scope_path = kwargs['scope_path']
statistic = self._getStatisticEntity(link_id, scope_path, logic)
if statistic is None:
raise task_responses.FatalTaskError
statistic, completed = logic.collectDispatcher(statistic)
self._updateCache(statistic, link_id, scope_path, logic)
if completed:
self._updateCacheList(statistic, scope_path, logic)
statistic.put()
else:
task_responses.startTask(url=request.path, queue_name='statistic')
return task_responses.terminateTask()
def _manageStatisticsPost(self,
request,
access_type,
page_name=None,
params=None,
**kwargs):
"""POST method for manage statistics request.
"""
logic = params['logic']
post_dict = request.POST
selections = simplejson.loads(post_dict.get('data', '[]'))
button_id = post_dict.get('button_id', '')
modified_entities = []
for selection in selections:
entity = logic.getFromKeyName(selection['key'])
if not entity:
logging.error('No notification found for %(key)s' % selection)
continue
# check if the current user can modify the statistic
if button_id == 'collect_stats':
fields = {
'url_name': params['url_name'],
'key_name': entity.key().name()
}
task_url = '/%(url_name)s/collect_task/%(key_name)s' % fields
task = task_responses.startTask(task_url,
queue_name='statistic')
if not task:
logging.error(
'There was an error. The task is not started')
elif button_id == 'clear_stats':
modified_entities.append(logic.clearStatistic(entity))
db.put(modified_entities)
return http.HttpResponseRedirect('')
def setCollectTask(self, request, access_type, page_name=None,
params=None, **kwargs):
"""Starts a statistic collecting task.
"""
logic = params['logic']
link_id = kwargs['link_id']
scope_path = kwargs['scope_path']
fields = {
'url_name': params['url_name'],
'scope_path': scope_path,
'link_id': link_id
}
task_url = '/%(url_name)s/collect_task/%(scope_path)s/%(link_id)s' % fields
task = task_responses.startTask(task_url, queue_name='statistic')
if task is not None:
return self.json(request, {'response': 'success'})
else:
return self.json(request, {'response': 'failure'})
def _manageStatisticsPost(self, request, access_type, page_name=None,
params=None, **kwargs):
"""POST method for manage statistics request.
"""
logic = params['logic']
post_dict = request.POST
selections = simplejson.loads(post_dict.get('data', '[]'))
button_id = post_dict.get('button_id', '')
modified_entities = []
for selection in selections:
entity = logic.getFromKeyName(selection['key'])
if not entity:
logging.error('No notification found for %(key)s' % selection)
continue
# check if the current user can modify the statistic
if button_id == 'collect_stats':
fields = {
'url_name': params['url_name'],
'key_name': entity.key().name()
}
task_url = '/%(url_name)s/collect_task/%(key_name)s' % fields
task = task_responses.startTask(task_url, queue_name='statistic')
if not task:
logging.error('There was an error. The task is not started')
elif button_id == 'clear_stats':
modified_entities.append(logic.clearStatistic(entity))
db.put(modified_entities)
return http.HttpResponseRedirect('')
def _processProgramFreezing(program_entity, mode):
"""Main function which dispatches entities to change status for into a set
of new tasks.
Args:
program_entity: GSoCProgram which needs to be frozen/unfrozen
mode: String containing 'freeze' or 'unfreeze' depending on the type
of action to perform.
"""
# process role models
new_context = {}
new_context['new_status'] = 'active' if mode == 'unfreeze' else 'inactive'
old_status = 'inactive' if mode == 'unfreeze' else 'active'
# process models which refer to program using scope field
new_context['fields'] = pickle.dumps({
'scope': program_entity.key(),
'status': old_status
})
for pattern in ROLE_PER_SCOPE_MODELS_URL_PATTERNS:
responses.startTask(_constructRequestURL(pattern), context=new_context)
# process models which refer to program using program field
new_context['fields'] = pickle.dumps({
'program': program_entity.key(),
'status': old_status
})
for pattern in ROLE_PER_PROGRAM_MODELS_URL_PATTERNS:
responses.startTask(_constructRequestURL(pattern), context=new_context)
# process organization model
new_context = {}
old_status = 'inactive' if mode == 'unfreeze' else ['active', 'new']
new_context['fields'] = pickle.dumps({
'scope': program_entity.key(),
'status': old_status
})
responses.startTask(_constructRequestURL(ORG_MODEL_URL_PATTERNS[0]),
context=new_context)
return
def _processProgramFreezing(program_entity, mode):
"""Main function which dispatches entities to change status for into a set
of new tasks.
Args:
program_entity: GSoCProgram which needs to be frozen/unfrozen
mode: String containing 'freeze' or 'unfreeze' depending on the type
of action to perform.
"""
# process role models
new_context = {}
new_context['new_status'] = 'active' if mode == 'unfreeze' else 'inactive'
old_status = 'inactive' if mode == 'unfreeze' else 'active'
# process models which refer to program using scope field
new_context['fields'] = pickle.dumps({
'scope': program_entity.key(),
'status': old_status
})
for pattern in ROLE_PER_SCOPE_MODELS_URL_PATTERNS:
responses.startTask(_constructRequestURL(pattern), context=new_context)
# process models which refer to program using program field
new_context['fields'] = pickle.dumps({
'program': program_entity.key(),
'status': old_status
})
for pattern in ROLE_PER_PROGRAM_MODELS_URL_PATTERNS:
responses.startTask(_constructRequestURL(pattern), context=new_context)
# process organization model
new_context = {}
old_status = 'inactive' if mode == 'unfreeze' else ['active', 'new']
new_context['fields'] = pickle.dumps({
'scope': program_entity.key(),
'status': old_status
})
responses.startTask(_constructRequestURL(ORG_MODEL_URL_PATTERNS[0]),
context=new_context)
return
def start(self, program_entity):
"""Starts the Task to bulk process OrgAppSurveyRecords.
"""
context = {'program_key': program_entity.key().id_or_name()}
return responses.startTask(self.path, context=context)
def start(self, program_entity):
"""Starts the Task to bulk process OrgAppSurveyRecords.
"""
context = {'program_key': program_entity.key().id_or_name()}
return responses.startTask(self.path, context=context)
def iterative_wrapped(request, *args, **kwargs):
"""Decorator wrapper method
Args:
request: Django HTTP Request object
request.POST usage:
fields: a JSON dict for the properties that the entities should have.
This updates values from the task_default entry.
start_key: the key of the next entity to fetch
Returns:
Standard HTTP Django response
"""
post_dict = request.POST
fields = task_default.get("fields", {})
if "fields" in post_dict:
fields.update(pickle.loads(str(post_dict["fields"])))
start_key = task_default.get("start_key", None)
if "start_key" in post_dict:
# get the key where to start this iteration
start_key = post_dict["start_key"]
if start_key:
start_key = db.Key(start_key)
# get the entities for this iteration
entities, next_start_key = logic.getBatchOfData(filter=fields, start_key=start_key)
# copy the post_dict so that the wrapped function can edit what it needs
context = post_dict.copy()
# when true, the iterative task will not be repeated when completed
do_not_repeat = False
try:
func(request, entities=entities, context=context, *args, **kwargs)
except task_responses.DoNotRepeatException as exception:
do_not_repeat = True
except task_responses.FatalTaskError as error:
logging.debug(post_dict)
logging.error(error)
return task_responses.terminateTask()
except Exception as exception:
logging.debug(post_dict)
logging.error(exception)
return task_responses.repeatTask()
if next_start_key:
# set the key to use for the next iteration
context.update({"start_key": next_start_key})
task_responses.startTask(url=request.path, context=context)
elif not do_not_repeat and repeat_in is not None:
# the task will be executed again after repeat_in seconds
if "start_key" in context:
del context["start_key"]
task_responses.startTask(url=request.path, countdown=repeat_in, context=context)
return task_responses.terminateTask()
try:
func(request, entities=entities, context=context, *args, **kwargs)
except task_responses.DoNotRepeatException, exception:
do_not_repeat = True
except task_responses.FatalTaskError, error:
logging.debug(post_dict)
logging.error(error)
return task_responses.terminateTask()
except Exception, exception:
logging.debug(post_dict)
logging.error(exception)
return task_responses.repeatTask()
if next_start_key:
# set the key to use for the next iteration
context.update({'start_key': next_start_key})
task_responses.startTask(url=request.path, context=context)
elif not do_not_repeat and repeat_in is not None:
# the task will be executed again after repeat_in seconds
if 'start_key' in context:
del context['start_key']
task_responses.startTask(url=request.path, countdown=repeat_in,
context=context)
return task_responses.terminateTask()
return iterative_wrapped
return wrapper
except task_responses.DoNotRepeatException, exception:
do_not_repeat = True
except task_responses.FatalTaskError, error:
logging.debug(post_dict)
logging.error(error)
return task_responses.terminateTask()
except Exception, exception:
logging.debug(post_dict)
logging.error(exception)
return task_responses.repeatTask()
if next_start_key:
# set the key to use for the next iteration
context.update({'start_key': next_start_key})
task_responses.startTask(url=request.path, context=context)
elif not do_not_repeat and repeat_in is not None:
# the task will be executed again after repeat_in seconds
if 'start_key' in context:
del context['start_key']
task_responses.startTask(url=request.path,
countdown=repeat_in,
context=context)
return task_responses.terminateTask()
return iterative_wrapped
return wrapper
def iterative_task(logic, **task_default):
"""Iterative wrapper method
Args:
logic: the Logic instance to get entities for
task_default: keyword arguments which can contain the following options:
fields: dictionary to filter the entities on
start_key: the default key where to start this iterative task
"""
def wrapper(func):
def iterative_wrapped(request, *args, **kwargs):
"""Decorator wrapper method
Args:
request: Django HTTP Request object
request.POST usage:
fields: a JSON dict for the properties that the entities should have.
This updates values from the task_default entry.
start_key: the key of the next entity to fetch
Returns:
Standard HTTP Django response
"""
post_dict = request.POST
fields = task_default.get('fields', {})
if 'fields' in post_dict:
fields.update(pickle.loads(str(post_dict['fields'])))
start_key = task_default.get('start_key', None)
if 'start_key' in post_dict:
# get the key where to start this iteration
start_key = post_dict['start_key']
if start_key:
start_key = db.Key(start_key)
# get the entities for this iteration
entities, next_start_key = logic.getBatchOfData(
filter=fields, start_key=start_key)
# copy the post_dict so that the wrapped function can edit what it needs
context = post_dict.copy()
try:
func(request,
entities=entities,
context=context,
*args,
**kwargs)
except task_responses.FatalTaskError, error:
logging.debug(post_dict)
logging.error(error)
return task_responses.terminateTask()
except Exception, exception:
logging.debug(post_dict)
logging.error(exception)
return task_responses.repeatTask()
if next_start_key:
# set the key to use for the next iteration
context.update({'start_key': next_start_key})
task_responses.startTask(url=request.path, context=context)
return task_responses.terminateTask()
def iterative_wrapped(request, *args, **kwargs):
"""Decorator wrapper method
Args:
request: Django HTTP Request object
request.POST usage:
fields: a JSON dict for the properties that the entities should have.
This updates values from the task_default entry.
start_key: the key of the next entity to fetch
Returns:
Standard HTTP Django response
"""
post_dict = request.POST
fields = task_default.get('fields', {})
if 'fields' in post_dict:
fields.update(pickle.loads(str(post_dict['fields'])))
start_key = task_default.get('start_key', None)
if 'start_key' in post_dict:
# get the key where to start this iteration
start_key = post_dict['start_key']
if start_key:
start_key = db.Key(start_key)
# get the entities for this iteration
entities, next_start_key = logic.getBatchOfData(filter=fields,
start_key=start_key)
# copy the post_dict so that the wrapped function can edit what it needs
context = post_dict.copy()
# when true, the iterative task will not be repeated when completed
do_not_repeat = False
try:
func(request, entities=entities, context=context, *args, **kwargs)
except task_responses.DoNotRepeatException as exception:
do_not_repeat = True
except task_responses.FatalTaskError as error:
logging.debug(post_dict)
logging.error(error)
return task_responses.terminateTask()
except Exception as exception:
logging.debug(post_dict)
logging.error(exception)
return task_responses.repeatTask()
if next_start_key:
# set the key to use for the next iteration
context.update({'start_key': next_start_key})
task_responses.startTask(url=request.path, context=context)
elif not do_not_repeat and repeat_in is not None:
# the task will be executed again after repeat_in seconds
if 'start_key' in context:
del context['start_key']
task_responses.startTask(url=request.path, countdown=repeat_in,
context=context)
return task_responses.terminateTask()