def convertGCIBulkCreateData(bulk_create_data_key):
"""Converts the specified bulk create data by changing values of its profile
related properties to the new NDB based profile entities.
Args:
org_key: BulkCreateData key.
"""
bulk_create_data = GCIBulkCreateData.get(bulk_create_data_key)
bulk_create_data.created_by = _convertReferenceProperty(
GCIBulkCreateData.created_by, bulk_create_data)
bulk_create_data.put()
def convertGCIBulkCreateData(bulk_create_data_key):
"""Converts the specified bulk create data by changing values of its profile
related properties to the new NDB based profile entities.
Args:
org_key: BulkCreateData key.
"""
bulk_create_data = GCIBulkCreateData.get(bulk_create_data_key)
bulk_create_data.created_by = _convertReferenceProperty(
GCIBulkCreateData.created_by, bulk_create_data)
bulk_create_data.put()
def spawnBulkCreateTasks(data, org, org_admin):
"""Spawns a task to bulk post the given data.
The data given to this method should be in CSV format with the following
columns:
title, description, time_to_complete, mentors, difficulty, task_type,
arbit_tag
Fields where multiple values are allowed should be comma separated as well.
These fields are task_type, arbit_tag and mentors. Rows of data which can not
be properly resolved to form a valid Task will be safely ignored.
Args:
data: string with data in csv format
org: GCIOrganization for which the tasks are created
org_admin: GCIProfile of the org admin uploading these tasks
"""
data = StringIO.StringIO(data.encode('UTF-8'))
tasks = csv.DictReader(data, fieldnames=DATA_HEADERS, restval="")
task_list = []
for task in tasks:
# pop any extra columns
task.pop(None,None)
task_list.append(db.Text(simplejson.dumps(task)))
bulk_data = GCIBulkCreateData(
tasks=task_list, org=org, created_by=org_admin,
total_tasks=len(task_list))
bulk_data.put()
task_params = {
'bulk_create_key': bulk_data.key()
}
logging.info('Enqueued bulk_create with: %s' % task_params)
new_task = taskqueue.Task(params=task_params,
url=BULK_CREATE_URL)
new_task.add()
def spawnBulkCreateTasks(data, org, org_admin):
"""Spawns a task to bulk post the given data.
The data given to this method should be in CSV format with the following
columns:
title, description, time_to_complete, mentors, difficulty, task_type,
arbit_tag
Fields where multiple values are allowed should be comma separated as well.
These fields are task_type, arbit_tag and mentors. Rows of data which can not
be properly resolved to form a valid Task will be safely ignored.
Args:
data: string with data in csv format
org: GCIOrganization for which the tasks are created
org_admin: GCIProfile of the org admin uploading these tasks
"""
data = StringIO.StringIO(data.encode('UTF-8'))
tasks = csv.DictReader(data, fieldnames=DATA_HEADERS, restval="")
task_list = []
for task in tasks:
# pop any extra columns
task.pop(None, None)
task_list.append(db.Text(json.dumps(task)))
bulk_data = GCIBulkCreateData(tasks=task_list,
org=org,
created_by=org_admin.key.to_old_key(),
total_tasks=len(task_list))
bulk_data.put()
task_params = {'bulk_create_key': bulk_data.key()}
logging.info('Enqueued bulk_create with: %s', task_params)
new_task = taskqueue.Task(params=task_params, url=BULK_CREATE_URL)
new_task.add()
def bulkCreateTasks(self, request, *args, **kwargs):
"""Task that creates GCI Tasks from bulk data specified in the POST dict.
The POST dict should have the following information present:
bulk_create_key: the key of the bulk_create entity
"""
import settings
# keep track of our own timelimit (20 seconds)
timelimit = 20000
timekeeper = Timekeeper(timelimit)
post_dict = request.POST
bulk_create_key = post_dict.get('bulk_create_key')
if not bulk_create_key:
return error_handler.logErrorAndReturnOK(
'Not all POST data specified in: %s' % post_dict)
bulk_data = GCIBulkCreateData.get(bulk_create_key)
if not bulk_data:
return error_handler.logErrorAndReturnOK(
'No valid data found for key: %s' % bulk_create_key)
# note that we only query for the quota once
org_admin = ndb.Key.from_old_key(
GCIBulkCreateData.created_by.get_value_for_datastore(
bulk_data)).get()
org = bulk_data.org
task_quota = getRemainingTaskQuota(org)
# TODO(ljvderijk): Add transactions
tasks = bulk_data.tasks
while len(tasks) > 0:
try:
# check if we have time
timekeeper.ping()
if settings.GCI_TASK_QUOTA_LIMIT_ENABLED and task_quota <= 0:
return error_handler.logErrorAndReturnOK(
'Task quota reached for %s' % (org.name))
# remove the first task
task_as_string = tasks.pop(0)
loaded_task = json.loads(task_as_string)
task = {}
for key, value in loaded_task.iteritems():
# If we don't do this python will complain about kwargs not being
# strings when we try to save the new task.
task[key.encode('UTF-8')] = value
logging.info('Uncleaned task: %s', task)
# clean the data
errors = self._cleanTask(task, org)
if errors:
logging.warning(
'Invalid task data uploaded, the following errors occurred: %s',
errors)
bulk_data.errors.append(db.Text(
'The task in row %i contains the following errors.\n %s' \
%(bulk_data.tasksRemoved(), '\n'.join(errors))))
# at-most-once semantics for creating tasks
bulk_data.put()
if errors:
# do the next task
continue
# set other properties
task['org'] = org
# TODO(daniel): access program in more efficient way
task['program'] = org_admin.program.to_old_key()
task['status'] = task_model.UNPUBLISHED
task['created_by'] = org_admin.to_old_key()
task['modified_by'] = org_admin.to_old_key()
# TODO(ljv): Remove difficulty level completely if needed.
# Difficulty is hardcoded to easy since GCI2012 has no difficulty.
task['difficulty_level'] = DifficultyLevel.EASY
subscribers_entities = task['mentor_entities'] + [org_admin]
task['subscribers'] = list(
set([
ent.key() for ent in subscribers_entities
if ent.automatic_task_subscription
]))
# create the new task
logging.info('Creating new task with fields: %s', task)
task_entity = GCITask(**task)
task_entity.put()
task_quota = task_quota - 1
except DeadlineExceededError:
# time to bail out
break
if len(tasks) == 0:
# send out a message
notifications.sendBulkCreationCompleted(bulk_data)
bulk_data.delete()
else:
# there is still work to be done, do a non 500 response and requeue
task_params = {'bulk_create_key': bulk_data.key()}
new_task = taskqueue.Task(params=task_params, url=BULK_CREATE_URL)
# add to the gci queue
new_task.add(queue_name='gci-update')
# we're done here
return http.HttpResponse('OK')
def bulkCreateTasks(self, request, *args, **kwargs):
"""Task that creates GCI Tasks from bulk data specified in the POST dict.
The POST dict should have the following information present:
bulk_create_key: the key of the bulk_create entity
"""
import settings
# keep track of our own timelimit (20 seconds)
timelimit = 20000
timekeeper = Timekeeper(timelimit)
post_dict = request.POST
bulk_create_key = post_dict.get('bulk_create_key')
if not bulk_create_key:
return error_handler.logErrorAndReturnOK(
'Not all POST data specified in: %s' % post_dict)
bulk_data = GCIBulkCreateData.get(bulk_create_key)
if not bulk_data:
return error_handler.logErrorAndReturnOK(
'No valid data found for key: %s' % bulk_create_key)
# note that we only query for the quota once
org_admin = ndb.Key.from_old_key(
GCIBulkCreateData.created_by.get_value_for_datastore(bulk_data)).get()
org = bulk_data.org
task_quota = getRemainingTaskQuota(org)
# TODO(ljvderijk): Add transactions
tasks = bulk_data.tasks
while len(tasks) > 0:
try:
# check if we have time
timekeeper.ping()
if settings.GCI_TASK_QUOTA_LIMIT_ENABLED and task_quota <= 0:
return error_handler.logErrorAndReturnOK(
'Task quota reached for %s' %(org.name))
# remove the first task
task_as_string = tasks.pop(0)
loaded_task = json.loads(task_as_string)
task = {}
for key, value in loaded_task.iteritems():
# If we don't do this python will complain about kwargs not being
# strings when we try to save the new task.
task[key.encode('UTF-8')] = value
logging.info('Uncleaned task: %s', task)
# clean the data
errors = self._cleanTask(task, org)
if errors:
logging.warning(
'Invalid task data uploaded, the following errors occurred: %s',
errors)
bulk_data.errors.append(db.Text(
'The task in row %i contains the following errors.\n %s' \
%(bulk_data.tasksRemoved(), '\n'.join(errors))))
# at-most-once semantics for creating tasks
bulk_data.put()
if errors:
# do the next task
continue
# set other properties
task['org'] = org
# TODO(daniel): access program in more efficient way
task['program'] = org_admin.program.to_old_key()
task['status'] = task_model.UNPUBLISHED
task['created_by'] = org_admin.to_old_key()
task['modified_by'] = org_admin.to_old_key()
# TODO(ljv): Remove difficulty level completely if needed.
# Difficulty is hardcoded to easy since GCI2012 has no difficulty.
task['difficulty_level'] = DifficultyLevel.EASY
subscribers_entities = task['mentor_entities'] + [org_admin]
task['subscribers'] = list(set([ent.key() for ent in
subscribers_entities if ent.automatic_task_subscription]))
# create the new task
logging.info('Creating new task with fields: %s', task)
task_entity = GCITask(**task)
task_entity.put()
task_quota = task_quota - 1
except DeadlineExceededError:
# time to bail out
break
if len(tasks) == 0:
# send out a message
notifications.sendBulkCreationCompleted(bulk_data)
bulk_data.delete()
else:
# there is still work to be done, do a non 500 response and requeue
task_params = {
'bulk_create_key': bulk_data.key()
}
new_task = taskqueue.Task(params=task_params, url=BULK_CREATE_URL)
# add to the gci queue
new_task.add(queue_name='gci-update')
# we're done here
return http.HttpResponse('OK')
