def on_all_data_removed(cls, user_id):
"""Called back from DataRemovalCronHandler when batch deletion done."""
# Any user_id we are called for has had all wipeout batch jobs run.
# This means that all un-indexed items have been removed for that
# user. However, analysis map/reduce jobs may have been running in
# parallel with wipeout and re-added items indexed by user ID. Do one
# more pass of removing indexed items before we declare the user to be
# done.
cls._remove_per_course_indexed_items(user_id)
# Look through peer courses to see if the user is registered in any.
# If not, we can also remove any global settings items.
in_other_courses = False
for app_context in sites.get_course_index().get_all_courses():
with common_utils.Namespace(app_context.get_namespace_name()):
student = models.Student.get_by_user_id(user_id)
if student is not None:
in_other_courses = True
if not in_other_courses:
cls._remove_sitewide_indexed_items(user_id)
# When the foregoing deletion has completed w/o raising any
# exceptions, clean up the final two items that have any user-related
# PII. If this fails, the BatchRemovalState record will not be
# removed, and the next call to the cron handler will again see the
# user has no more batch items to remove, and call us again.
@db.transactional(xg=True)
def remove_deletion_state_records(user_id):
removal_models.ImmediateRemovalState.delete_by_user_id(user_id)
removal_models.BatchRemovalState.delete_by_user_id(user_id)
remove_deletion_state_records(user_id)
def _add_course_field(cls, message):
if cls._COURSE not in message:
namespace = namespace_manager.get_namespace()
app_context = (sites.get_course_index().
get_app_context_for_namespace(namespace))
course = courses.Course(None, app_context=app_context)
message[cls._COURSE] = cls._get_random_course_id(course)
def on_all_data_removed(cls, user_id):
"""Called back from DataRemovalCronHandler when batch deletion done."""
# Any user_id we are called for has had all wipeout batch jobs run.
# This means that all un-indexed items have been removed for that
# user. However, analysis map/reduce jobs may have been running in
# parallel with wipeout and re-added items indexed by user ID. Do one
# more pass of removing indexed items before we declare the user to be
# done.
cls._remove_per_course_indexed_items(user_id)
# Look through peer courses to see if the user is registered in any.
# If not, we can also remove any global settings items.
in_other_courses = False
for app_context in sites.get_course_index().get_all_courses():
with common_utils.Namespace(app_context.get_namespace_name()):
student = models.Student.get_by_user_id(user_id)
if student is not None:
in_other_courses = True
if not in_other_courses:
cls._remove_sitewide_indexed_items(user_id)
# When the foregoing deletion has completed w/o raising any
# exceptions, clean up the final two items that have any user-related
# PII. If this fails, the BatchRemovalState record will not be
# removed, and the next call to the cron handler will again see the
# user has no more batch items to remove, and call us again.
@db.transactional(xg=True)
def remove_deletion_state_records(user_id):
removal_models.ImmediateRemovalState.delete_by_user_id(user_id)
removal_models.BatchRemovalState.delete_by_user_id(user_id)
remove_deletion_state_records(user_id)
def _add_course_field(cls, message):
if cls._COURSE not in message:
namespace = namespace_manager.get_namespace()
app_context = (
sites.get_course_index().get_app_context_for_namespace(
namespace))
course = courses.Course(None, app_context=app_context)
message[cls._COURSE] = cls._get_random_course_id(course)
def post_add_first_course(self):
"""Adds first course to the deployment."""
uid = 'first'
course = sites.get_course_index().get_course_for_path('/%s' % uid)
if course:
self._redirect(course, '/dashboard')
return
course = self._make_new_course(uid, 'My First Course')
self._redirect(course, '/dashboard')
def post_add_first_course(self):
"""Adds first course to the deployment."""
uid = 'first'
course = sites.get_course_index().get_course_for_path('/%s' % uid)
if course:
self._redirect(course, '/dashboard')
return
course = self._make_new_course(uid, 'My First Course')
self._redirect(course, '/dashboard')
def map(student):
params = context.get().mapreduce_spec.mapper.params
ns = params['course_namespace']
app_context = sites.get_course_index().get_app_context_for_namespace(ns)
course = courses.Course(None, app_context=app_context)
if student_is_qualified(student, course):
yield(TOTAL_CERTIFICATES, 1)
if student.scores:
yield(TOTAL_ACTIVE_STUDENTS, 1)
yield(TOTAL_STUDENTS, 1)
def post_explore_sample(self):
"""Navigate to or import sample course."""
uid = 'sample'
course = sites.get_course_index().get_app_context_for_namespace(
'ns_%s' % uid)
if course:
self._redirect(course, '/dashboard')
return
course = self._copy_sample_course(uid)
self._redirect(course, '/dashboard')
def map(student):
params = context.get().mapreduce_spec.mapper.params
ns = params['course_namespace']
app_context = sites.get_course_index().get_app_context_for_namespace(ns)
course = courses.Course(None, app_context=app_context)
if student_is_qualified(student, course):
yield(TOTAL_CERTIFICATES, 1)
if student.scores:
yield(TOTAL_ACTIVE_STUDENTS, 1)
yield(TOTAL_STUDENTS, 1)
def can_handle_route_method_path_now(cls, route, method, path):
index = sites.get_course_index()
app_context = index.get_course_for_path('/')
if app_context:
config = get_config(app_context)
if config.get(WEBSERV_ENABLED):
slug = get_slug(config)
if slug == '/':
return False
return True
def post_explore_sample(self):
"""Navigate to or import sample course."""
uid = 'sample'
course = sites.get_course_index(
).get_app_context_for_namespace('ns_%s' % uid)
if course:
self._redirect(course, '/dashboard')
return
course = self._copy_sample_course(uid)
self._redirect(course, '/dashboard')
def post_explore_sample(self):
"""Navigate to or import sample course."""
course = None
for uid in ['sample', 'sample_%s' % os.environ[
'GCB_PRODUCT_VERSION'].replace('.', '_')]:
course = sites.get_course_index(
).get_app_context_for_namespace('ns_%s' % uid)
if not course:
course = self._copy_sample_course(uid)
break
assert course is not None
self._redirect(course, '/dashboard')
return course
def post_explore_sample(self):
"""Navigate to or import sample course."""
course = None
for uid in ['sample', 'sample_%s' % os.environ[
'GCB_PRODUCT_VERSION'].replace('.', '_')]:
course = sites.get_course_index(
).get_app_context_for_namespace('ns_%s' % uid)
if not course:
course = self._copy_sample_course(uid)
break
assert course is not None
self._redirect(course, '/dashboard')
return course
def get(self):
"""Handles GET requests."""
if course_explorer.GCB_ENABLE_COURSE_EXPLORER_PAGE.value:
self.redirect('/explorer')
return
index = sites.get_course_index()
if index.get_all_courses():
course = index.get_course_for_path('/')
if not course:
course = index.get_all_courses()[0]
self.redirect(ApplicationHandler.canonicalize_url_for(
course, '/course?use_last_location=true'))
else:
self.redirect('/admin?action=welcome')
def get(self):
"""Handles GET requests."""
if course_explorer.GCB_ENABLE_COURSE_EXPLORER_PAGE.value:
self.redirect('/explorer')
return
index = sites.get_course_index()
if index.get_all_courses():
course = index.get_course_for_path('/')
if not course:
course = index.get_all_courses()[0]
self.redirect(
ApplicationHandler.canonicalize_url_for(
course, '/course?use_last_location=true'))
else:
self.redirect('/admin/welcome')
def get(self):
index = sites.get_course_index()
if index.get_all_courses():
course = index.get_course_for_path('/')
if not course:
course = index.get_all_courses()[0]
config = get_config(course)
if config.get(WEBSERV_ENABLED):
location = get_slug(config)
if location != '/':
location += '/'
else:
location = COURSE_HOME_PAGE
self.redirect(utils.ApplicationHandler.canonicalize_url_for(
course, location), normalize=False)
else:
self.redirect(ADMIN_HOME_PAGE)
def _get_current_context():
namespace = namespace_manager.get_namespace()
course_index = sites.get_course_index()
app_context = course_index.get_app_context_for_namespace(namespace)
return app_context
def reduce(user_id, values):
# Convenience for collections: Pre-load Student and Course objects.
student = None
try:
student = models.Student.get_by_user_id(user_id)
# pylint: disable=broad-except
except Exception:
common_utils.log_exception_origin()
if not student:
logging.warning(
'Student for student aggregation with user ID %s '
'was not loaded. Ignoring records for this student.', user_id)
return
params = context.get().mapreduce_spec.mapper.params
ns = params['course_namespace']
app_context = sites.get_course_index().get_app_context_for_namespace(
ns)
course = courses.Course(None, app_context=app_context)
# Bundle items together into lists by collection name
event_items = collections.defaultdict(list)
for value in values:
component_name, payload = value.split(':', 1)
event_items[component_name].append(transforms.loads(payload))
# Build up per-Student aggregate by calling each component. Note that
# we call each component whether or not its mapper produced any
# output.
aggregate = {}
for component in StudentAggregateComponentRegistry.get_components():
component_name = component.get_name()
static_value = params.get(component_name)
value = {}
try:
value = component.produce_aggregate(
course, student, static_value,
event_items.get(component_name, []))
if not value:
continue
# pylint: disable=broad-except
except Exception, ex:
common_utils.log_exception_origin()
logging.critical(
'Student aggregation reduce function '
'component handler %s failed: %s', component_name, str(ex))
continue
schema_name = params['schema_names'][component_name]
if schema_name not in value:
logging.critical(
'Student aggregation reduce handler %s produced '
'a dict which does not contain the top-level '
'name (%s) from its registered schema.', component_name,
schema_name)
continue
variances = transforms.validate_object_matches_json_schema(
value[schema_name], params['schemas'][component_name])
if variances:
logging.critical(
'Student aggregation reduce handler %s produced '
'a value which does not match its schema: %s',
component_name, ' '.join(variances))
continue
aggregate.update(value)
def reduce(user_id, values):
# Convenience for collections: Pre-load Student and Course objects.
student = None
try:
student = models.Student.get_by_user_id(user_id)
# pylint: disable=broad-except
except Exception:
common_utils.log_exception_origin()
if not student:
logging.warning(
'Student for student aggregation with user ID %s '
'was not loaded. Ignoring records for this student.', user_id)
return
params = context.get().mapreduce_spec.mapper.params
ns = params['course_namespace']
app_context = sites.get_course_index().get_app_context_for_namespace(ns)
course = courses.Course(None, app_context=app_context)
# Bundle items together into lists by collection name
event_items = collections.defaultdict(list)
for value in values:
component_name, payload = value.split(':', 1)
event_items[component_name].append(transforms.loads(payload))
# Build up per-Student aggregate by calling each component. Note that
# we call each component whether or not its mapper produced any
# output.
aggregate = {}
for component in StudentAggregateComponentRegistry.get_components():
component_name = component.get_name()
static_value = params.get(component_name)
value = {}
try:
value = component.produce_aggregate(
course, student, static_value,
event_items.get(component_name, []))
if not value:
continue
# pylint: disable=broad-except
except Exception, ex:
common_utils.log_exception_origin()
logging.critical('Student aggregation reduce function '
'component handler %s failed: %s',
component_name, str(ex))
continue
schema_name = params['schema_names'][component_name]
if schema_name not in value:
logging.critical(
'Student aggregation reduce handler %s produced '
'a dict which does not contain the top-level '
'name (%s) from its registered schema.',
component_name, schema_name)
continue
variances = transforms.validate_object_matches_json_schema(
value[schema_name], params['schemas'][component_name])
if variances:
logging.critical(
'Student aggregation reduce handler %s produced '
'a value which does not match its schema: %s',
component_name, ' '.join(variances))
continue
aggregate.update(value)
def _get_current_context():
namespace = namespace_manager.get_namespace()
course_index = sites.get_course_index()
app_context = course_index.get_app_context_for_namespace(namespace)
return app_context