def submit(self):
if self.is_active():
return -1
with Namespace(self._namespace):
if not self._pre_transaction_setup():
return -1
return db.run_in_transaction_options(self.xg_on,
self.non_transactional_submit)
def run(self, job_name, sequence_num, kwargs, namespace, complete_fn):
with Namespace(namespace):
db.run_in_transaction(
DurableJobEntity._start_job, job_name, sequence_num,
MapReduceJob.build_output(self.root_pipeline_id, []))
output = yield mapreduce_pipeline.MapreducePipeline(**kwargs)
yield StoreMapReduceResults(job_name, sequence_num, namespace, output,
complete_fn, kwargs)
def update_course_details(cls,
namespace,
slug=None,
filesystem_path=None,
should_list=None,
visible=None,
can_register=None,
registration=None,
allowlist=None,
browsable=None,
title=None,
blurb=None,
category=None,
closed=None,
tags=None,
course_admin_email=None,
featured=None,
last_announcement=None):
with Namespace(cls.TARGET_NAMESPACE):
c = CourseList.get_by_key_name(namespace)
if not c:
return
memcache_key = cls._memcache_key(c.slug)
memcache.delete(memcache_key, namespace=cls.TARGET_NAMESPACE)
if slug is not None:
c.slug = slug
if filesystem_path is not None:
c.filesystem_path = filesystem_path
if should_list is not None:
c.should_list = should_list
if visible is not None:
c.visible = visible
if can_register is not None:
c.can_register = can_register
if registration is not None:
c.registration = registration
if allowlist is not None:
c.allowlist = allowlist
if browsable is not None:
c.browsable = browsable
if title is not None:
c.title = title
if blurb is not None:
c.blurb = blurb
if category is not None:
c.category = category
if closed is not None:
c.closed = closed
if tags is not None:
c.tags = tags
if course_admin_email is not None:
c.course_admin_email = course_admin_email
if featured is not None:
c.featured = featured
if last_announcement is not None:
c.last_announcement = last_announcement
c.put()
def delete_local_chapter(cls, key, errors):
with Namespace(cls.TARGET_NAMESPACE):
c = LocalChapter.get_by_id(key)
if not c:
errors.append('Unable to find Local Chapter %s.' % key)
return
c.key.delete()
models.MemcacheManager.delete(cls.list_memcache_key,
namespace=cls.TARGET_NAMESPACE)
def delete_queue_settings(cls, queue_id, errors):
with Namespace(cls.TARGET_NAMESPACE):
c = QueueSettings.get_by_id(queue_id)
if not c:
errors.append('Unable to find QueueSettings %s.' % queue)
return
c.key.delete()
models.MemcacheManager.delete(cls.list_memcache_key,
namespace=cls.TARGET_NAMESPACE)
def run(self, job_name, kwargs, namespace):
time_started = time.time()
with Namespace(namespace):
db.run_in_transaction(
DurableJobEntity._start_job, job_name,
MapReduceJob.build_output(self.root_pipeline_id, []))
output = yield mapreduce_pipeline.MapreducePipeline(**kwargs)
yield StoreMapReduceResults(job_name, time_started, namespace, output)
def ui_access_wrapper(self, *args, **kwargs):
content_is_static = (
self.request.path.startswith('/mapreduce/ui/') and
(self.request.path.endswith('.css') or
self.request.path.endswith('.js')))
xsrf_token = self.request.get('xsrf_token')
user_is_course_admin = utils.XsrfTokenManager.is_xsrf_token_valid(
xsrf_token, XSRF_ACTION_NAME)
ui_enabled = GCB_ENABLE_MAPREDUCE_DETAIL_ACCESS.value
if ui_enabled and (content_is_static or
user_is_course_admin or
users.is_current_user_admin()):
namespace = self.request.get('namespace')
with Namespace(namespace):
self.real_dispatch(*args, **kwargs)
# Some places in the pipeline UI are good about passing the
# URL's search string along to RPC calls back to Ajax RPCs,
# which automatically picks up our extra namespace and xsrf
# tokens. However, some do not, and so we patch it
# here, rather than trying to keep up-to-date with the library.
params = {}
if namespace:
params['namespace'] = namespace
if xsrf_token:
params['xsrf_token'] = xsrf_token
extra_url_params = urllib.urlencode(params)
if self.request.path == '/mapreduce/ui/pipeline/status.js':
self.response.body = self.response.body.replace(
'rpc/tree?',
'rpc/tree\' + window.location.search + \'&')
elif self.request.path == '/mapreduce/ui/pipeline/rpc/tree':
self.response.body = self.response.body.replace(
'/mapreduce/worker/detail?',
'/mapreduce/ui/detail?' + extra_url_params + '&')
elif self.request.path == '/mapreduce/ui/detail':
self.response.body = self.response.body.replace(
'src="status.js"',
'src="status.js?%s"' % extra_url_params)
elif self.request.path == '/mapreduce/ui/status.js':
replacement = (
'\'namespace\': \'%s\', '
'\'xsrf_token\': \'%s\', '
'\'mapreduce_id\':' % (
namespace if namespace else '',
xsrf_token if xsrf_token else ''))
self.response.charset = 'utf8'
self.response.text = self.response.body.replace(
'\'mapreduce_id\':', replacement)
else:
self.response.out.write('Forbidden')
self.response.set_status(403)
def test_dashboard_access_method(self):
with Namespace(self.course_with_access.app_context.namespace):
self.assertFalse(
DashboardHandler.current_user_has_access(
self.course_with_access.app_context))
with Namespace(self.course_without_access.app_context.namespace):
self.assertFalse(
DashboardHandler.current_user_has_access(
self.course_without_access.app_context))
actions.login(self.USER_EMAIL, is_admin=False)
with Namespace(self.course_with_access.app_context.namespace):
self.assertTrue(
DashboardHandler.current_user_has_access(
self.course_with_access.app_context))
with Namespace(self.course_without_access.app_context.namespace):
self.assertFalse(
DashboardHandler.current_user_has_access(
self.course_without_access.app_context))
actions.logout()
def delete_category(cls, key, errors):
with Namespace(cls.TARGET_NAMESPACE):
c = CourseCategory.get_by_key_name(key)
if not c:
errors.append('Unable to find category %s.' % key)
return
c.delete()
models.MemcacheManager.delete(cls._memcache_key(key),
namespace=cls.TARGET_NAMESPACE)
def get_queue_settings_by_email_settings(cls, email_settings_key):
with Namespace(cls.TARGET_NAMESPACE):
email_settings = EmailSettings.get_by_id(email_settings_key)
query = QueueSettings.query().filter(
QueueSettings.email_settings == email_settings.key)
c = query.get()
if c:
return QueueSettingsDTO(c)
else:
return None
def force_fail_job(self, sequence_num, data, duration):
data = transforms.dumps(data)
with Namespace(self._namespace):
db.run_in_transaction(
# pylint: disable=protected-access
jobs.DurableJobEntity._fail_job,
self._job_name,
sequence_num,
data,
duration)
def add_new_local_chapter(cls, key, name, city, state, errors):
with Namespace(cls.TARGET_NAMESPACE):
if cls.get_local_chapter_by_key(key):
errors.append('Unable to add new LocalChapter. Entry with the '
'same key \'%s\' already exists.' % key)
return
chapter = LocalChapter(id=key, name=name, city=city, state=state)
chapter_key = chapter.put()
models.MemcacheManager.delete(cls.list_memcache_key,
namespace=cls.TARGET_NAMESPACE)
return chapter_key
def add_new_course(cls, namespace, path, title, errors):
with Namespace(cls.TARGET_NAMESPACE):
if cls.get_course_for_namespace(namespace):
errors.append('Unable to add new entry. Entry with the '
'same name \'%s\' already exists.' % namespace)
return
cli = CourseList(key_name=namespace)
cli.slug = path
cli.title = title
cli.put()
memcache_key = cls._memcache_key(path)
memcache.delete(memcache_key, namespace=cls.TARGET_NAMESPACE)
def add_new_queue_settings(cls, queue_id, email_settings, errors):
with Namespace(cls.TARGET_NAMESPACE):
if cls.get_queue_settings_by_key(queue_id):
errors.append(
'Unable to add new QueueSettings. Entry with the '
'same key \'%s\' already exists.' % queue)
return
settings = QueueSettings(id=queue_id,
email_settings=email_settings)
settings = settings.put()
models.MemcacheManager.delete(cls.list_memcache_key,
namespace=cls.TARGET_NAMESPACE)
def update_queue_settings(cls, queue_id, email_settings, errors):
with Namespace(cls.TARGET_NAMESPACE):
c = QueueSettings.get_by_id(queue_id)
if not c:
errors.append('Unable to find QueueSettings %s.' % queue)
return
if email_settings is not None:
c.email_settings = email_settings
c.put()
models.MemcacheManager.delete(cls.list_memcache_key,
namespace=cls.TARGET_NAMESPACE)
def setUp(self):
super(DashboardAccessTestCase, self).setUp()
actions.login(self.ADMIN_EMAIL, is_admin=True)
context = actions.simple_add_course(self.ACCESS_COURSE_NAME,
self.ADMIN_EMAIL,
'Course with access')
self.course_with_access = courses.Course(None, context)
with Namespace(self.course_with_access.app_context.namespace):
role_dto = models.RoleDTO(
None, {
'name': self.ROLE,
'users': [self.USER_EMAIL],
'permissions': {
dashboard.custom_module.name: [self.PERMISSION]
}
})
models.RoleDAO.save(role_dto)
context = actions.simple_add_course(self.NO_ACCESS_COURSE_NAME,
self.ADMIN_EMAIL,
'Course with no access')
self.course_without_access = courses.Course(None, context)
def test_content(self):
return self.render_page({
'main_content': 'test',
'page_title': 'test'
})
# save properties
self.old_menu_group = DashboardHandler.root_menu_group
# pylint: disable=W0212
self.old_get_acitons = DashboardHandler._custom_get_actions
# pylint: enable=W0212
# put a dummy method in
menu_group = menus.MenuGroup('test', 'Test Dashboard')
DashboardHandler.root_menu_group = menu_group
DashboardHandler.default_action = self.ACTION
DashboardHandler.add_nav_mapping(self.ACTION, self.ACTION)
DashboardHandler.add_sub_nav_mapping(self.ACTION,
self.ACTION,
self.ACTION,
action=self.ACTION,
contents=test_content)
DashboardHandler.map_action_to_permission('get_%s' % self.ACTION,
self.PERMISSION)
actions.logout()
def update_local_chapter(cls, key, name, city, state, errors):
with Namespace(cls.TARGET_NAMESPACE):
c = LocalChapter.get_by_id(key)
if not c:
errors.append('Unable to find local chapter %s.' % key)
return
if name is not None:
c.name = name
c.city = city
c.state = state
c.put()
models.MemcacheManager.delete(cls.list_memcache_key,
namespace=cls.TARGET_NAMESPACE)
def add_new_category(cls, key, description, visible, errors):
with Namespace(cls.TARGET_NAMESPACE):
if cls.get_category_by_key(key):
errors.append('Unable to add new category. Entry with the '
'same key \'%s\' already exists.' % key)
return
cli = CourseCategory(key_name=key)
cli.description = description
cli.visible = visible
cli.put()
models.MemcacheManager.set(cls._memcache_key(key),
cli,
namespace=cls.TARGET_NAMESPACE)
def test_progress_pre_assessment_submitted_and_fully_correct(self):
self.unit.pre_assessment = self.assessment_one.unit_id
self.unit.post_assessment = self.assessment_two.unit_id
self.course.save()
self._get_unit_page(self.unit)
response = self._post_assessment(self.assessment_one.unit_id, '100')
# Reload student; assessment scores are cached in student.
with Namespace(NAMESPACE):
self.student = models.Student.get_by_user(users.get_current_user())
# 100% progress because pre-assessment was 100% correct.
with Namespace(NAMESPACE):
self.assertEquals(1.000, self.tracker.get_unit_percent_complete(
self.student)[self.unit.unit_id])
# Still 100% after navigating onto a lesson
response = response.follow()
with Namespace(NAMESPACE):
self.assertEquals(1.000, self.tracker.get_unit_percent_complete(
self.student)[self.unit.unit_id])
def update_category(cls, key, description, visible, errors):
with Namespace(cls.TARGET_NAMESPACE):
c = CourseCategory.get_by_key_name(key)
if not c:
errors.append('Unable to find category %s.' % key)
return
if description is not None:
c.description = description
c.visible = visible
c.put()
models.MemcacheManager.set(cls._memcache_key(key),
c,
namespace=cls.TARGET_NAMESPACE)
def run(self, job_name, sequence_num, time_started, namespace, output,
complete_fn, kwargs):
results = []
# TODO(mgainer): Notice errors earlier in pipeline, and mark job
# as failed in that case as well.
try:
iterator = input_readers.GoogleCloudStorageInputReader(output, 0)
for file_reader in iterator:
for item in file_reader:
# Map/reduce puts reducer output into blobstore files as a
# string obtained via "str(result)". Use AST as a safe
# alternative to eval() to get the Python object back.
results.append(ast.literal_eval(item))
if complete_fn:
util.for_name(complete_fn)(kwargs, results)
time_completed = time.time()
with Namespace(namespace):
db.run_in_transaction(
DurableJobEntity._complete_job, job_name, sequence_num,
MapReduceJob.build_output(self.root_pipeline_id, results),
long(time_completed - time_started))
# Don't know what exceptions are currently, or will be in future,
# thrown from Map/Reduce or Pipeline libraries; these are under
# active development.
#
# pylint: disable=broad-except
except Exception, ex:
logging.critical('Failed running map/reduce job %s: %s', job_name,
str(ex))
common_utils.log_exception_origin()
time_completed = time.time()
with Namespace(namespace):
db.run_in_transaction(
DurableJobEntity._fail_job, job_name, sequence_num,
MapReduceJob.build_output(self.root_pipeline_id, results,
str(ex)),
long(time_completed - time_started))
def test_progress_no_pre_assessment(self):
# Zero progress when no unit actions taken.
with Namespace(NAMESPACE):
self.assertEquals(0.0, self.tracker.get_unit_percent_complete(
self.student)[self.unit.unit_id])
# Progress is counted when navigating _on to_ lesson.
response = self._get_unit_page(self.unit)
with Namespace(NAMESPACE):
self.assertEquals(0.333, self.tracker.get_unit_percent_complete(
self.student)[self.unit.unit_id])
# Navigate to next lesson. Verify rounding to 3 decimal places.
response = self._click_next_button(response)
with Namespace(NAMESPACE):
self.assertEquals(0.667, self.tracker.get_unit_percent_complete(
self.student)[self.unit.unit_id])
# Navigate to next lesson.
response = self._click_next_button(response)
with Namespace(NAMESPACE):
self.assertEquals(1.000, self.tracker.get_unit_percent_complete(
self.student)[self.unit.unit_id])
def get_category_by_key(cls, key):
with Namespace(cls.TARGET_NAMESPACE):
c = models.MemcacheManager.get(cls._memcache_key(key),
namespace=cls.TARGET_NAMESPACE)
if c:
return CourseCategoryDTO(c)
c = CourseCategory.get_by_key_name(key)
if c:
models.MemcacheManager.set(cls._memcache_key(key),
c,
namespace=cls.TARGET_NAMESPACE)
return CourseCategoryDTO(c)
else:
return None
def get_layout(cls):
with Namespace(cls.TARGET_NAMESPACE):
c = MemcacheManager.get(cls._memcache_key(),
namespace=cls.TARGET_NAMESPACE)
if c:
return ExplorerLayoutDTO(c)
c = ExplorerLayout.get_by_key_name(cls.KEY)
if c:
MemcacheManager.set(cls._memcache_key(),
c,
namespace=cls.TARGET_NAMESPACE)
return ExplorerLayoutDTO(c)
else:
return ExplorerLayoutDTO(None)
def get_email_settings_list(cls):
with Namespace(cls.TARGET_NAMESPACE):
email_settings_list = models.MemcacheManager.get(
cls.list_memcache_key, namespace=cls.TARGET_NAMESPACE)
if email_settings_list is not None:
return email_settings_list
email_settings = []
for settings in EmailSettings.query().iter():
email_settings.append(EmailSettingsDTO(settings))
models.MemcacheManager.set(cls.list_memcache_key,
email_settings,
namespace=cls.TARGET_NAMESPACE)
return email_settings
def update_email_settings(cls, unique_name, provider, from_email, api_key,
errors):
with Namespace(cls.TARGET_NAMESPACE):
c = EmailSettings.get_by_id(unique_name)
if not c:
errors.append('Unable to find EmailSettings %s.' % queue)
return
if c is not None:
c.provider = provider
c.from_email = from_email
c.api_key = api_key
c.put()
models.MemcacheManager.delete(cls.list_memcache_key,
namespace=cls.TARGET_NAMESPACE)
def get_queue_settings_list(cls):
with Namespace(cls.TARGET_NAMESPACE):
queue_settings_list = models.MemcacheManager.get(
cls.list_memcache_key, namespace=cls.TARGET_NAMESPACE)
if queue_settings_list is not None:
return queue_settings_list
queue_settings = []
for settings in QueueSettings.query().iter():
queue_settings.append(QueueSettingsDTO(settings))
models.MemcacheManager.set(cls.list_memcache_key,
queue_settings,
namespace=cls.TARGET_NAMESPACE)
return queue_settings
def setUp(self):
super(ProgressPercent, self).setUp()
context = actions.simple_add_course(COURSE_NAME, ADMIN_EMAIL,
COURSE_TITLE)
self.course = courses.Course(None, context)
self.unit = self.course.add_unit()
self.unit.title = 'No Lessons'
self.unit.availability = courses.AVAILABILITY_AVAILABLE
self.lesson_one = self.course.add_lesson(self.unit)
self.lesson_one.title = 'Lesson One'
self.lesson_one.objectives = 'body of lesson'
self.lesson_one.availability = courses.AVAILABILITY_AVAILABLE
self.lesson_two = self.course.add_lesson(self.unit)
self.lesson_two.title = 'Lesson Two'
self.lesson_two.objectives = 'body of lesson'
self.lesson_two.availability = courses.AVAILABILITY_AVAILABLE
self.lesson_three = self.course.add_lesson(self.unit)
self.lesson_three.title = 'Lesson Three'
self.lesson_three.objectives = 'body of lesson'
self.lesson_three.availability = courses.AVAILABILITY_AVAILABLE
self.assessment_one = self.course.add_assessment()
self.assessment_one.title = 'Assessment One'
self.assessment_one.html_content = 'assessment one content'
self.assessment_one.availability = courses.AVAILABILITY_AVAILABLE
self.assessment_two = self.course.add_assessment()
self.assessment_two.title = 'Assessment Two'
self.assessment_two.html_content = 'assessment two content'
self.assessment_two.availability = courses.AVAILABILITY_AVAILABLE
self.course.save()
actions.login(STUDENT_EMAIL)
actions.register(self, STUDENT_EMAIL, COURSE_NAME)
self.overridden_environment = actions.OverriddenEnvironment(
{'course': {
analytics.CAN_RECORD_STUDENT_EVENTS: 'true'
}})
self.overridden_environment.__enter__()
self.tracker = self.course.get_progress_tracker()
with Namespace(NAMESPACE):
self.student = models.Student.get_by_user(users.get_current_user())
def add_new_email_settings(cls, unique_name, provider, from_email, api_key,
errors):
with Namespace(cls.TARGET_NAMESPACE):
if cls.get_email_settings_by_key(unique_name):
errors.append(
'Unable to add new EmailSettings. Entry with the '
'same key \'%s\' already exists.' % queue)
return
settings = EmailSettings(id=unique_name,
provider=provider,
from_email=from_email,
api_key=api_key)
settings = settings.put()
models.MemcacheManager.delete(cls.list_memcache_key,
namespace=cls.TARGET_NAMESPACE)
return unique_name
def mark_cleaned_up(self):
job = self.load()
# If the job has already finished, then the cleanup is a
# no-op; we are just reclaiming transient state. However, if
# our DurableJobEntity still thinks the job is running and it
# is actually not, then mark the status message to indicate
# the cleanup.
if job and not job.has_finished:
duration = int(
(datetime.datetime.utcnow() - job.updated_on).total_seconds())
with Namespace(self._namespace):
return db.run_in_transaction(
self._mark_job_canceled, job,
'Job has not completed; assumed to have failed after %s' %
str(datetime.timedelta(seconds=duration)), duration)
return job
