def submit(self): if self.is_active(): return -1 with Namespace(self._namespace): if not self._pre_transaction_setup(): return -1 return db.run_in_transaction_options(self.xg_on, self.non_transactional_submit)
def run(self, job_name, sequence_num, kwargs, namespace, complete_fn): with Namespace(namespace): db.run_in_transaction( DurableJobEntity._start_job, job_name, sequence_num, MapReduceJob.build_output(self.root_pipeline_id, [])) output = yield mapreduce_pipeline.MapreducePipeline(**kwargs) yield StoreMapReduceResults(job_name, sequence_num, namespace, output, complete_fn, kwargs)
def update_course_details(cls, namespace, slug=None, filesystem_path=None, should_list=None, visible=None, can_register=None, registration=None, allowlist=None, browsable=None, title=None, blurb=None, category=None, closed=None, tags=None, course_admin_email=None, featured=None, last_announcement=None): with Namespace(cls.TARGET_NAMESPACE): c = CourseList.get_by_key_name(namespace) if not c: return memcache_key = cls._memcache_key(c.slug) memcache.delete(memcache_key, namespace=cls.TARGET_NAMESPACE) if slug is not None: c.slug = slug if filesystem_path is not None: c.filesystem_path = filesystem_path if should_list is not None: c.should_list = should_list if visible is not None: c.visible = visible if can_register is not None: c.can_register = can_register if registration is not None: c.registration = registration if allowlist is not None: c.allowlist = allowlist if browsable is not None: c.browsable = browsable if title is not None: c.title = title if blurb is not None: c.blurb = blurb if category is not None: c.category = category if closed is not None: c.closed = closed if tags is not None: c.tags = tags if course_admin_email is not None: c.course_admin_email = course_admin_email if featured is not None: c.featured = featured if last_announcement is not None: c.last_announcement = last_announcement c.put()
def delete_local_chapter(cls, key, errors): with Namespace(cls.TARGET_NAMESPACE): c = LocalChapter.get_by_id(key) if not c: errors.append('Unable to find Local Chapter %s.' % key) return c.key.delete() models.MemcacheManager.delete(cls.list_memcache_key, namespace=cls.TARGET_NAMESPACE)
def delete_queue_settings(cls, queue_id, errors): with Namespace(cls.TARGET_NAMESPACE): c = QueueSettings.get_by_id(queue_id) if not c: errors.append('Unable to find QueueSettings %s.' % queue) return c.key.delete() models.MemcacheManager.delete(cls.list_memcache_key, namespace=cls.TARGET_NAMESPACE)
def run(self, job_name, kwargs, namespace): time_started = time.time() with Namespace(namespace): db.run_in_transaction( DurableJobEntity._start_job, job_name, MapReduceJob.build_output(self.root_pipeline_id, [])) output = yield mapreduce_pipeline.MapreducePipeline(**kwargs) yield StoreMapReduceResults(job_name, time_started, namespace, output)
def ui_access_wrapper(self, *args, **kwargs): content_is_static = ( self.request.path.startswith('/mapreduce/ui/') and (self.request.path.endswith('.css') or self.request.path.endswith('.js'))) xsrf_token = self.request.get('xsrf_token') user_is_course_admin = utils.XsrfTokenManager.is_xsrf_token_valid( xsrf_token, XSRF_ACTION_NAME) ui_enabled = GCB_ENABLE_MAPREDUCE_DETAIL_ACCESS.value if ui_enabled and (content_is_static or user_is_course_admin or users.is_current_user_admin()): namespace = self.request.get('namespace') with Namespace(namespace): self.real_dispatch(*args, **kwargs) # Some places in the pipeline UI are good about passing the # URL's search string along to RPC calls back to Ajax RPCs, # which automatically picks up our extra namespace and xsrf # tokens. However, some do not, and so we patch it # here, rather than trying to keep up-to-date with the library. params = {} if namespace: params['namespace'] = namespace if xsrf_token: params['xsrf_token'] = xsrf_token extra_url_params = urllib.urlencode(params) if self.request.path == '/mapreduce/ui/pipeline/status.js': self.response.body = self.response.body.replace( 'rpc/tree?', 'rpc/tree\' + window.location.search + \'&') elif self.request.path == '/mapreduce/ui/pipeline/rpc/tree': self.response.body = self.response.body.replace( '/mapreduce/worker/detail?', '/mapreduce/ui/detail?' + extra_url_params + '&') elif self.request.path == '/mapreduce/ui/detail': self.response.body = self.response.body.replace( 'src="status.js"', 'src="status.js?%s"' % extra_url_params) elif self.request.path == '/mapreduce/ui/status.js': replacement = ( '\'namespace\': \'%s\', ' '\'xsrf_token\': \'%s\', ' '\'mapreduce_id\':' % ( namespace if namespace else '', xsrf_token if xsrf_token else '')) self.response.charset = 'utf8' self.response.text = self.response.body.replace( '\'mapreduce_id\':', replacement) else: self.response.out.write('Forbidden') self.response.set_status(403)
def test_dashboard_access_method(self): with Namespace(self.course_with_access.app_context.namespace): self.assertFalse( DashboardHandler.current_user_has_access( self.course_with_access.app_context)) with Namespace(self.course_without_access.app_context.namespace): self.assertFalse( DashboardHandler.current_user_has_access( self.course_without_access.app_context)) actions.login(self.USER_EMAIL, is_admin=False) with Namespace(self.course_with_access.app_context.namespace): self.assertTrue( DashboardHandler.current_user_has_access( self.course_with_access.app_context)) with Namespace(self.course_without_access.app_context.namespace): self.assertFalse( DashboardHandler.current_user_has_access( self.course_without_access.app_context)) actions.logout()
def delete_category(cls, key, errors): with Namespace(cls.TARGET_NAMESPACE): c = CourseCategory.get_by_key_name(key) if not c: errors.append('Unable to find category %s.' % key) return c.delete() models.MemcacheManager.delete(cls._memcache_key(key), namespace=cls.TARGET_NAMESPACE)
def get_queue_settings_by_email_settings(cls, email_settings_key): with Namespace(cls.TARGET_NAMESPACE): email_settings = EmailSettings.get_by_id(email_settings_key) query = QueueSettings.query().filter( QueueSettings.email_settings == email_settings.key) c = query.get() if c: return QueueSettingsDTO(c) else: return None
def force_fail_job(self, sequence_num, data, duration): data = transforms.dumps(data) with Namespace(self._namespace): db.run_in_transaction( # pylint: disable=protected-access jobs.DurableJobEntity._fail_job, self._job_name, sequence_num, data, duration)
def add_new_local_chapter(cls, key, name, city, state, errors): with Namespace(cls.TARGET_NAMESPACE): if cls.get_local_chapter_by_key(key): errors.append('Unable to add new LocalChapter. Entry with the ' 'same key \'%s\' already exists.' % key) return chapter = LocalChapter(id=key, name=name, city=city, state=state) chapter_key = chapter.put() models.MemcacheManager.delete(cls.list_memcache_key, namespace=cls.TARGET_NAMESPACE) return chapter_key
def add_new_course(cls, namespace, path, title, errors): with Namespace(cls.TARGET_NAMESPACE): if cls.get_course_for_namespace(namespace): errors.append('Unable to add new entry. Entry with the ' 'same name \'%s\' already exists.' % namespace) return cli = CourseList(key_name=namespace) cli.slug = path cli.title = title cli.put() memcache_key = cls._memcache_key(path) memcache.delete(memcache_key, namespace=cls.TARGET_NAMESPACE)
def add_new_queue_settings(cls, queue_id, email_settings, errors): with Namespace(cls.TARGET_NAMESPACE): if cls.get_queue_settings_by_key(queue_id): errors.append( 'Unable to add new QueueSettings. Entry with the ' 'same key \'%s\' already exists.' % queue) return settings = QueueSettings(id=queue_id, email_settings=email_settings) settings = settings.put() models.MemcacheManager.delete(cls.list_memcache_key, namespace=cls.TARGET_NAMESPACE)
def update_queue_settings(cls, queue_id, email_settings, errors): with Namespace(cls.TARGET_NAMESPACE): c = QueueSettings.get_by_id(queue_id) if not c: errors.append('Unable to find QueueSettings %s.' % queue) return if email_settings is not None: c.email_settings = email_settings c.put() models.MemcacheManager.delete(cls.list_memcache_key, namespace=cls.TARGET_NAMESPACE)
def setUp(self): super(DashboardAccessTestCase, self).setUp() actions.login(self.ADMIN_EMAIL, is_admin=True) context = actions.simple_add_course(self.ACCESS_COURSE_NAME, self.ADMIN_EMAIL, 'Course with access') self.course_with_access = courses.Course(None, context) with Namespace(self.course_with_access.app_context.namespace): role_dto = models.RoleDTO( None, { 'name': self.ROLE, 'users': [self.USER_EMAIL], 'permissions': { dashboard.custom_module.name: [self.PERMISSION] } }) models.RoleDAO.save(role_dto) context = actions.simple_add_course(self.NO_ACCESS_COURSE_NAME, self.ADMIN_EMAIL, 'Course with no access') self.course_without_access = courses.Course(None, context) def test_content(self): return self.render_page({ 'main_content': 'test', 'page_title': 'test' }) # save properties self.old_menu_group = DashboardHandler.root_menu_group # pylint: disable=W0212 self.old_get_acitons = DashboardHandler._custom_get_actions # pylint: enable=W0212 # put a dummy method in menu_group = menus.MenuGroup('test', 'Test Dashboard') DashboardHandler.root_menu_group = menu_group DashboardHandler.default_action = self.ACTION DashboardHandler.add_nav_mapping(self.ACTION, self.ACTION) DashboardHandler.add_sub_nav_mapping(self.ACTION, self.ACTION, self.ACTION, action=self.ACTION, contents=test_content) DashboardHandler.map_action_to_permission('get_%s' % self.ACTION, self.PERMISSION) actions.logout()
def update_local_chapter(cls, key, name, city, state, errors): with Namespace(cls.TARGET_NAMESPACE): c = LocalChapter.get_by_id(key) if not c: errors.append('Unable to find local chapter %s.' % key) return if name is not None: c.name = name c.city = city c.state = state c.put() models.MemcacheManager.delete(cls.list_memcache_key, namespace=cls.TARGET_NAMESPACE)
def add_new_category(cls, key, description, visible, errors): with Namespace(cls.TARGET_NAMESPACE): if cls.get_category_by_key(key): errors.append('Unable to add new category. Entry with the ' 'same key \'%s\' already exists.' % key) return cli = CourseCategory(key_name=key) cli.description = description cli.visible = visible cli.put() models.MemcacheManager.set(cls._memcache_key(key), cli, namespace=cls.TARGET_NAMESPACE)
def test_progress_pre_assessment_submitted_and_fully_correct(self): self.unit.pre_assessment = self.assessment_one.unit_id self.unit.post_assessment = self.assessment_two.unit_id self.course.save() self._get_unit_page(self.unit) response = self._post_assessment(self.assessment_one.unit_id, '100') # Reload student; assessment scores are cached in student. with Namespace(NAMESPACE): self.student = models.Student.get_by_user(users.get_current_user()) # 100% progress because pre-assessment was 100% correct. with Namespace(NAMESPACE): self.assertEquals(1.000, self.tracker.get_unit_percent_complete( self.student)[self.unit.unit_id]) # Still 100% after navigating onto a lesson response = response.follow() with Namespace(NAMESPACE): self.assertEquals(1.000, self.tracker.get_unit_percent_complete( self.student)[self.unit.unit_id])
def update_category(cls, key, description, visible, errors): with Namespace(cls.TARGET_NAMESPACE): c = CourseCategory.get_by_key_name(key) if not c: errors.append('Unable to find category %s.' % key) return if description is not None: c.description = description c.visible = visible c.put() models.MemcacheManager.set(cls._memcache_key(key), c, namespace=cls.TARGET_NAMESPACE)
def run(self, job_name, sequence_num, time_started, namespace, output, complete_fn, kwargs): results = [] # TODO(mgainer): Notice errors earlier in pipeline, and mark job # as failed in that case as well. try: iterator = input_readers.GoogleCloudStorageInputReader(output, 0) for file_reader in iterator: for item in file_reader: # Map/reduce puts reducer output into blobstore files as a # string obtained via "str(result)". Use AST as a safe # alternative to eval() to get the Python object back. results.append(ast.literal_eval(item)) if complete_fn: util.for_name(complete_fn)(kwargs, results) time_completed = time.time() with Namespace(namespace): db.run_in_transaction( DurableJobEntity._complete_job, job_name, sequence_num, MapReduceJob.build_output(self.root_pipeline_id, results), long(time_completed - time_started)) # Don't know what exceptions are currently, or will be in future, # thrown from Map/Reduce or Pipeline libraries; these are under # active development. # # pylint: disable=broad-except except Exception, ex: logging.critical('Failed running map/reduce job %s: %s', job_name, str(ex)) common_utils.log_exception_origin() time_completed = time.time() with Namespace(namespace): db.run_in_transaction( DurableJobEntity._fail_job, job_name, sequence_num, MapReduceJob.build_output(self.root_pipeline_id, results, str(ex)), long(time_completed - time_started))
def test_progress_no_pre_assessment(self): # Zero progress when no unit actions taken. with Namespace(NAMESPACE): self.assertEquals(0.0, self.tracker.get_unit_percent_complete( self.student)[self.unit.unit_id]) # Progress is counted when navigating _on to_ lesson. response = self._get_unit_page(self.unit) with Namespace(NAMESPACE): self.assertEquals(0.333, self.tracker.get_unit_percent_complete( self.student)[self.unit.unit_id]) # Navigate to next lesson. Verify rounding to 3 decimal places. response = self._click_next_button(response) with Namespace(NAMESPACE): self.assertEquals(0.667, self.tracker.get_unit_percent_complete( self.student)[self.unit.unit_id]) # Navigate to next lesson. response = self._click_next_button(response) with Namespace(NAMESPACE): self.assertEquals(1.000, self.tracker.get_unit_percent_complete( self.student)[self.unit.unit_id])
def get_category_by_key(cls, key): with Namespace(cls.TARGET_NAMESPACE): c = models.MemcacheManager.get(cls._memcache_key(key), namespace=cls.TARGET_NAMESPACE) if c: return CourseCategoryDTO(c) c = CourseCategory.get_by_key_name(key) if c: models.MemcacheManager.set(cls._memcache_key(key), c, namespace=cls.TARGET_NAMESPACE) return CourseCategoryDTO(c) else: return None
def get_layout(cls): with Namespace(cls.TARGET_NAMESPACE): c = MemcacheManager.get(cls._memcache_key(), namespace=cls.TARGET_NAMESPACE) if c: return ExplorerLayoutDTO(c) c = ExplorerLayout.get_by_key_name(cls.KEY) if c: MemcacheManager.set(cls._memcache_key(), c, namespace=cls.TARGET_NAMESPACE) return ExplorerLayoutDTO(c) else: return ExplorerLayoutDTO(None)
def get_email_settings_list(cls): with Namespace(cls.TARGET_NAMESPACE): email_settings_list = models.MemcacheManager.get( cls.list_memcache_key, namespace=cls.TARGET_NAMESPACE) if email_settings_list is not None: return email_settings_list email_settings = [] for settings in EmailSettings.query().iter(): email_settings.append(EmailSettingsDTO(settings)) models.MemcacheManager.set(cls.list_memcache_key, email_settings, namespace=cls.TARGET_NAMESPACE) return email_settings
def update_email_settings(cls, unique_name, provider, from_email, api_key, errors): with Namespace(cls.TARGET_NAMESPACE): c = EmailSettings.get_by_id(unique_name) if not c: errors.append('Unable to find EmailSettings %s.' % queue) return if c is not None: c.provider = provider c.from_email = from_email c.api_key = api_key c.put() models.MemcacheManager.delete(cls.list_memcache_key, namespace=cls.TARGET_NAMESPACE)
def get_queue_settings_list(cls): with Namespace(cls.TARGET_NAMESPACE): queue_settings_list = models.MemcacheManager.get( cls.list_memcache_key, namespace=cls.TARGET_NAMESPACE) if queue_settings_list is not None: return queue_settings_list queue_settings = [] for settings in QueueSettings.query().iter(): queue_settings.append(QueueSettingsDTO(settings)) models.MemcacheManager.set(cls.list_memcache_key, queue_settings, namespace=cls.TARGET_NAMESPACE) return queue_settings
def setUp(self): super(ProgressPercent, self).setUp() context = actions.simple_add_course(COURSE_NAME, ADMIN_EMAIL, COURSE_TITLE) self.course = courses.Course(None, context) self.unit = self.course.add_unit() self.unit.title = 'No Lessons' self.unit.availability = courses.AVAILABILITY_AVAILABLE self.lesson_one = self.course.add_lesson(self.unit) self.lesson_one.title = 'Lesson One' self.lesson_one.objectives = 'body of lesson' self.lesson_one.availability = courses.AVAILABILITY_AVAILABLE self.lesson_two = self.course.add_lesson(self.unit) self.lesson_two.title = 'Lesson Two' self.lesson_two.objectives = 'body of lesson' self.lesson_two.availability = courses.AVAILABILITY_AVAILABLE self.lesson_three = self.course.add_lesson(self.unit) self.lesson_three.title = 'Lesson Three' self.lesson_three.objectives = 'body of lesson' self.lesson_three.availability = courses.AVAILABILITY_AVAILABLE self.assessment_one = self.course.add_assessment() self.assessment_one.title = 'Assessment One' self.assessment_one.html_content = 'assessment one content' self.assessment_one.availability = courses.AVAILABILITY_AVAILABLE self.assessment_two = self.course.add_assessment() self.assessment_two.title = 'Assessment Two' self.assessment_two.html_content = 'assessment two content' self.assessment_two.availability = courses.AVAILABILITY_AVAILABLE self.course.save() actions.login(STUDENT_EMAIL) actions.register(self, STUDENT_EMAIL, COURSE_NAME) self.overridden_environment = actions.OverriddenEnvironment( {'course': { analytics.CAN_RECORD_STUDENT_EVENTS: 'true' }}) self.overridden_environment.__enter__() self.tracker = self.course.get_progress_tracker() with Namespace(NAMESPACE): self.student = models.Student.get_by_user(users.get_current_user())
def add_new_email_settings(cls, unique_name, provider, from_email, api_key, errors): with Namespace(cls.TARGET_NAMESPACE): if cls.get_email_settings_by_key(unique_name): errors.append( 'Unable to add new EmailSettings. Entry with the ' 'same key \'%s\' already exists.' % queue) return settings = EmailSettings(id=unique_name, provider=provider, from_email=from_email, api_key=api_key) settings = settings.put() models.MemcacheManager.delete(cls.list_memcache_key, namespace=cls.TARGET_NAMESPACE) return unique_name
def mark_cleaned_up(self): job = self.load() # If the job has already finished, then the cleanup is a # no-op; we are just reclaiming transient state. However, if # our DurableJobEntity still thinks the job is running and it # is actually not, then mark the status message to indicate # the cleanup. if job and not job.has_finished: duration = int( (datetime.datetime.utcnow() - job.updated_on).total_seconds()) with Namespace(self._namespace): return db.run_in_transaction( self._mark_job_canceled, job, 'Job has not completed; assumed to have failed after %s' % str(datetime.timedelta(seconds=duration)), duration) return job