def ajax_get_report_html(self, report_name=None): ''' return HTML for specified custom report ''' crm, pdata, auth_ok, msg = self.custom_report_auth_check(report_name) # crm = CourseReport model if not auth_ok: return self.no_auth_sorry() if self.request.get('save_parameters'): self.session['edit_report_parameter_values'] = json.dumps({x:v for x,v in pdata.items() if v is not None}) logging.info("Saved edit_report_parameter_values = %s" % self.session['edit_report_parameter_values']) parameters = {x:v for x,v in pdata.items() if v is not None} parameters['orgname'] = self.ORGNAME parameters['dashboard_mode'] = self.MODE # 'mooc' or '' (empty meaning residential, non-mooc) parameters['feature_flags'] = self.FEATURE_FLAGS uuid = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) render_data = {'report_name': report_name, 'report_uuid': "%s_%s" % (report_name, uuid), 'orgname': self.ORGNAME, 'parameters': json.dumps(parameters), # for js 'parameter_values': parameters, # for html template variables 'custom_report': self.custom_report_container(self.is_authorized_for_custom_report, **parameters), # pass pdata so children of page also get parameters 'is_staff': self.is_superuser(), 'is_pm': self.is_pm(), 'cr_js_library': self.get_custom_report_library_javascript(), } render_data.update(pdata) #html = crm.html html = JINJA_ENVIRONMENT.from_string(crm.html).render(render_data) # allows custom reports to be included js_src = "<script type='text/javascript'>" js_src += "$(document).ready( function () {%s} );" % crm.javascript # js goes in html js_src += "</script>" try: js = Template(js_src).render(render_data) # render template variables, ideally w/o global context except Exception as err: logging.info("Warning: could not render js in custom_reports, err=%s" % str(err)) js = JINJA_ENVIRONMENT.from_string(js_src).render(render_data) #template = JINJA_ENVIRONMENT.from_string(html) # logging.info("get_report_html name=%s, parameters=%s" % (report_name, parameters)) data = {'html': html + js, 'js': crm.javascript, } self.response.headers['Content-Type'] = 'application/json' self.response.out.write(json.dumps(data))
def get_main(self): ''' Main page: show list of all courses ''' courses = self.get_course_listings() # logging.info('course listings: %s' % courses['data']) def add_link(xstr, row=None): '''add link to course page to table element''' cid = row.get('course_id', None) if cid: return "<a href=/course/%s>%s</a>" % (cid, xstr) else: return xstr html = self.list2table(map(DataTableField, [{'field': 'launch', 'title':'Course launch', 'width': '12%'}, {'field': 'course_number', 'title': 'Course #', 'width': '5%'}, {'field': 'course_image', 'title': 'Course image'}, {'field': 'title', 'title': 'Course Title', 'width': '40%'}, {'field': 'course_id', 'title': 'course ID', 'width': '12%'}, ]), courses['data'], eformat={'course_number': add_link, 'title': add_link}, ) data = self.common_data data.update({'data': {}, 'is_staff': self.is_superuser(), 'is_pm': self.is_pm(), 'table': html, }) template = JINJA_ENVIRONMENT.get_template('courses.html') self.response.out.write(template.render(data))
def get_custom_report_page(self, report_name): ''' Single custom report which behaves as a HTML page ''' crm, pdata, auth_ok, msg = self.custom_report_auth_check( report_name) # crm = CourseReport model if not auth_ok: return self.no_auth_sorry() if not crm: self.response.write(msg) return html = crm.html html += "<script type='text/javascript'>" html += "$(document).ready( function () {%s} );" % crm.javascript # js goes in html, and thus gets template vars rendered html += "</script>" template = JINJA_ENVIRONMENT.from_string(html) parameters = {x: v for x, v in pdata.items() if v is not None} parameters['orgname'] = self.ORGNAME parameters['feature_flags'] = self.FEATURE_FLAGS render_data = self.common_data.copy() if crm.meta_info.get('need_tags'): render_data['course_tags'] = self.get_course_listings_tags() if crm.meta_info.get('need_listings'): render_data['course_listings'] = self.get_course_listings() if ('course_id' in pdata) and pdata['course_id']: render_data['base'] = self.get_base(pdata['course_id']) logging.info('[page] base=%s' % render_data['base']) render_data.update({ 'report_name': report_name, 'parameters': json.dumps(parameters), # for js 'parameter_values': parameters, # for html template variables 'custom_report': self.custom_report_container( self.is_authorized_for_custom_report, **pdata), # pass pdata so children of page also get parameters 'is_staff': self.is_superuser(), 'is_pm': self.is_pm(), 'msg': msg, 'nav_is_active': self.nav_is_active(report_name), 'cr_js_library': self.get_custom_report_library_javascript(), }) render_data.update(pdata) self.response.out.write(template.render(render_data))
def get_main(self): ''' Main page: show list of all courses ''' courses = self.get_course_listings() # logging.info('course listings: %s' % courses['data']) def add_link(xstr, row=None): '''add link to course page to table element''' cid = row.get('course_id', None) if cid: return "<a href=/course/%s>%s</a>" % (cid, xstr) else: return xstr html = self.list2table( map(DataTableField, [ { 'field': 'launch', 'title': 'Course launch', 'width': '12%' }, { 'field': 'course_number', 'title': 'Course #', 'width': '5%' }, { 'field': 'course_image', 'title': 'Course image' }, { 'field': 'title', 'title': 'Course Title', 'width': '40%' }, { 'field': 'course_id', 'title': 'course ID', 'width': '12%' }, ]), courses['data'], eformat={ 'course_number': add_link, 'title': add_link }, ) data = self.common_data data.update({ 'data': {}, 'is_staff': self.is_superuser(), 'is_pm': self.is_pm(), 'table': html, }) template = JINJA_ENVIRONMENT.get_template('courses.html') self.response.out.write(template.render(data))
def get_video(self, **kwargs): ''' single video analytics view - iframe ''' data = self.setup_module_info(**kwargs) template = JINJA_ENVIRONMENT.get_template('video.html') self.response.out.write(template.render(data))
def get_axis(self, org=None, number=None, semester=None): ''' show full course axis -- mainly for debugging ''' course_id = '/'.join([org, number, semester]) if not self.is_user_authorized_for_course(course_id): return self.no_auth_sorry() caxis = self.load_course_axis(course_id, dtype='data') if self.request.get('ajax'): # return JSON instead of HTML self.response.headers['Content-Type'] = 'application/json' self.response.out.write(json.dumps(self.course_axis, default=self.json_serializer_with_datetime, indent=4)) return if self.request.get('chapters'): # return JSON of just chapters chapters = [x for x in caxis if x['category']=='chapter'] self.response.headers['Content-Type'] = 'application/json' self.response.out.write(json.dumps(chapters, indent=4)) return # logging.info("caxis=%s" % json.dumps(caxis, indent=4)) for cae in caxis: try: # caxis[row]['name'] = fix_bad_unicode(caxis[row]['name']) #caxis[row]['name'] = caxis[row]['name'].replace('\u2013','-') #caxis[row]['name'] = str(caxis[row]['name']) cae['name'] = unidecode(cae['name']) # desparation: perhaps data wasn't encoded properly originally? if cae['gformat']: cae['gformat'] = unidecode(cae['gformat']) # desparation: perhaps data wasn't encoded properly originally? # cae['name'] = str(cae['name']) except Exception as err: print "unicode error for course axis row=%s, name=" % repr(cae), repr(cae['name']) print "type = ", type(cae['name']) raise if 1: fields = ['category', 'index', 'url_name', 'name', 'gformat', 'due', 'start', 'module_id', 'path', 'data_ytid', 'data_weight', 'chapter_mid'] #fields = ['category', 'index', 'name', 'due', 'start', # 'module_id', 'path', 'data_ytid', 'data_weight', 'chapter_mid'] tablehtml = self.list2table(fields, caxis, eformat={'due': self.fix_date, 'start': self.fix_date}, ) data = self.common_data data.update({'course_id': course_id, 'table': tablehtml, }) template = JINJA_ENVIRONMENT.get_template('course_axis.html') self.response.headers['Content-Type'] = 'text/html; charset=utf-8' self.response.out.write(template.render(data))
def get_glossary(self, type=None): ''' Show Glossary data ''' data = '' template = JINJA_ENVIRONMENT.get_template('glossary/glossary_%s.html' % type) self.response.headers['Content-Type'] = 'text/html; charset=utf-8' self.response.out.write(template.render(data))
def render_user_items(user): user_data = User.get_user(user) items = ndb.get_multi(user_data['items']) template = JINJA_ENVIRONMENT.get_template('templates/items.html') context = {'items': [i.to_dict() if i else {} for i in items]} result = template.render(context) return result
def get_developer(self): """ Dashboard page: show cross-course comparisons """ if not self.user in self.AUTHORIZED_USERS: # require superuser return self.no_auth_sorry() data = self.common_data data.update({"is_staff": self.is_superuser()}) template = JINJA_ENVIRONMENT.get_template("dev-editor.html") self.response.out.write(template.render(data))
def get_problem(self, **kwargs): ''' single problem analytics view - iframe ''' data = self.setup_module_info(**kwargs) # template = os.path.join(os.path.dirname(__file__), 'problem.html') template = JINJA_ENVIRONMENT.get_template('problem.html') self.response.out.write(template.render(data))
def get(self): logging.info('overview') template = JINJA_ENVIRONMENT.get_template('templates/jobs.html') context = {} context['descriptions'] = get_cron_descriptions() context['jobs'] = get_cron_jobs() context['laststatus'] = {} if self.request.get('success'): context['laststatus']['success'] = int(self.request.get('success')) self.response.headers['Cache-Control'] = 'no-cache' self.response.write(template.render(context))
def get_developer(self): ''' Dashboard page: show cross-course comparisons ''' if not self.user in self.AUTHORIZED_USERS: # require superuser return self.no_auth_sorry() data = self.common_data data.update({ 'is_staff': self.is_superuser(), }) template = JINJA_ENVIRONMENT.get_template('dev-editor.html') self.response.out.write(template.render(data))
def get_course(self, org=None, number=None, semester=None): ''' single course analytics view - overall statistics (number of components of various categories) - show table of chapters ''' course_id = '/'.join([org, number, semester]) # handle forced recomputation requests action = self.request.POST.get('action', None) logging.info('post keys = %s' % self.request.POST.keys()) logging.info('post action = %s' % action) if action=='force recompute enrollment': self.reset_enrollment_by_day(course_id) # show table with just chapters, and present sequentials as extra information when clicked fields = [ DataTableField(x) for x in [{'field': 'index', 'title': 'Time index', 'width': '8%', 'class': 'dt-center'}, {'field': 'name', 'title': "Chapter name"}, {'field': 'start', 'title': "Start date", 'width': '18%'}, {'field': 'nuser_views', 'title': '# user-views', 'width': '10%', 'class': 'dt-center'}, ] ] # logging.info('sm_usage:') # logging.info(self.bqdata['stats_module_usage']['data_by_key']) tablehtml = self.list2table([' '] + fields, []) tablefields = json.dumps([ { "class": 'details-control', "orderable": False, "data": None, 'width': '5%', "defaultContent": '' },] + [x.colinfo() for x in fields]) data = self.common_data.copy() data.update({'course_id': course_id, 'fields': tablefields, 'table': tablehtml, 'is_staff': self.is_superuser(), 'is_pm': self.is_pm(), 'does_user_have_role': self.does_user_have_role, 'image': self.get_course_image(course_id), 'nav_is_active': self.nav_is_active('onecourse'), 'custom_report': self.custom_report_container(self.is_authorized_for_custom_report, course_id=course_id), }) template = JINJA_ENVIRONMENT.get_template('one_course.html') self.response.out.write(template.render(data))
def get_table(self, dataset=None, table=None, org=None, number=None, semester=None): ''' show arbitrary table from bigquery -- mainly for debugging ''' if dataset is None: course_id = '/'.join([org, number, semester]) dataset = bqutil.course_id2dataset( course_id, use_dataset_latest=self.use_dataset_latest()) if not self.is_user_authorized_for_course(course_id): return self.no_auth_sorry() if ('person' in table) or ('track' in table) or ('student' in table): if not self.does_user_have_role('instructor', course_id): return self.no_auth_sorry() # be more restrictive: researchers only if not (self.does_user_have_role('researcher', course_id)): return self.no_auth_sorry() else: course_id = None if not self.user in self.AUTHORIZED_USERS: return self.no_auth_sorry() tableinfo = bqutil.get_bq_table_info(dataset, table) fields = tableinfo['schema']['fields'] field_names = [x['name'] for x in fields] tablecolumns = json.dumps([{ 'data': x, 'title': x, 'class': 'dt-center' } for x in field_names]) logging.info(tablecolumns) data = self.common_data data.update({ 'dataset': dataset, 'table': table, 'course_id': course_id, 'tablecolumns': tablecolumns, }) template = JINJA_ENVIRONMENT.get_template('show_table.html') self.response.out.write(template.render(data))
def get_custom_report_page(self, report_name): ''' Single custom report which behaves as a HTML page ''' crm, pdata, auth_ok, msg = self.custom_report_auth_check(report_name) # crm = CourseReport model if not auth_ok: return self.no_auth_sorry() if not crm: self.response.write(msg) return html = crm.html html += "<script type='text/javascript'>" html += "$(document).ready( function () {%s} );" % crm.javascript # js goes in html, and thus gets template vars rendered html += "</script>" template = JINJA_ENVIRONMENT.from_string(html) parameters = {x:v for x,v in pdata.items() if v is not None} parameters['orgname'] = self.ORGNAME parameters['feature_flags'] = self.FEATURE_FLAGS render_data = self.common_data.copy() if crm.meta_info.get('need_tags'): render_data['course_tags'] = self.get_course_listings_tags() if crm.meta_info.get('need_listings'): render_data['course_listings'] = self.get_course_listings() if ('course_id' in pdata) and pdata['course_id']: render_data['base'] = self.get_base(pdata['course_id']) logging.info('[page] base=%s' % render_data['base']) render_data.update({'report_name': report_name, 'parameters': json.dumps(parameters), # for js 'parameter_values': parameters, # for html template variables 'custom_report': self.custom_report_container(self.is_authorized_for_custom_report, **pdata), # pass pdata so children of page also get parameters 'is_staff': self.is_superuser(), 'is_pm': self.is_pm(), 'msg': msg, 'nav_is_active': self.nav_is_active(report_name), 'cr_js_library': self.get_custom_report_library_javascript(), }) render_data.update(pdata) self.response.out.write(template.render(render_data))
class ProcedureBase(object): xmpp_template = JINJA_ENVIRONMENT.get_template( 'templates/proc_base.xmpp.template') @classmethod def do_work_core(cls, args): """return ((title, content)...)""" raise NotImplemented @classmethod def render_msg(cls, args, new_items): return cls.xmpp_template.render({ 'message': [i[0] for i in new_items], 'name': args['__name__'] }) @classmethod def do_work(cls, args): logging.debug('%s working with %s' % (cls, args)) result = [] data = cls.do_work_core(args) for title, content in data: ret = ProcItem.add_item(cls.__module__ + '.' + cls.__name__, title, content) if ret: content['title'] = title result.append(content) else: logging.debug('store item fail: %s' % title) if 'xmpp' in args and args['xmpp'].strip(): msg = cls.render_msg(args, result) if msg: xmpp.send_message([args['xmpp']], msg) if 'post' in args: #ret = requests.post(args['post'], json={'items': result}) ret = urlfetch.fetch(args['post'], payload=json.dumps(result), method=urlfetch.POST) if ret.status_code != 200: logging.warn('data post failed: %d' % ret.status_code) return result
def get_problem(self, **kwargs): ''' single problem analytics view - iframe ''' data = self.setup_module_info(**kwargs) data.update({'custom_report': self.custom_report_container(self.is_authorized_for_custom_report, course_id=data['course_id'], chapter_id=data['chapter_mid'], module_id=data['module_id'], ), }) # template = os.path.join(os.path.dirname(__file__), 'problem.html') template = JINJA_ENVIRONMENT.get_template('problem.html') self.response.out.write(template.render(data))
def get_dashboard(self): ''' Dashboard page: show cross-course comparisons ''' courses = self.get_course_listings() data = self.common_data.copy() html = self.list2table(['Registration Open', 'course_id', 'title', 'Course Launch', 'Course Wrap', 'New or Rerun', 'Instructors'], courses['data']) data.update({'is_staff': self.is_superuser(), 'courses': courses, 'table': html, 'ncourses': len(courses['data']), 'custom_report': self.custom_report_container(self.is_authorized_for_custom_report), 'nav_is_active': self.nav_is_active('allcourse'), 'is_pm': self.is_pm(), }) logging.info('session: %s' % dict(self.session)) logging.info('================================================== dataset_latest=%s' % self.use_dataset_latest()) template = JINJA_ENVIRONMENT.get_template('dashboard.html') self.response.out.write(template.render(data))
def get_chapter(self, org=None, number=None, semester=None, url_name=None): ''' single chapter analytics view: container for table data - sequentials and problems ''' course_id = '/'.join([org, number, semester]) caxis = self.load_course_axis(course_id) # get chapter info the_chapter = caxis[url_name] chapter_mid = the_chapter['module_id'] chapter_name = the_chapter['name'] fields = [ DataTableField(x) for x in [{'field': 'index', 'title': 'Time index', 'width': '8%', 'class': 'dt-center'}, {'field': 'category', 'title': "Module category", 'width': '10%'}, {'field': 'name', 'title': "Module name"}, {'field': 'nsubmissions', 'title': "# submissions", 'width': '7%', 'class': 'dt-center'}, {'field': 'avg_grade', 'title': "AVG grade", 'width': '7%', 'class': 'dt-center'}, {'field': 'max_grade', 'title': "MAX grade", 'width': '7%', 'class': 'dt-center'}, {'field': 'avg_attempts', 'title': "AVG attempts", 'width': '7%', 'class': 'dt-center'}, {'field': 'max_attempts', 'title': "MAX attempts", 'width': '7%', 'class': 'dt-center'}, {'field': 'start', 'title': "Start date", 'width': '12%', 'class': 'dt-center'}, {'field': 'nuser_views', 'title': '# user views', 'width': '7%', 'class': 'dt-center'}, # {'field': 'url_name', 'title': 'url_name'}, ] ] tablehtml = self.list2table(fields, []) tablefields = json.dumps([x.colinfo() for x in fields]) data = self.common_data data.update({'fields': tablefields, 'table': tablehtml, 'course_id': course_id, 'chapter_name': chapter_name, 'url_name': url_name, }) template = JINJA_ENVIRONMENT.get_template('chapter.html') self.response.out.write(template.render(data))
def get_table(self, dataset=None, table=None, org=None, number=None,semester=None): ''' show arbitrary table from bigquery -- mainly for debugging ''' if dataset is None: course_id = '/'.join([org, number, semester]) dataset = bqutil.course_id2dataset(course_id, use_dataset_latest=self.use_dataset_latest()) if not self.is_user_authorized_for_course(course_id): return self.no_auth_sorry() if ('person' in table) or ('track' in table) or ('student' in table): if not self.does_user_have_role('instructor', course_id): return self.no_auth_sorry() # be more restrictive: researchers only if not (self.does_user_have_role('researcher', course_id)): return self.no_auth_sorry() else: course_id = None if not self.user in self.AUTHORIZED_USERS: return self.no_auth_sorry() tableinfo = bqutil.get_bq_table_info(dataset, table) fields = tableinfo['schema']['fields'] field_names = [x['name'] for x in fields] tablecolumns = json.dumps([ { 'data': x, 'title': x, 'class': 'dt-center' } for x in field_names ]) logging.info(tablecolumns) data = self.common_data data.update({'dataset': dataset, 'table': table, 'course_id': course_id, 'tablecolumns': tablecolumns, }) template = JINJA_ENVIRONMENT.get_template('show_table.html') self.response.out.write(template.render(data))
def get_problem(self, **kwargs): ''' single problem analytics view - iframe ''' data = self.setup_module_info(**kwargs) data.update({ 'custom_report': self.custom_report_container( self.is_authorized_for_custom_report, course_id=data['course_id'], chapter_id=data['chapter_mid'], module_id=data['module_id'], ), }) # template = os.path.join(os.path.dirname(__file__), 'problem.html') template = JINJA_ENVIRONMENT.get_template('problem.html') self.response.out.write(template.render(data))
def get_admin(self): ''' Admin page: show authorized users, clear cache ''' if not self.user in self.AUTHORIZED_USERS: # require superuser return self.no_auth_sorry() msg = "" action = self.request.POST.get('action', None) custom_reports_standard_source_dir = "ANALYTICS_STANDARD_REPORTS" custom_reports_standard_source_file = "ANALYTICS_STANDARD_REPORTS.yaml" crssd = 'data/%s' % custom_reports_standard_source_dir crssf = 'data/%s' % custom_reports_standard_source_file if os.path.exists(crssd): custom_reports_standard_source = custom_reports_standard_source_dir else: custom_reports_standard_source = custom_reports_standard_source_file try: custom_reports_standard_source = local_config.CUSTOM_REPORTS_SOURCE except Exception as err: pass if action=='Flush cache': memcache.flush_all() msg = "Cache flushed" elif action=='Reload staff table': self.get_staff_table(reload=True) msg = "Staff table reloaded" elif action=='Reload course listings': self.get_course_listings(ignore_cache=True) msg = "Course listings reloaded" elif action=='List current course tags': tags = self.get_course_listings_tags() msg = "Current course tags = %s" % tags elif action=='Check access': username = self.request.get('username') course_id = self.request.get('course_id') if self.is_user_authorized_for_course(course_id=course_id, user=username): msg = "User %s IS authorized for %s" % (username, course_id) else: msg = "User %s is NOT authorized for %s" % (username, course_id) elif action=='Reload Course Listings': collection = self.request.POST.get('collection') self.get_course_listings(ignore_cache=True, collection=collection) msg = "Course listings for '%s' reloaded" % collection elif action=='Reload Standard Reports': crssd = 'data/%s' % custom_reports_standard_source if os.path.exists(crssd): if os.path.isdir(crssd): files = glob.glob('%s/*.yaml' % crssd) elif os.path.isfile(crssd): files = [crssd] msg = "<ul>" for fn in files: msg += "<li>Standard Reports loading from %s<br/>" % (fn) report_file_data = open(fn).read() try: msg += self.import_custom_report_from_file_data(report_file_data, overwrite=True) except Exception as err: logging.error("Oops! Error importing custom report from %s" % fn) raise msg += "</li>" msg += "</ul>" else: msg = "Error: cannot find file or directory %s" % crssd elif action=='Reload Custom Reports': collection = self.request.POST.get('collection') cnt = self.import_custom_report_metadata(ignore_cache=True, collection=collection) msg = "Custom Report Metadata for '%s' reloaded (%d reports)" % (collection, cnt) elif action=='Export Custom Reports': collection = self.request.POST.get('collection') cnt, destination = self.export_custom_report_metadata(ignore_cache=True, collection=collection) msg = "Custom Report Metadata for '%s' exported to %s (%d reports)" % (collection, destination, cnt) elif action=='Add staff': fields = ['username', 'role', 'course_id', 'notes'] data = { x: (self.request.POST.get(x) or '') for x in fields } self.add_staff_table_entry(data) msg = "New staff %s added" % data todelete = self.request.POST.get('do-delete', None) if todelete is not None: self.disable_staff_table_entry(int(todelete)) msg = "Deleted staff table row %s" % todelete stable = self.get_staff_table() stafftable = self.list2table([DataTableField({'icon':'delete', 'field': 'sid', 'title':' '}), 'username', 'role', 'course_id', 'notes'], stable) data = self.common_data.copy() course_listings_source = self.get_collection_metadata('COURSE_LISTINGS_TABLE') data.update({'superusers': self.AUTHORIZED_USERS, 'table': stafftable, 'msg': msg, 'listings_source': course_listings_source, 'staff_source': local_config.STAFF_COURSE_TABLE, 'collections': self.collections_available(asdict=True), 'custom_reports_standard_source': custom_reports_standard_source, }) template = JINJA_ENVIRONMENT.get_template('admin.html') self.response.out.write(template.render(data))
def get_axis(self, org=None, number=None, semester=None): ''' show full course axis -- mainly for debugging ''' course_id = '/'.join([org, number, semester]) if not self.is_user_authorized_for_course(course_id): return self.no_auth_sorry() caxis = self.load_course_axis(course_id, dtype='data') if self.request.get('ajax'): # return JSON instead of HTML self.response.headers['Content-Type'] = 'application/json' self.response.out.write( json.dumps(self.course_axis, default=self.json_serializer_with_datetime, indent=4)) return if self.request.get('chapters'): # return JSON of just chapters chapters = [x for x in caxis if x['category'] == 'chapter'] self.response.headers['Content-Type'] = 'application/json' self.response.out.write(json.dumps(chapters, indent=4)) return # logging.info("caxis=%s" % json.dumps(caxis, indent=4)) for cae in caxis: try: # caxis[row]['name'] = fix_bad_unicode(caxis[row]['name']) #caxis[row]['name'] = caxis[row]['name'].replace('\u2013','-') #caxis[row]['name'] = str(caxis[row]['name']) cae['name'] = unidecode( cae['name'] ) # desparation: perhaps data wasn't encoded properly originally? if cae['gformat']: cae['gformat'] = unidecode( cae['gformat'] ) # desparation: perhaps data wasn't encoded properly originally? # cae['name'] = str(cae['name']) except Exception as err: print "unicode error for course axis row=%s, name=" % repr( cae), repr(cae['name']) print "type = ", type(cae['name']) raise if 1: fields = [ 'category', 'index', 'url_name', 'name', 'gformat', 'due', 'start', 'module_id', 'path', 'data_ytid', 'data_weight', 'chapter_mid' ] #fields = ['category', 'index', 'name', 'due', 'start', # 'module_id', 'path', 'data_ytid', 'data_weight', 'chapter_mid'] tablehtml = self.list2table( fields, caxis, eformat={ 'due': self.fix_date, 'start': self.fix_date }, ) data = self.common_data data.update({ 'course_id': course_id, 'table': tablehtml, }) template = JINJA_ENVIRONMENT.get_template('course_axis.html') self.response.headers['Content-Type'] = 'text/html; charset=utf-8' self.response.out.write(template.render(data))
def get_chapter(self, org=None, number=None, semester=None, url_name=None, seq=None): ''' single chapter analytics view: container for table data - sequentials and problems ''' course_id = '/'.join([org, number, semester]) caxis = self.load_course_axis(course_id) # get chapter info the_chapter = caxis[url_name] # if seq is specified, then look for the chapter before or after, as requested error = None if seq in ["next", "prev"]: chapters = [x for x in caxis.values() if (x['category']=='chapter')] chapters.sort(cmp=lambda x,y: int(x['index'])-int(y['index'])) # list of chapters ordered by course axis index try: cidx = chapters.index(the_chapter) except Exception as err: error = {'msg': "Module %s not found in list of chapters %s" % (url_name, chapters)} cidx = None if (cidx is not None) and seq=='next': if cidx < len(chapters)-1: the_chapter = chapters[cidx+1] module_id = the_chapter['module_id'] url_name = module_id.rsplit('/',1)[-1] else: error = {'msg': "No chapter available after %s in this course" % (url_name)} if (cidx is not None) and seq=='prev': if cidx > 0: the_chapter = chapters[cidx-1] module_id = the_chapter['module_id'] url_name = module_id.rsplit('/',1)[-1] else: error = {'msg': "No chapter available before %s in this course" % (url_name)} chapter_mid = the_chapter['module_id'] chapter_name = the_chapter['name'] fields = [ DataTableField(x) for x in [{'field': 'index', 'title': 'Time index', 'width': '8%', 'class': 'dt-center'}, {'field': 'category', 'title': "Module category", 'width': '10%'}, {'field': 'name', 'title': "Module name"}, {'field': 'nsubmissions', 'title': "# submissions", 'width': '7%', 'class': 'dt-center'}, {'field': 'avg_grade', 'title': "AVG grade", 'width': '7%', 'class': 'dt-center'}, {'field': 'max_grade', 'title': "MAX grade", 'width': '7%', 'class': 'dt-center'}, {'field': 'avg_attempts', 'title': "AVG attempts", 'width': '7%', 'class': 'dt-center'}, {'field': 'max_attempts', 'title': "MAX attempts", 'width': '7%', 'class': 'dt-center'}, {'field': 'start', 'title': "Start date", 'width': '12%', 'class': 'dt-center'}, {'field': 'nuser_views', 'title': '# user views', 'width': '7%', 'class': 'dt-center'}, # {'field': 'url_name', 'title': 'url_name'}, ] ] tablehtml = self.list2table(fields, []) tablefields = json.dumps([x.colinfo() for x in fields]) data = self.common_data data.update({'fields': tablefields, 'table': tablehtml, 'course_id': course_id, 'chapter_name': chapter_name, 'url_name': url_name, 'error': error, }) template = JINJA_ENVIRONMENT.get_template('chapter.html') self.response.out.write(template.render(data))
def get_course(self, org=None, number=None, semester=None): ''' single course analytics view - overall statistics (number of components of various categories) - show table of chapters ''' course_id = '/'.join([org, number, semester]) # handle forced recomputation requests action = self.request.POST.get('action', None) logging.info('post keys = %s' % self.request.POST.keys()) logging.info('post action = %s' % action) if action == 'force recompute enrollment': self.reset_enrollment_by_day(course_id) # show table with just chapters, and present sequentials as extra information when clicked fields = [ DataTableField(x) for x in [ { 'field': 'index', 'title': 'Time index', 'width': '8%', 'class': 'dt-center' }, { 'field': 'name', 'title': "Chapter name" }, { 'field': 'start', 'title': "Start date", 'width': '18%' }, { 'field': 'nuser_views', 'title': '# user-views', 'width': '10%', 'class': 'dt-center' }, ] ] # logging.info('sm_usage:') # logging.info(self.bqdata['stats_module_usage']['data_by_key']) tablehtml = self.list2table([' '] + fields, []) tablefields = json.dumps([ { "class": 'details-control', "orderable": False, "data": None, 'width': '5%', "defaultContent": '' }, ] + [x.colinfo() for x in fields]) data = self.common_data.copy() data.update({ 'course_id': course_id, 'fields': tablefields, 'table': tablehtml, 'is_staff': self.is_superuser(), 'is_pm': self.is_pm(), 'does_user_have_role': self.does_user_have_role, 'image': self.get_course_image(course_id), 'nav_is_active': self.nav_is_active('onecourse'), 'custom_report': self.custom_report_container(self.is_authorized_for_custom_report, course_id=course_id), }) template = JINJA_ENVIRONMENT.get_template('one_course.html') self.response.out.write(template.render(data))
def get_chapter(self, org=None, number=None, semester=None, url_name=None, seq=None): ''' single chapter analytics view: container for table data - sequentials and problems ''' course_id = '/'.join([org, number, semester]) caxis = self.load_course_axis(course_id) # get chapter info the_chapter = caxis[url_name] # if seq is specified, then look for the chapter before or after, as requested error = None if seq in ["next", "prev"]: chapters = [ x for x in caxis.values() if (x['category'] == 'chapter') ] chapters.sort(cmp=lambda x, y: int(x['index']) - int(y['index']) ) # list of chapters ordered by course axis index try: cidx = chapters.index(the_chapter) except Exception as err: error = { 'msg': "Module %s not found in list of chapters %s" % (url_name, chapters) } cidx = None if (cidx is not None) and seq == 'next': if cidx < len(chapters) - 1: the_chapter = chapters[cidx + 1] module_id = the_chapter['module_id'] url_name = module_id.rsplit('/', 1)[-1] else: error = { 'msg': "No chapter available after %s in this course" % (url_name) } if (cidx is not None) and seq == 'prev': if cidx > 0: the_chapter = chapters[cidx - 1] module_id = the_chapter['module_id'] url_name = module_id.rsplit('/', 1)[-1] else: error = { 'msg': "No chapter available before %s in this course" % (url_name) } chapter_mid = the_chapter['module_id'] chapter_name = the_chapter['name'] fields = [ DataTableField(x) for x in [ { 'field': 'index', 'title': 'Time index', 'width': '8%', 'class': 'dt-center' }, { 'field': 'category', 'title': "Module category", 'width': '10%' }, { 'field': 'name', 'title': "Module name" }, { 'field': 'nsubmissions', 'title': "# submissions", 'width': '7%', 'class': 'dt-center' }, { 'field': 'avg_grade', 'title': "AVG grade", 'width': '7%', 'class': 'dt-center' }, { 'field': 'max_grade', 'title': "MAX grade", 'width': '7%', 'class': 'dt-center' }, { 'field': 'avg_attempts', 'title': "AVG attempts", 'width': '7%', 'class': 'dt-center' }, { 'field': 'max_attempts', 'title': "MAX attempts", 'width': '7%', 'class': 'dt-center' }, { 'field': 'start', 'title': "Start date", 'width': '12%', 'class': 'dt-center' }, { 'field': 'nuser_views', 'title': '# user views', 'width': '7%', 'class': 'dt-center' }, # {'field': 'url_name', 'title': 'url_name'}, ] ] tablehtml = self.list2table(fields, []) tablefields = json.dumps([x.colinfo() for x in fields]) data = self.common_data data.update({ 'fields': tablefields, 'table': tablehtml, 'course_id': course_id, 'chapter_name': chapter_name, 'url_name': url_name, 'error': error, }) template = JINJA_ENVIRONMENT.get_template('chapter.html') self.response.out.write(template.render(data))
def edit_custom_report(self, report_name, crm=None): ''' edit custom report html, javascript, sql, description, ... ''' if not self.user in self.AUTHORIZED_USERS: # require superuser return self.no_auth_sorry() parameter_values = self.session.get('edit_report_parameter_values') if not parameter_values or parameter_values == "None": parameter_values = {} # self.session['edit_report_parameter_values'] = parameter_values editor_heights = self.request.POST.get('editor_heights') if editor_heights: self.session[ 'editor_heights'] = editor_heights # keep editor heights for session else: editor_heights = self.session.get('editor_heights') logging.info('[custom_reports] editor heights=%s' % editor_heights) msg = '' if (self.request.POST.get('action') == 'Download Report'): try: data = self.export_custom_report_metadata( report_name=report_name, download=True) dump = yaml.dump(data, default_style="|", default_flow_style=False) # logging.info("custom report yaml=%s" % dump) except Exception as err: logging.error("Failed to find custom report named %s!" % report_name) raise self.response.headers['Content-Type'] = 'application/text' self.response.headers[ 'Content-Disposition'] = 'attachment; filename=ANALYTICS_REPORT_%s.yaml' % report_name self.response.out.write(dump) return elif (self.request.POST.get('action') == 'Download ALL Reports'): try: data = self.export_custom_report_metadata(report_name=None, download=True) dump = yaml.dump(data, default_style="|", default_flow_style=False) logging.info("custom report yaml=%s" % dump) except Exception as err: raise dtstr = self.TheNow().strftime('%Y-%m-%d_%H%M') self.response.headers['Content-Type'] = 'application/text' self.response.headers[ 'Content-Disposition'] = 'attachment; filename=ANALYTICS_ALL_REPORTS_%s.yaml' % dtstr self.response.out.write(dump) return elif (self.request.POST.get('action') == 'Delete Report'): try: crm = self.get_custom_report_metadata(report_name) except Exception as err: logging.error("Failed to find custom report named %s!" % report_name) raise crm.key.delete() msg = "Deleted course report %s" % report_name return self.get_custom_report(msg=msg) elif (self.request.POST.get('action') == 'Save Changes'): fields = [ 'table_name', 'title', 'depends_on', 'html', 'sql', 'javascript', 'description', 'collection', 'group_tags', 'meta_info' ] try: crm = self.get_custom_report_metadata(report_name) except Exception as err: logging.error("Failed to find custom report named %s!" % report_name) raise for field in fields: fval = self.request.POST.get(field) if field == 'group_tags': fval = [x.strip() for x in fval.split(',')] elif field == 'meta_info': fval = eval(fval) or {} if fval is None: logging.error( "oops, expected value for field=%s, but got fval=%s" % (field, fval)) else: setattr(crm, field, fval) crm.put() logging.info('saved crm = %s' % crm) msg = "Saved custom report %s" % report_name if not crm.table_name: msg += "...Warning! table_name cannot be left empty" try: if not crm: crm = self.get_custom_report_metadata(report_name) except Exception as err: logging.error("Cannot get custom report %s" % report_name) raise if not crm: msg = "Cannot get custom report %s" % report_name logging.error(msg) raise Exception(msg) # backward compatability: to accommodate jhints, ignore lines in javascript with jinja2 template commands if ('{{parameters}}' in crm.javascript) and not ('jshint ignore' in crm.javascript): newjs = [] for k in crm.javascript.split('\n'): if (("{% autoescape" in k)) and not ('jshint ignore' in k): k += " // jshint ignore:line" elif ('{{parameters}}' in k) and not ('jshint ignore' in k): newjs.append("/* jshint ignore:start */") newjs.append(k) k = "/* jshint ignore:end */" newjs.append(k) crm.javascript = '\n'.join(newjs) data = { 'html': crm.html, 'js': crm.javascript, 'report_name': report_name, 'report': crm, 'msg': msg, 'parameter_values': parameter_values, 'meta_info': json.dumps(crm.meta_info), 'editor_heights': editor_heights or "{}", 'cr_js_library': self.get_custom_report_library_javascript(), } data.update(self.common_data) template = JINJA_ENVIRONMENT.get_template('edit_custom_report.html') self.response.out.write(template.render(data))
def edit_custom_report(self, report_name, crm=None): ''' edit custom report html, javascript, sql, description, ... ''' if not self.user in self.AUTHORIZED_USERS: # require superuser return self.no_auth_sorry() parameter_values = self.session.get('edit_report_parameter_values') if not parameter_values or parameter_values=="None": parameter_values = {} # self.session['edit_report_parameter_values'] = parameter_values editor_heights = self.request.POST.get('editor_heights') if editor_heights: self.session['editor_heights'] = editor_heights # keep editor heights for session else: editor_heights = self.session.get('editor_heights') logging.info('[custom_reports] editor heights=%s' % editor_heights) msg = '' if (self.request.POST.get('action')=='Download Report'): try: data = self.export_custom_report_metadata(report_name=report_name, download=True) dump = yaml.dump(data, default_style="|", default_flow_style=False) # logging.info("custom report yaml=%s" % dump) except Exception as err: logging.error("Failed to find custom report named %s!" % report_name) raise self.response.headers['Content-Type'] = 'application/text' self.response.headers['Content-Disposition'] = 'attachment; filename=ANALYTICS_REPORT_%s.yaml' % report_name self.response.out.write(dump) return elif (self.request.POST.get('action')=='Download ALL Reports'): try: data = self.export_custom_report_metadata(report_name=None, download=True) dump = yaml.dump(data, default_style="|", default_flow_style=False) logging.info("custom report yaml=%s" % dump) except Exception as err: raise dtstr = self.TheNow().strftime('%Y-%m-%d_%H%M') self.response.headers['Content-Type'] = 'application/text' self.response.headers['Content-Disposition'] = 'attachment; filename=ANALYTICS_ALL_REPORTS_%s.yaml' % dtstr self.response.out.write(dump) return elif (self.request.POST.get('action')=='Delete Report'): try: crm = self.get_custom_report_metadata(report_name) except Exception as err: logging.error("Failed to find custom report named %s!" % report_name) raise crm.key.delete() msg = "Deleted course report %s" % report_name return self.get_custom_report(msg=msg) elif (self.request.POST.get('action')=='Save Changes'): fields = ['table_name', 'title', 'depends_on', 'html', 'sql', 'javascript', 'description', 'collection', 'group_tags', 'meta_info'] try: crm = self.get_custom_report_metadata(report_name) except Exception as err: logging.error("Failed to find custom report named %s!" % report_name) raise for field in fields: fval = self.request.POST.get(field) if field=='group_tags': fval = [x.strip() for x in fval.split(',')] elif field=='meta_info': fval = eval(fval) or {} if fval is None: logging.error("oops, expected value for field=%s, but got fval=%s" % (field, fval)) else: setattr(crm, field, fval) crm.put() logging.info('saved crm = %s' % crm) msg = "Saved custom report %s" % report_name if not crm.table_name: msg += "...Warning! table_name cannot be left empty" try: if not crm: crm = self.get_custom_report_metadata(report_name) except Exception as err: logging.error("Cannot get custom report %s" % report_name) raise # backward compatability: to accommodate jhints, ignore lines in javascript with jinja2 template commands if ('{{parameters}}' in crm.javascript) and not ('jshint ignore' in crm.javascript): newjs = [] for k in crm.javascript.split('\n'): if (("{% autoescape" in k)) and not ('jshint ignore' in k): k += " // jshint ignore:line" elif ('{{parameters}}' in k) and not ('jshint ignore' in k): newjs.append("/* jshint ignore:start */") newjs.append(k) k = "/* jshint ignore:end */" newjs.append(k) crm.javascript = '\n'.join(newjs) data = {'html': crm.html, 'js': crm.javascript, 'report_name': report_name, 'report': crm, 'msg': msg, 'parameter_values': parameter_values, 'meta_info': json.dumps(crm.meta_info), 'editor_heights': editor_heights or "{}", } data.update(self.common_data) template = JINJA_ENVIRONMENT.get_template('edit_custom_report.html') self.response.out.write(template.render(data))
def get_admin(self): ''' Admin page: show authorized users, clear cache ''' if not self.user in self.AUTHORIZED_USERS: # require superuser return self.no_auth_sorry() msg = "" action = self.request.POST.get('action', None) custom_reports_standard_source_dir = "ANALYTICS_STANDARD_REPORTS" custom_reports_standard_source_file = "ANALYTICS_STANDARD_REPORTS.yaml" crssd = 'data/%s' % custom_reports_standard_source_dir crssf = 'data/%s' % custom_reports_standard_source_file if os.path.exists(crssd): custom_reports_standard_source = custom_reports_standard_source_dir else: custom_reports_standard_source = custom_reports_standard_source_file try: custom_reports_standard_source = local_config.CUSTOM_REPORTS_SOURCE except Exception as err: pass if action == 'Flush cache': memcache.flush_all() msg = "Cache flushed" elif action == 'Reload staff table': self.get_staff_table(reload=True) msg = "Staff table reloaded" elif action == 'Reload course listings': self.get_course_listings(ignore_cache=True) msg = "Course listings reloaded" elif action == 'List current course tags': tags = self.get_course_listings_tags() msg = "Current course tags = %s" % tags elif action == 'Check access': username = self.request.get('username') course_id = self.request.get('course_id') if self.is_user_authorized_for_course(course_id=course_id, user=username): msg = "User %s IS authorized for %s" % (username, course_id) else: msg = "User %s is NOT authorized for %s" % (username, course_id) elif action == 'Reload Course Listings': collection = self.request.POST.get('collection') self.get_course_listings(ignore_cache=True, collection=collection) msg = "Course listings for '%s' reloaded" % collection elif action == 'Reload Standard Reports': crssd = 'data/%s' % custom_reports_standard_source if os.path.exists(crssd): if os.path.isdir(crssd): files = glob.glob('%s/*.yaml' % crssd) elif os.path.isfile(crssd): files = [crssd] msg = "<ul>" for fn in files: msg += "<li>Standard Reports loading from %s<br/>" % (fn) report_file_data = open(fn).read() try: msg += self.import_custom_report_from_file_data( report_file_data, overwrite=True) except Exception as err: logging.error( "Oops! Error importing custom report from %s" % fn) raise msg += "</li>" msg += "</ul>" else: msg = "Error: cannot find file or directory %s" % crssd elif action == 'Reload Custom Reports': collection = self.request.POST.get('collection') cnt = self.import_custom_report_metadata(ignore_cache=True, collection=collection) msg = "Custom Report Metadata for '%s' reloaded (%d reports)" % ( collection, cnt) elif action == 'Export Custom Reports': collection = self.request.POST.get('collection') cnt, destination = self.export_custom_report_metadata( ignore_cache=True, collection=collection) msg = "Custom Report Metadata for '%s' exported to %s (%d reports)" % ( collection, destination, cnt) elif action == 'Add staff': fields = ['username', 'role', 'course_id', 'notes'] data = {x: (self.request.POST.get(x) or '') for x in fields} self.add_staff_table_entry(data) msg = "New staff %s added" % data todelete = self.request.POST.get('do-delete', None) if todelete is not None: self.disable_staff_table_entry(int(todelete)) msg = "Deleted staff table row %s" % todelete stable = self.get_staff_table() stafftable = self.list2table([ DataTableField({ 'icon': 'delete', 'field': 'sid', 'title': ' ' }), 'username', 'role', 'course_id', 'notes' ], stable) data = self.common_data.copy() course_listings_source = self.get_collection_metadata( 'COURSE_LISTINGS_TABLE') data.update({ 'superusers': self.AUTHORIZED_USERS, 'table': stafftable, 'msg': msg, 'listings_source': course_listings_source, 'staff_source': local_config.STAFF_COURSE_TABLE, 'collections': self.collections_available(asdict=True), 'custom_reports_standard_source': custom_reports_standard_source, }) template = JINJA_ENVIRONMENT.get_template('admin.html') self.response.out.write(template.render(data))
def get_custom_report(self, msg=""): ''' custom reports page ''' if not self.user in self.AUTHORIZED_USERS: # require superuser return self.no_auth_sorry() if (self.request.POST.get('action')=='Create new Custom Report'): title = self.request.POST.get('title') name = self.request.POST.get('name') existing_crm = self.get_custom_report_metadata(name, single=False) if existing_crm.count(): msg = "Cannot create report '%s', already exists" % name else: crm = CustomReport(title=title, name=name) #crm.html = """<div id="contain-{{report_name}}" style="min-width: 310px; height: 400px; margin: 0 auto"> crm.html = """<div id="contain-{{report_name}}" style="min-width: 310px; margin: 0 auto"> <img src="/images/loading_icon.gif"/>\n</div>""" jstemp, jsfn, uptodate = JINJA_ENVIRONMENT.loader.get_source(JINJA_ENVIRONMENT, 'custom_report_default.js') crm.javascript = str(jstemp) #jstemp = JINJA_ENVIRONMENT.get_template('custom_report_default.js') #crm.javascript = jstemp.render({}) logging.info("[cr] creating new custom report %s" % crm) crm.put() # return self.redirect('/custom/edit_report/%s' % name) # # because of how NDB may take awhile to store the new entry, we cannot go directly # to the report edit page, but instead, must go to a transition page asking # for the user to click on a button first. data = self.common_data.copy() data.update({'report': crm, }) template = JINJA_ENVIRONMENT.get_template('edit_custom_report_transition.html') self.response.out.write(template.render(data)) return elif (self.request.POST.get('action')=='Edit this report'): name = self.request.POST.get('name') return self.redirect('/custom/edit_report/%s' % name) elif (self.request.POST.get('action')=='Upload Custom Report(s)'): report_file_data = self.request.get('file') overwrite = (self.request.get('overwrite')=='yes') msg += self.import_custom_report_from_file_data(report_file_data, overwrite) cr_page_title = "Custom Reports" try: cr_page_title = local_config.CUSTOM_REPORTS_PAGE_TITLE except Exception as err: pass data = self.common_data.copy() data.update({'is_staff': self.is_superuser(), 'reports': self.get_custom_report_metadata(single=False), 'msg': msg, 'custom_report': self.custom_report_container(self.is_authorized_for_custom_report, staff=True, group_tag = "{{group_tag}}", ), 'cr_page_title': cr_page_title, }) template = JINJA_ENVIRONMENT.get_template('custom_reports.html') self.response.out.write(template.render(data))
def __getitem__(self, report_name): try: crm = other.get_custom_report_metadata(report_name) err = None except Exception as err: crm = None if not crm: logging.info("No custom report '%s' found, err=%s" % (report_name, err)) return "Missing custom report %s" % report_name # check access authorization # logging.info('[crc] checking auth for report %s, pdata=%s' % (crm.name, pdata)) auth_ok, msg = is_authorized_for_custom_report(crm, pdata) if not auth_ok: return "" # return empty string if not authorized # logging.info('[cr] name=%s, title=%s' % (crm.name, crm.title)) # debugging title = JINJA_ENVIRONMENT.from_string(crm.title) try: title_rendered = title.render(pdata) except Exception as err: logging.error('[cr] Failed to render report %s title %s' % (crm.name, crm.title)) title = crm.title parameters = {x:v for x,v in pdata.items() if v is not None} parameters['orgname'] = other.ORGNAME parameters['dashboard_mode'] = other.MODE # 'mooc' or '' (empty meaning residential, non-mooc) parameters['course_report'] = other.get_course_report_dataset() parameters['course_report_org'] = other.get_course_report_dataset(force_use_org=True) parameters['orgname'] = other.ORGNAME if 'require_table' in (crm.meta_info or []): dataset = None table = crm.meta_info['require_table'] if '{' in table: try: table = table.format(**parameters) except Exception as err: logging.error("Cannot substitute for parameters in require_table=%s, err=%s" % (table, err)) if '.' in table: (dataset, table) = table.split('.', 1) else: course_id = parameters.get('course_id') if course_id: try: dataset = bqutil.course_id2dataset(course_id, use_dataset_latest=other.use_dataset_latest()) except Exception as err: logging.error("failed to get dataset for course_id=%s" % course_id) raise else: logging.info("Suppressing report %s because dataset not specifid in require_table %s" % (title, table)) dataset = None if dataset is not None: try: tinfo = bqutil.get_bq_table_info(dataset, table) or None except Exception as err: tinfo = None if not "Not Found" in str(err): logging.error(err) logging.error(traceback.format_exc()) if not tinfo: logging.info("Suppressing report %s because %s.%s doesn't exist" % (title, dataset, table)) return "" else: logging.info("Skipping require_table check") report_id = hashlib.sha224("%s %s" % (crm.name, json.dumps(pdata))).hexdigest() if crm.description: try: crm.description = crm.description.format(**parameters) except Exception as err: logging.info('[cr] %s cannot format description %s' % (crm.name, crm.description)) if self.do_no_embed and 'embedded' in (crm.meta_info or {}): crm.meta_info.pop('embedded') if self.force_embed: crm.meta_info['embedded'] = True if self.do_always_show: crm.meta_info['always_show'] = True template = JINJA_ENVIRONMENT.get_template('custom_report_container.html') data = {'is_staff': other.is_superuser(), 'report': crm, 'report_params': json.dumps(parameters), 'report_is_staff': pdata.get('staff'), 'report_meta_info': json.dumps(crm.meta_info or {}), 'immediate_view': json.dumps(self.immediate_view), 'do_embed' : (crm.meta_info or {}).get('embedded') or self.force_embed, 'always_show': self.do_always_show, 'title': title_rendered, 'id': report_id, } self.immediate_view = False # return to non-immediate view by default self.do_no_embed = False # return to default self.force_embed = False # return to default return template.render(data)
def ajax_get_report_html(self, report_name=None): ''' return HTML for specified custom report ''' crm, pdata, auth_ok, msg = self.custom_report_auth_check( report_name) # crm = CourseReport model if not auth_ok: return self.no_auth_sorry() if self.request.get('save_parameters'): self.session['edit_report_parameter_values'] = json.dumps( {x: v for x, v in pdata.items() if v is not None}) logging.info("Saved edit_report_parameter_values = %s" % self.session['edit_report_parameter_values']) parameters = {x: v for x, v in pdata.items() if v is not None} parameters['orgname'] = self.ORGNAME parameters[ 'dashboard_mode'] = self.MODE # 'mooc' or '' (empty meaning residential, non-mooc) parameters['feature_flags'] = self.FEATURE_FLAGS uuid = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) render_data = { 'report_name': report_name, 'report_uuid': "%s_%s" % (report_name, uuid), 'orgname': self.ORGNAME, 'parameters': json.dumps(parameters), # for js 'parameter_values': parameters, # for html template variables 'custom_report': self.custom_report_container( self.is_authorized_for_custom_report, **parameters ), # pass pdata so children of page also get parameters 'is_staff': self.is_superuser(), 'is_pm': self.is_pm(), 'cr_js_library': self.get_custom_report_library_javascript(), } render_data.update(pdata) #html = crm.html html = JINJA_ENVIRONMENT.from_string(crm.html).render( render_data) # allows custom reports to be included js_src = "<script type='text/javascript'>" js_src += "$(document).ready( function () {%s} );" % crm.javascript # js goes in html js_src += "</script>" try: js = Template(js_src).render( render_data ) # render template variables, ideally w/o global context except Exception as err: logging.info( "Warning: could not render js in custom_reports, err=%s" % str(err)) js = JINJA_ENVIRONMENT.from_string(js_src).render(render_data) #template = JINJA_ENVIRONMENT.from_string(html) # logging.info("get_report_html name=%s, parameters=%s" % (report_name, parameters)) data = { 'html': html + js, 'js': crm.javascript, } self.response.headers['Content-Type'] = 'application/json' self.response.out.write(json.dumps(data))
def get_custom_report(self, msg=""): ''' custom reports page ''' if not self.user in self.AUTHORIZED_USERS: # require superuser return self.no_auth_sorry() if (self.request.POST.get('action') == 'Create new Custom Report'): title = self.request.POST.get('title') name = self.request.POST.get('name') existing_crm = self.get_custom_report_metadata(name, single=False) if existing_crm.count(): msg = "Cannot create report '%s', already exists" % name else: crm = CustomReport(title=title, name=name) #crm.html = """<div id="contain-{{report_name}}" style="min-width: 310px; height: 400px; margin: 0 auto"> crm.html = """<div id="contain-{{report_name}}" style="min-width: 310px; margin: 0 auto"> <img src="/images/loading_icon.gif"/>\n</div>""" jstemp, jsfn, uptodate = JINJA_ENVIRONMENT.loader.get_source( JINJA_ENVIRONMENT, 'custom_report_default.js') crm.javascript = str(jstemp) #jstemp = JINJA_ENVIRONMENT.get_template('custom_report_default.js') #crm.javascript = jstemp.render({}) logging.info("[cr] creating new custom report %s" % crm) crm.put() # return self.redirect('/custom/edit_report/%s' % name) # # because of how NDB may take awhile to store the new entry, we cannot go directly # to the report edit page, but instead, must go to a transition page asking # for the user to click on a button first. data = self.common_data.copy() data.update({ 'report': crm, }) template = JINJA_ENVIRONMENT.get_template( 'edit_custom_report_transition.html') self.response.out.write(template.render(data)) return elif (self.request.POST.get('action') == 'Edit this report'): name = self.request.POST.get('name') return self.redirect('/custom/edit_report/%s' % name) elif (self.request.POST.get('action') == 'Upload Custom Report(s)'): report_file_data = self.request.get('file') overwrite = (self.request.get('overwrite') == 'yes') msg += self.import_custom_report_from_file_data( report_file_data, overwrite) cr_page_title = "Custom Reports" try: cr_page_title = local_config.CUSTOM_REPORTS_PAGE_TITLE except Exception as err: pass data = self.common_data.copy() data.update({ 'is_staff': self.is_superuser(), 'reports': self.get_custom_report_metadata(single=False), 'msg': msg, 'custom_report': self.custom_report_container( self.is_authorized_for_custom_report, staff=True, group_tag="{{group_tag}}", ), 'cr_page_title': cr_page_title, }) template = JINJA_ENVIRONMENT.get_template('custom_reports.html') self.response.out.write(template.render(data))