def ajax_get_report_html(self, report_name=None):
'''
return HTML for specified custom report
'''
crm, pdata, auth_ok, msg = self.custom_report_auth_check(report_name) # crm = CourseReport model
if not auth_ok:
return self.no_auth_sorry()
if self.request.get('save_parameters'):
self.session['edit_report_parameter_values'] = json.dumps({x:v for x,v in pdata.items() if v is not None})
logging.info("Saved edit_report_parameter_values = %s" % self.session['edit_report_parameter_values'])
parameters = {x:v for x,v in pdata.items() if v is not None}
parameters['orgname'] = self.ORGNAME
parameters['dashboard_mode'] = self.MODE # 'mooc' or '' (empty meaning residential, non-mooc)
parameters['feature_flags'] = self.FEATURE_FLAGS
uuid = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
render_data = {'report_name': report_name,
'report_uuid': "%s_%s" % (report_name, uuid),
'orgname': self.ORGNAME,
'parameters': json.dumps(parameters), # for js
'parameter_values': parameters, # for html template variables
'custom_report': self.custom_report_container(self.is_authorized_for_custom_report,
**parameters), # pass pdata so children of page also get parameters
'is_staff': self.is_superuser(),
'is_pm': self.is_pm(),
'cr_js_library': self.get_custom_report_library_javascript(),
}
render_data.update(pdata)
#html = crm.html
html = JINJA_ENVIRONMENT.from_string(crm.html).render(render_data) # allows custom reports to be included
js_src = "<script type='text/javascript'>"
js_src += "$(document).ready( function () {%s} );" % crm.javascript # js goes in html
js_src += "</script>"
try:
js = Template(js_src).render(render_data) # render template variables, ideally w/o global context
except Exception as err:
logging.info("Warning: could not render js in custom_reports, err=%s" % str(err))
js = JINJA_ENVIRONMENT.from_string(js_src).render(render_data)
#template = JINJA_ENVIRONMENT.from_string(html)
# logging.info("get_report_html name=%s, parameters=%s" % (report_name, parameters))
data = {'html': html + js,
'js': crm.javascript,
}
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(data))
def get_main(self):
'''
Main page: show list of all courses
'''
courses = self.get_course_listings()
# logging.info('course listings: %s' % courses['data'])
def add_link(xstr, row=None):
'''add link to course page to table element'''
cid = row.get('course_id', None)
if cid:
return "<a href=/course/%s>%s</a>" % (cid, xstr)
else:
return xstr
html = self.list2table(map(DataTableField,
[{'field': 'launch', 'title':'Course launch', 'width': '12%'},
{'field': 'course_number', 'title': 'Course #', 'width': '5%'},
{'field': 'course_image', 'title': 'Course image'},
{'field': 'title', 'title': 'Course Title', 'width': '40%'},
{'field': 'course_id', 'title': 'course ID', 'width': '12%'},
]),
courses['data'],
eformat={'course_number': add_link, 'title': add_link},
)
data = self.common_data
data.update({'data': {},
'is_staff': self.is_superuser(),
'is_pm': self.is_pm(),
'table': html,
})
template = JINJA_ENVIRONMENT.get_template('courses.html')
self.response.out.write(template.render(data))
def get_custom_report_page(self, report_name):
'''
Single custom report which behaves as a HTML page
'''
crm, pdata, auth_ok, msg = self.custom_report_auth_check(
report_name) # crm = CourseReport model
if not auth_ok:
return self.no_auth_sorry()
if not crm:
self.response.write(msg)
return
html = crm.html
html += "<script type='text/javascript'>"
html += "$(document).ready( function () {%s} );" % crm.javascript # js goes in html, and thus gets template vars rendered
html += "</script>"
template = JINJA_ENVIRONMENT.from_string(html)
parameters = {x: v for x, v in pdata.items() if v is not None}
parameters['orgname'] = self.ORGNAME
parameters['feature_flags'] = self.FEATURE_FLAGS
render_data = self.common_data.copy()
if crm.meta_info.get('need_tags'):
render_data['course_tags'] = self.get_course_listings_tags()
if crm.meta_info.get('need_listings'):
render_data['course_listings'] = self.get_course_listings()
if ('course_id' in pdata) and pdata['course_id']:
render_data['base'] = self.get_base(pdata['course_id'])
logging.info('[page] base=%s' % render_data['base'])
render_data.update({
'report_name':
report_name,
'parameters':
json.dumps(parameters), # for js
'parameter_values':
parameters, # for html template variables
'custom_report':
self.custom_report_container(
self.is_authorized_for_custom_report,
**pdata), # pass pdata so children of page also get parameters
'is_staff':
self.is_superuser(),
'is_pm':
self.is_pm(),
'msg':
msg,
'nav_is_active':
self.nav_is_active(report_name),
'cr_js_library':
self.get_custom_report_library_javascript(),
})
render_data.update(pdata)
self.response.out.write(template.render(render_data))
def get_main(self):
'''
Main page: show list of all courses
'''
courses = self.get_course_listings()
# logging.info('course listings: %s' % courses['data'])
def add_link(xstr, row=None):
'''add link to course page to table element'''
cid = row.get('course_id', None)
if cid:
return "<a href=/course/%s>%s</a>" % (cid, xstr)
else:
return xstr
html = self.list2table(
map(DataTableField, [
{
'field': 'launch',
'title': 'Course launch',
'width': '12%'
},
{
'field': 'course_number',
'title': 'Course #',
'width': '5%'
},
{
'field': 'course_image',
'title': 'Course image'
},
{
'field': 'title',
'title': 'Course Title',
'width': '40%'
},
{
'field': 'course_id',
'title': 'course ID',
'width': '12%'
},
]),
courses['data'],
eformat={
'course_number': add_link,
'title': add_link
},
)
data = self.common_data
data.update({
'data': {},
'is_staff': self.is_superuser(),
'is_pm': self.is_pm(),
'table': html,
})
template = JINJA_ENVIRONMENT.get_template('courses.html')
self.response.out.write(template.render(data))
def get_video(self, **kwargs):
'''
single video analytics view
- iframe
'''
data = self.setup_module_info(**kwargs)
template = JINJA_ENVIRONMENT.get_template('video.html')
self.response.out.write(template.render(data))
def get_video(self, **kwargs):
'''
single video analytics view
- iframe
'''
data = self.setup_module_info(**kwargs)
template = JINJA_ENVIRONMENT.get_template('video.html')
self.response.out.write(template.render(data))
def get_axis(self, org=None, number=None, semester=None):
'''
show full course axis -- mainly for debugging
'''
course_id = '/'.join([org, number, semester])
if not self.is_user_authorized_for_course(course_id):
return self.no_auth_sorry()
caxis = self.load_course_axis(course_id, dtype='data')
if self.request.get('ajax'):
# return JSON instead of HTML
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(self.course_axis, default=self.json_serializer_with_datetime, indent=4))
return
if self.request.get('chapters'):
# return JSON of just chapters
chapters = [x for x in caxis if x['category']=='chapter']
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(chapters, indent=4))
return
# logging.info("caxis=%s" % json.dumps(caxis, indent=4))
for cae in caxis:
try:
# caxis[row]['name'] = fix_bad_unicode(caxis[row]['name'])
#caxis[row]['name'] = caxis[row]['name'].replace('\u2013','-')
#caxis[row]['name'] = str(caxis[row]['name'])
cae['name'] = unidecode(cae['name']) # desparation: perhaps data wasn't encoded properly originally?
if cae['gformat']:
cae['gformat'] = unidecode(cae['gformat']) # desparation: perhaps data wasn't encoded properly originally?
# cae['name'] = str(cae['name'])
except Exception as err:
print "unicode error for course axis row=%s, name=" % repr(cae), repr(cae['name'])
print "type = ", type(cae['name'])
raise
if 1:
fields = ['category', 'index', 'url_name', 'name', 'gformat', 'due', 'start',
'module_id', 'path', 'data_ytid', 'data_weight', 'chapter_mid']
#fields = ['category', 'index', 'name', 'due', 'start',
# 'module_id', 'path', 'data_ytid', 'data_weight', 'chapter_mid']
tablehtml = self.list2table(fields,
caxis,
eformat={'due': self.fix_date, 'start': self.fix_date}, )
data = self.common_data
data.update({'course_id': course_id,
'table': tablehtml,
})
template = JINJA_ENVIRONMENT.get_template('course_axis.html')
self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
self.response.out.write(template.render(data))
def get_glossary(self, type=None):
'''
Show Glossary data
'''
data = ''
template = JINJA_ENVIRONMENT.get_template('glossary/glossary_%s.html' % type)
self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
self.response.out.write(template.render(data))
def render_user_items(user):
user_data = User.get_user(user)
items = ndb.get_multi(user_data['items'])
template = JINJA_ENVIRONMENT.get_template('templates/items.html')
context = {'items': [i.to_dict() if i else {} for i in items]}
result = template.render(context)
return result
def get_glossary(self, type=None):
'''
Show Glossary data
'''
data = ''
template = JINJA_ENVIRONMENT.get_template('glossary/glossary_%s.html' %
type)
self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
self.response.out.write(template.render(data))
def get_developer(self):
"""
Dashboard page: show cross-course comparisons
"""
if not self.user in self.AUTHORIZED_USERS: # require superuser
return self.no_auth_sorry()
data = self.common_data
data.update({"is_staff": self.is_superuser()})
template = JINJA_ENVIRONMENT.get_template("dev-editor.html")
self.response.out.write(template.render(data))
def get_problem(self, **kwargs):
'''
single problem analytics view
- iframe
'''
data = self.setup_module_info(**kwargs)
# template = os.path.join(os.path.dirname(__file__), 'problem.html')
template = JINJA_ENVIRONMENT.get_template('problem.html')
self.response.out.write(template.render(data))
def get(self):
logging.info('overview')
template = JINJA_ENVIRONMENT.get_template('templates/jobs.html')
context = {}
context['descriptions'] = get_cron_descriptions()
context['jobs'] = get_cron_jobs()
context['laststatus'] = {}
if self.request.get('success'):
context['laststatus']['success'] = int(self.request.get('success'))
self.response.headers['Cache-Control'] = 'no-cache'
self.response.write(template.render(context))
def get_developer(self):
'''
Dashboard page: show cross-course comparisons
'''
if not self.user in self.AUTHORIZED_USERS: # require superuser
return self.no_auth_sorry()
data = self.common_data
data.update({
'is_staff': self.is_superuser(),
})
template = JINJA_ENVIRONMENT.get_template('dev-editor.html')
self.response.out.write(template.render(data))
def get_course(self, org=None, number=None, semester=None):
'''
single course analytics view
- overall statistics (number of components of various categories)
- show table of chapters
'''
course_id = '/'.join([org, number, semester])
# handle forced recomputation requests
action = self.request.POST.get('action', None)
logging.info('post keys = %s' % self.request.POST.keys())
logging.info('post action = %s' % action)
if action=='force recompute enrollment':
self.reset_enrollment_by_day(course_id)
# show table with just chapters, and present sequentials as extra information when clicked
fields = [ DataTableField(x) for x in [{'field': 'index', 'title': 'Time index', 'width': '8%', 'class': 'dt-center'},
{'field': 'name', 'title': "Chapter name"},
{'field': 'start', 'title': "Start date", 'width': '18%'},
{'field': 'nuser_views', 'title': '# user-views', 'width': '10%', 'class': 'dt-center'},
] ]
# logging.info('sm_usage:')
# logging.info(self.bqdata['stats_module_usage']['data_by_key'])
tablehtml = self.list2table([' '] + fields, [])
tablefields = json.dumps([
{
"class": 'details-control',
"orderable": False,
"data": None,
'width': '5%',
"defaultContent": ''
},] + [x.colinfo() for x in fields])
data = self.common_data.copy()
data.update({'course_id': course_id,
'fields': tablefields,
'table': tablehtml,
'is_staff': self.is_superuser(),
'is_pm': self.is_pm(),
'does_user_have_role': self.does_user_have_role,
'image': self.get_course_image(course_id),
'nav_is_active': self.nav_is_active('onecourse'),
'custom_report': self.custom_report_container(self.is_authorized_for_custom_report,
course_id=course_id),
})
template = JINJA_ENVIRONMENT.get_template('one_course.html')
self.response.out.write(template.render(data))
def get_table(self,
dataset=None,
table=None,
org=None,
number=None,
semester=None):
'''
show arbitrary table from bigquery -- mainly for debugging
'''
if dataset is None:
course_id = '/'.join([org, number, semester])
dataset = bqutil.course_id2dataset(
course_id, use_dataset_latest=self.use_dataset_latest())
if not self.is_user_authorized_for_course(course_id):
return self.no_auth_sorry()
if ('person' in table) or ('track' in table) or ('student'
in table):
if not self.does_user_have_role('instructor', course_id):
return self.no_auth_sorry()
# be more restrictive: researchers only
if not (self.does_user_have_role('researcher', course_id)):
return self.no_auth_sorry()
else:
course_id = None
if not self.user in self.AUTHORIZED_USERS:
return self.no_auth_sorry()
tableinfo = bqutil.get_bq_table_info(dataset, table)
fields = tableinfo['schema']['fields']
field_names = [x['name'] for x in fields]
tablecolumns = json.dumps([{
'data': x,
'title': x,
'class': 'dt-center'
} for x in field_names])
logging.info(tablecolumns)
data = self.common_data
data.update({
'dataset': dataset,
'table': table,
'course_id': course_id,
'tablecolumns': tablecolumns,
})
template = JINJA_ENVIRONMENT.get_template('show_table.html')
self.response.out.write(template.render(data))
def get_custom_report_page(self, report_name):
'''
Single custom report which behaves as a HTML page
'''
crm, pdata, auth_ok, msg = self.custom_report_auth_check(report_name) # crm = CourseReport model
if not auth_ok:
return self.no_auth_sorry()
if not crm:
self.response.write(msg)
return
html = crm.html
html += "<script type='text/javascript'>"
html += "$(document).ready( function () {%s} );" % crm.javascript # js goes in html, and thus gets template vars rendered
html += "</script>"
template = JINJA_ENVIRONMENT.from_string(html)
parameters = {x:v for x,v in pdata.items() if v is not None}
parameters['orgname'] = self.ORGNAME
parameters['feature_flags'] = self.FEATURE_FLAGS
render_data = self.common_data.copy()
if crm.meta_info.get('need_tags'):
render_data['course_tags'] = self.get_course_listings_tags()
if crm.meta_info.get('need_listings'):
render_data['course_listings'] = self.get_course_listings()
if ('course_id' in pdata) and pdata['course_id']:
render_data['base'] = self.get_base(pdata['course_id'])
logging.info('[page] base=%s' % render_data['base'])
render_data.update({'report_name': report_name,
'parameters': json.dumps(parameters), # for js
'parameter_values': parameters, # for html template variables
'custom_report': self.custom_report_container(self.is_authorized_for_custom_report,
**pdata), # pass pdata so children of page also get parameters
'is_staff': self.is_superuser(),
'is_pm': self.is_pm(),
'msg': msg,
'nav_is_active': self.nav_is_active(report_name),
'cr_js_library': self.get_custom_report_library_javascript(),
})
render_data.update(pdata)
self.response.out.write(template.render(render_data))
class ProcedureBase(object):
xmpp_template = JINJA_ENVIRONMENT.get_template(
'templates/proc_base.xmpp.template')
@classmethod
def do_work_core(cls, args):
"""return ((title, content)...)"""
raise NotImplemented
@classmethod
def render_msg(cls, args, new_items):
return cls.xmpp_template.render({
'message': [i[0] for i in new_items],
'name': args['__name__']
})
@classmethod
def do_work(cls, args):
logging.debug('%s working with %s' % (cls, args))
result = []
data = cls.do_work_core(args)
for title, content in data:
ret = ProcItem.add_item(cls.__module__ + '.' + cls.__name__, title,
content)
if ret:
content['title'] = title
result.append(content)
else:
logging.debug('store item fail: %s' % title)
if 'xmpp' in args and args['xmpp'].strip():
msg = cls.render_msg(args, result)
if msg: xmpp.send_message([args['xmpp']], msg)
if 'post' in args:
#ret = requests.post(args['post'], json={'items': result})
ret = urlfetch.fetch(args['post'],
payload=json.dumps(result),
method=urlfetch.POST)
if ret.status_code != 200:
logging.warn('data post failed: %d' % ret.status_code)
return result
def get_problem(self, **kwargs):
'''
single problem analytics view
- iframe
'''
data = self.setup_module_info(**kwargs)
data.update({'custom_report': self.custom_report_container(self.is_authorized_for_custom_report,
course_id=data['course_id'],
chapter_id=data['chapter_mid'],
module_id=data['module_id'],
),
})
# template = os.path.join(os.path.dirname(__file__), 'problem.html')
template = JINJA_ENVIRONMENT.get_template('problem.html')
self.response.out.write(template.render(data))
def get_dashboard(self):
'''
Dashboard page: show cross-course comparisons
'''
courses = self.get_course_listings()
data = self.common_data.copy()
html = self.list2table(['Registration Open', 'course_id', 'title', 'Course Launch', 'Course Wrap', 'New or Rerun', 'Instructors'],
courses['data'])
data.update({'is_staff': self.is_superuser(),
'courses': courses,
'table': html,
'ncourses': len(courses['data']),
'custom_report': self.custom_report_container(self.is_authorized_for_custom_report),
'nav_is_active': self.nav_is_active('allcourse'),
'is_pm': self.is_pm(),
})
logging.info('session: %s' % dict(self.session))
logging.info('================================================== dataset_latest=%s' % self.use_dataset_latest())
template = JINJA_ENVIRONMENT.get_template('dashboard.html')
self.response.out.write(template.render(data))
def get_dashboard(self):
'''
Dashboard page: show cross-course comparisons
'''
courses = self.get_course_listings()
data = self.common_data.copy()
html = self.list2table(['Registration Open', 'course_id', 'title', 'Course Launch', 'Course Wrap', 'New or Rerun', 'Instructors'],
courses['data'])
data.update({'is_staff': self.is_superuser(),
'courses': courses,
'table': html,
'ncourses': len(courses['data']),
'custom_report': self.custom_report_container(self.is_authorized_for_custom_report),
'nav_is_active': self.nav_is_active('allcourse'),
'is_pm': self.is_pm(),
})
logging.info('session: %s' % dict(self.session))
logging.info('================================================== dataset_latest=%s' % self.use_dataset_latest())
template = JINJA_ENVIRONMENT.get_template('dashboard.html')
self.response.out.write(template.render(data))
def get_chapter(self, org=None, number=None, semester=None, url_name=None):
'''
single chapter analytics view: container for table data
- sequentials and problems
'''
course_id = '/'.join([org, number, semester])
caxis = self.load_course_axis(course_id)
# get chapter info
the_chapter = caxis[url_name]
chapter_mid = the_chapter['module_id']
chapter_name = the_chapter['name']
fields = [ DataTableField(x) for x in [{'field': 'index', 'title': 'Time index', 'width': '8%', 'class': 'dt-center'},
{'field': 'category', 'title': "Module category", 'width': '10%'},
{'field': 'name', 'title': "Module name"},
{'field': 'nsubmissions', 'title': "# submissions", 'width': '7%', 'class': 'dt-center'},
{'field': 'avg_grade', 'title': "AVG grade", 'width': '7%', 'class': 'dt-center'},
{'field': 'max_grade', 'title': "MAX grade", 'width': '7%', 'class': 'dt-center'},
{'field': 'avg_attempts', 'title': "AVG attempts", 'width': '7%', 'class': 'dt-center'},
{'field': 'max_attempts', 'title': "MAX attempts", 'width': '7%', 'class': 'dt-center'},
{'field': 'start', 'title': "Start date", 'width': '12%', 'class': 'dt-center'},
{'field': 'nuser_views', 'title': '# user views', 'width': '7%', 'class': 'dt-center'},
# {'field': 'url_name', 'title': 'url_name'},
] ]
tablehtml = self.list2table(fields, [])
tablefields = json.dumps([x.colinfo() for x in fields])
data = self.common_data
data.update({'fields': tablefields,
'table': tablehtml,
'course_id': course_id,
'chapter_name': chapter_name,
'url_name': url_name,
})
template = JINJA_ENVIRONMENT.get_template('chapter.html')
self.response.out.write(template.render(data))
def get_table(self, dataset=None, table=None, org=None, number=None,semester=None):
'''
show arbitrary table from bigquery -- mainly for debugging
'''
if dataset is None:
course_id = '/'.join([org, number, semester])
dataset = bqutil.course_id2dataset(course_id, use_dataset_latest=self.use_dataset_latest())
if not self.is_user_authorized_for_course(course_id):
return self.no_auth_sorry()
if ('person' in table) or ('track' in table) or ('student' in table):
if not self.does_user_have_role('instructor', course_id):
return self.no_auth_sorry()
# be more restrictive: researchers only
if not (self.does_user_have_role('researcher', course_id)):
return self.no_auth_sorry()
else:
course_id = None
if not self.user in self.AUTHORIZED_USERS:
return self.no_auth_sorry()
tableinfo = bqutil.get_bq_table_info(dataset, table)
fields = tableinfo['schema']['fields']
field_names = [x['name'] for x in fields]
tablecolumns = json.dumps([ { 'data': x, 'title': x, 'class': 'dt-center' } for x in field_names ])
logging.info(tablecolumns)
data = self.common_data
data.update({'dataset': dataset,
'table': table,
'course_id': course_id,
'tablecolumns': tablecolumns,
})
template = JINJA_ENVIRONMENT.get_template('show_table.html')
self.response.out.write(template.render(data))
def get_problem(self, **kwargs):
'''
single problem analytics view
- iframe
'''
data = self.setup_module_info(**kwargs)
data.update({
'custom_report':
self.custom_report_container(
self.is_authorized_for_custom_report,
course_id=data['course_id'],
chapter_id=data['chapter_mid'],
module_id=data['module_id'],
),
})
# template = os.path.join(os.path.dirname(__file__), 'problem.html')
template = JINJA_ENVIRONMENT.get_template('problem.html')
self.response.out.write(template.render(data))
def get_admin(self):
'''
Admin page: show authorized users, clear cache
'''
if not self.user in self.AUTHORIZED_USERS: # require superuser
return self.no_auth_sorry()
msg = ""
action = self.request.POST.get('action', None)
custom_reports_standard_source_dir = "ANALYTICS_STANDARD_REPORTS"
custom_reports_standard_source_file = "ANALYTICS_STANDARD_REPORTS.yaml"
crssd = 'data/%s' % custom_reports_standard_source_dir
crssf = 'data/%s' % custom_reports_standard_source_file
if os.path.exists(crssd):
custom_reports_standard_source = custom_reports_standard_source_dir
else:
custom_reports_standard_source = custom_reports_standard_source_file
try:
custom_reports_standard_source = local_config.CUSTOM_REPORTS_SOURCE
except Exception as err:
pass
if action=='Flush cache':
memcache.flush_all()
msg = "Cache flushed"
elif action=='Reload staff table':
self.get_staff_table(reload=True)
msg = "Staff table reloaded"
elif action=='Reload course listings':
self.get_course_listings(ignore_cache=True)
msg = "Course listings reloaded"
elif action=='List current course tags':
tags = self.get_course_listings_tags()
msg = "Current course tags = %s" % tags
elif action=='Check access':
username = self.request.get('username')
course_id = self.request.get('course_id')
if self.is_user_authorized_for_course(course_id=course_id, user=username):
msg = "User %s IS authorized for %s" % (username, course_id)
else:
msg = "User %s is NOT authorized for %s" % (username, course_id)
elif action=='Reload Course Listings':
collection = self.request.POST.get('collection')
self.get_course_listings(ignore_cache=True, collection=collection)
msg = "Course listings for '%s' reloaded" % collection
elif action=='Reload Standard Reports':
crssd = 'data/%s' % custom_reports_standard_source
if os.path.exists(crssd):
if os.path.isdir(crssd):
files = glob.glob('%s/*.yaml' % crssd)
elif os.path.isfile(crssd):
files = [crssd]
msg = "<ul>"
for fn in files:
msg += "<li>Standard Reports loading from %s<br/>" % (fn)
report_file_data = open(fn).read()
try:
msg += self.import_custom_report_from_file_data(report_file_data, overwrite=True)
except Exception as err:
logging.error("Oops! Error importing custom report from %s" % fn)
raise
msg += "</li>"
msg += "</ul>"
else:
msg = "Error: cannot find file or directory %s" % crssd
elif action=='Reload Custom Reports':
collection = self.request.POST.get('collection')
cnt = self.import_custom_report_metadata(ignore_cache=True, collection=collection)
msg = "Custom Report Metadata for '%s' reloaded (%d reports)" % (collection, cnt)
elif action=='Export Custom Reports':
collection = self.request.POST.get('collection')
cnt, destination = self.export_custom_report_metadata(ignore_cache=True, collection=collection)
msg = "Custom Report Metadata for '%s' exported to %s (%d reports)" % (collection, destination, cnt)
elif action=='Add staff':
fields = ['username', 'role', 'course_id', 'notes']
data = { x: (self.request.POST.get(x) or '') for x in fields }
self.add_staff_table_entry(data)
msg = "New staff %s added" % data
todelete = self.request.POST.get('do-delete', None)
if todelete is not None:
self.disable_staff_table_entry(int(todelete))
msg = "Deleted staff table row %s" % todelete
stable = self.get_staff_table()
stafftable = self.list2table([DataTableField({'icon':'delete', 'field': 'sid', 'title':' '}),
'username', 'role', 'course_id', 'notes'], stable)
data = self.common_data.copy()
course_listings_source = self.get_collection_metadata('COURSE_LISTINGS_TABLE')
data.update({'superusers': self.AUTHORIZED_USERS,
'table': stafftable,
'msg': msg,
'listings_source': course_listings_source,
'staff_source': local_config.STAFF_COURSE_TABLE,
'collections': self.collections_available(asdict=True),
'custom_reports_standard_source': custom_reports_standard_source,
})
template = JINJA_ENVIRONMENT.get_template('admin.html')
self.response.out.write(template.render(data))
def get_axis(self, org=None, number=None, semester=None):
'''
show full course axis -- mainly for debugging
'''
course_id = '/'.join([org, number, semester])
if not self.is_user_authorized_for_course(course_id):
return self.no_auth_sorry()
caxis = self.load_course_axis(course_id, dtype='data')
if self.request.get('ajax'):
# return JSON instead of HTML
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(
json.dumps(self.course_axis,
default=self.json_serializer_with_datetime,
indent=4))
return
if self.request.get('chapters'):
# return JSON of just chapters
chapters = [x for x in caxis if x['category'] == 'chapter']
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(chapters, indent=4))
return
# logging.info("caxis=%s" % json.dumps(caxis, indent=4))
for cae in caxis:
try:
# caxis[row]['name'] = fix_bad_unicode(caxis[row]['name'])
#caxis[row]['name'] = caxis[row]['name'].replace('\u2013','-')
#caxis[row]['name'] = str(caxis[row]['name'])
cae['name'] = unidecode(
cae['name']
) # desparation: perhaps data wasn't encoded properly originally?
if cae['gformat']:
cae['gformat'] = unidecode(
cae['gformat']
) # desparation: perhaps data wasn't encoded properly originally?
# cae['name'] = str(cae['name'])
except Exception as err:
print "unicode error for course axis row=%s, name=" % repr(
cae), repr(cae['name'])
print "type = ", type(cae['name'])
raise
if 1:
fields = [
'category', 'index', 'url_name', 'name', 'gformat', 'due',
'start', 'module_id', 'path', 'data_ytid', 'data_weight',
'chapter_mid'
]
#fields = ['category', 'index', 'name', 'due', 'start',
# 'module_id', 'path', 'data_ytid', 'data_weight', 'chapter_mid']
tablehtml = self.list2table(
fields,
caxis,
eformat={
'due': self.fix_date,
'start': self.fix_date
},
)
data = self.common_data
data.update({
'course_id': course_id,
'table': tablehtml,
})
template = JINJA_ENVIRONMENT.get_template('course_axis.html')
self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
self.response.out.write(template.render(data))
def get_chapter(self, org=None, number=None, semester=None, url_name=None, seq=None):
'''
single chapter analytics view: container for table data
- sequentials and problems
'''
course_id = '/'.join([org, number, semester])
caxis = self.load_course_axis(course_id)
# get chapter info
the_chapter = caxis[url_name]
# if seq is specified, then look for the chapter before or after, as requested
error = None
if seq in ["next", "prev"]:
chapters = [x for x in caxis.values() if (x['category']=='chapter')]
chapters.sort(cmp=lambda x,y: int(x['index'])-int(y['index'])) # list of chapters ordered by course axis index
try:
cidx = chapters.index(the_chapter)
except Exception as err:
error = {'msg': "Module %s not found in list of chapters %s" % (url_name, chapters)}
cidx = None
if (cidx is not None) and seq=='next':
if cidx < len(chapters)-1:
the_chapter = chapters[cidx+1]
module_id = the_chapter['module_id']
url_name = module_id.rsplit('/',1)[-1]
else:
error = {'msg': "No chapter available after %s in this course" % (url_name)}
if (cidx is not None) and seq=='prev':
if cidx > 0:
the_chapter = chapters[cidx-1]
module_id = the_chapter['module_id']
url_name = module_id.rsplit('/',1)[-1]
else:
error = {'msg': "No chapter available before %s in this course" % (url_name)}
chapter_mid = the_chapter['module_id']
chapter_name = the_chapter['name']
fields = [ DataTableField(x) for x in [{'field': 'index', 'title': 'Time index', 'width': '8%', 'class': 'dt-center'},
{'field': 'category', 'title': "Module category", 'width': '10%'},
{'field': 'name', 'title': "Module name"},
{'field': 'nsubmissions', 'title': "# submissions", 'width': '7%', 'class': 'dt-center'},
{'field': 'avg_grade', 'title': "AVG grade", 'width': '7%', 'class': 'dt-center'},
{'field': 'max_grade', 'title': "MAX grade", 'width': '7%', 'class': 'dt-center'},
{'field': 'avg_attempts', 'title': "AVG attempts", 'width': '7%', 'class': 'dt-center'},
{'field': 'max_attempts', 'title': "MAX attempts", 'width': '7%', 'class': 'dt-center'},
{'field': 'start', 'title': "Start date", 'width': '12%', 'class': 'dt-center'},
{'field': 'nuser_views', 'title': '# user views', 'width': '7%', 'class': 'dt-center'},
# {'field': 'url_name', 'title': 'url_name'},
] ]
tablehtml = self.list2table(fields, [])
tablefields = json.dumps([x.colinfo() for x in fields])
data = self.common_data
data.update({'fields': tablefields,
'table': tablehtml,
'course_id': course_id,
'chapter_name': chapter_name,
'url_name': url_name,
'error': error,
})
template = JINJA_ENVIRONMENT.get_template('chapter.html')
self.response.out.write(template.render(data))
def get_course(self, org=None, number=None, semester=None):
'''
single course analytics view
- overall statistics (number of components of various categories)
- show table of chapters
'''
course_id = '/'.join([org, number, semester])
# handle forced recomputation requests
action = self.request.POST.get('action', None)
logging.info('post keys = %s' % self.request.POST.keys())
logging.info('post action = %s' % action)
if action == 'force recompute enrollment':
self.reset_enrollment_by_day(course_id)
# show table with just chapters, and present sequentials as extra information when clicked
fields = [
DataTableField(x) for x in [
{
'field': 'index',
'title': 'Time index',
'width': '8%',
'class': 'dt-center'
},
{
'field': 'name',
'title': "Chapter name"
},
{
'field': 'start',
'title': "Start date",
'width': '18%'
},
{
'field': 'nuser_views',
'title': '# user-views',
'width': '10%',
'class': 'dt-center'
},
]
]
# logging.info('sm_usage:')
# logging.info(self.bqdata['stats_module_usage']['data_by_key'])
tablehtml = self.list2table([' '] + fields, [])
tablefields = json.dumps([
{
"class": 'details-control',
"orderable": False,
"data": None,
'width': '5%',
"defaultContent": ''
},
] + [x.colinfo() for x in fields])
data = self.common_data.copy()
data.update({
'course_id':
course_id,
'fields':
tablefields,
'table':
tablehtml,
'is_staff':
self.is_superuser(),
'is_pm':
self.is_pm(),
'does_user_have_role':
self.does_user_have_role,
'image':
self.get_course_image(course_id),
'nav_is_active':
self.nav_is_active('onecourse'),
'custom_report':
self.custom_report_container(self.is_authorized_for_custom_report,
course_id=course_id),
})
template = JINJA_ENVIRONMENT.get_template('one_course.html')
self.response.out.write(template.render(data))
def get_chapter(self,
org=None,
number=None,
semester=None,
url_name=None,
seq=None):
'''
single chapter analytics view: container for table data
- sequentials and problems
'''
course_id = '/'.join([org, number, semester])
caxis = self.load_course_axis(course_id)
# get chapter info
the_chapter = caxis[url_name]
# if seq is specified, then look for the chapter before or after, as requested
error = None
if seq in ["next", "prev"]:
chapters = [
x for x in caxis.values() if (x['category'] == 'chapter')
]
chapters.sort(cmp=lambda x, y: int(x['index']) - int(y['index'])
) # list of chapters ordered by course axis index
try:
cidx = chapters.index(the_chapter)
except Exception as err:
error = {
'msg':
"Module %s not found in list of chapters %s" %
(url_name, chapters)
}
cidx = None
if (cidx is not None) and seq == 'next':
if cidx < len(chapters) - 1:
the_chapter = chapters[cidx + 1]
module_id = the_chapter['module_id']
url_name = module_id.rsplit('/', 1)[-1]
else:
error = {
'msg':
"No chapter available after %s in this course" %
(url_name)
}
if (cidx is not None) and seq == 'prev':
if cidx > 0:
the_chapter = chapters[cidx - 1]
module_id = the_chapter['module_id']
url_name = module_id.rsplit('/', 1)[-1]
else:
error = {
'msg':
"No chapter available before %s in this course" %
(url_name)
}
chapter_mid = the_chapter['module_id']
chapter_name = the_chapter['name']
fields = [
DataTableField(x) for x in [
{
'field': 'index',
'title': 'Time index',
'width': '8%',
'class': 'dt-center'
},
{
'field': 'category',
'title': "Module category",
'width': '10%'
},
{
'field': 'name',
'title': "Module name"
},
{
'field': 'nsubmissions',
'title': "# submissions",
'width': '7%',
'class': 'dt-center'
},
{
'field': 'avg_grade',
'title': "AVG grade",
'width': '7%',
'class': 'dt-center'
},
{
'field': 'max_grade',
'title': "MAX grade",
'width': '7%',
'class': 'dt-center'
},
{
'field': 'avg_attempts',
'title': "AVG attempts",
'width': '7%',
'class': 'dt-center'
},
{
'field': 'max_attempts',
'title': "MAX attempts",
'width': '7%',
'class': 'dt-center'
},
{
'field': 'start',
'title': "Start date",
'width': '12%',
'class': 'dt-center'
},
{
'field': 'nuser_views',
'title': '# user views',
'width': '7%',
'class': 'dt-center'
},
# {'field': 'url_name', 'title': 'url_name'},
]
]
tablehtml = self.list2table(fields, [])
tablefields = json.dumps([x.colinfo() for x in fields])
data = self.common_data
data.update({
'fields': tablefields,
'table': tablehtml,
'course_id': course_id,
'chapter_name': chapter_name,
'url_name': url_name,
'error': error,
})
template = JINJA_ENVIRONMENT.get_template('chapter.html')
self.response.out.write(template.render(data))
def edit_custom_report(self, report_name, crm=None):
'''
edit custom report html, javascript, sql, description, ...
'''
if not self.user in self.AUTHORIZED_USERS: # require superuser
return self.no_auth_sorry()
parameter_values = self.session.get('edit_report_parameter_values')
if not parameter_values or parameter_values == "None":
parameter_values = {}
# self.session['edit_report_parameter_values'] = parameter_values
editor_heights = self.request.POST.get('editor_heights')
if editor_heights:
self.session[
'editor_heights'] = editor_heights # keep editor heights for session
else:
editor_heights = self.session.get('editor_heights')
logging.info('[custom_reports] editor heights=%s' % editor_heights)
msg = ''
if (self.request.POST.get('action') == 'Download Report'):
try:
data = self.export_custom_report_metadata(
report_name=report_name, download=True)
dump = yaml.dump(data,
default_style="|",
default_flow_style=False)
# logging.info("custom report yaml=%s" % dump)
except Exception as err:
logging.error("Failed to find custom report named %s!" %
report_name)
raise
self.response.headers['Content-Type'] = 'application/text'
self.response.headers[
'Content-Disposition'] = 'attachment; filename=ANALYTICS_REPORT_%s.yaml' % report_name
self.response.out.write(dump)
return
elif (self.request.POST.get('action') == 'Download ALL Reports'):
try:
data = self.export_custom_report_metadata(report_name=None,
download=True)
dump = yaml.dump(data,
default_style="|",
default_flow_style=False)
logging.info("custom report yaml=%s" % dump)
except Exception as err:
raise
dtstr = self.TheNow().strftime('%Y-%m-%d_%H%M')
self.response.headers['Content-Type'] = 'application/text'
self.response.headers[
'Content-Disposition'] = 'attachment; filename=ANALYTICS_ALL_REPORTS_%s.yaml' % dtstr
self.response.out.write(dump)
return
elif (self.request.POST.get('action') == 'Delete Report'):
try:
crm = self.get_custom_report_metadata(report_name)
except Exception as err:
logging.error("Failed to find custom report named %s!" %
report_name)
raise
crm.key.delete()
msg = "Deleted course report %s" % report_name
return self.get_custom_report(msg=msg)
elif (self.request.POST.get('action') == 'Save Changes'):
fields = [
'table_name', 'title', 'depends_on', 'html', 'sql',
'javascript', 'description', 'collection', 'group_tags',
'meta_info'
]
try:
crm = self.get_custom_report_metadata(report_name)
except Exception as err:
logging.error("Failed to find custom report named %s!" %
report_name)
raise
for field in fields:
fval = self.request.POST.get(field)
if field == 'group_tags':
fval = [x.strip() for x in fval.split(',')]
elif field == 'meta_info':
fval = eval(fval) or {}
if fval is None:
logging.error(
"oops, expected value for field=%s, but got fval=%s" %
(field, fval))
else:
setattr(crm, field, fval)
crm.put()
logging.info('saved crm = %s' % crm)
msg = "Saved custom report %s" % report_name
if not crm.table_name:
msg += "...Warning! table_name cannot be left empty"
try:
if not crm:
crm = self.get_custom_report_metadata(report_name)
except Exception as err:
logging.error("Cannot get custom report %s" % report_name)
raise
if not crm:
msg = "Cannot get custom report %s" % report_name
logging.error(msg)
raise Exception(msg)
# backward compatability: to accommodate jhints, ignore lines in javascript with jinja2 template commands
if ('{{parameters}}'
in crm.javascript) and not ('jshint ignore' in crm.javascript):
newjs = []
for k in crm.javascript.split('\n'):
if (("{% autoescape" in k)) and not ('jshint ignore' in k):
k += " // jshint ignore:line"
elif ('{{parameters}}' in k) and not ('jshint ignore' in k):
newjs.append("/* jshint ignore:start */")
newjs.append(k)
k = "/* jshint ignore:end */"
newjs.append(k)
crm.javascript = '\n'.join(newjs)
data = {
'html': crm.html,
'js': crm.javascript,
'report_name': report_name,
'report': crm,
'msg': msg,
'parameter_values': parameter_values,
'meta_info': json.dumps(crm.meta_info),
'editor_heights': editor_heights or "{}",
'cr_js_library': self.get_custom_report_library_javascript(),
}
data.update(self.common_data)
template = JINJA_ENVIRONMENT.get_template('edit_custom_report.html')
self.response.out.write(template.render(data))
def edit_custom_report(self, report_name, crm=None):
'''
edit custom report html, javascript, sql, description, ...
'''
if not self.user in self.AUTHORIZED_USERS: # require superuser
return self.no_auth_sorry()
parameter_values = self.session.get('edit_report_parameter_values')
if not parameter_values or parameter_values=="None":
parameter_values = {}
# self.session['edit_report_parameter_values'] = parameter_values
editor_heights = self.request.POST.get('editor_heights')
if editor_heights:
self.session['editor_heights'] = editor_heights # keep editor heights for session
else:
editor_heights = self.session.get('editor_heights')
logging.info('[custom_reports] editor heights=%s' % editor_heights)
msg = ''
if (self.request.POST.get('action')=='Download Report'):
try:
data = self.export_custom_report_metadata(report_name=report_name, download=True)
dump = yaml.dump(data, default_style="|", default_flow_style=False)
# logging.info("custom report yaml=%s" % dump)
except Exception as err:
logging.error("Failed to find custom report named %s!" % report_name)
raise
self.response.headers['Content-Type'] = 'application/text'
self.response.headers['Content-Disposition'] = 'attachment; filename=ANALYTICS_REPORT_%s.yaml' % report_name
self.response.out.write(dump)
return
elif (self.request.POST.get('action')=='Download ALL Reports'):
try:
data = self.export_custom_report_metadata(report_name=None, download=True)
dump = yaml.dump(data, default_style="|", default_flow_style=False)
logging.info("custom report yaml=%s" % dump)
except Exception as err:
raise
dtstr = self.TheNow().strftime('%Y-%m-%d_%H%M')
self.response.headers['Content-Type'] = 'application/text'
self.response.headers['Content-Disposition'] = 'attachment; filename=ANALYTICS_ALL_REPORTS_%s.yaml' % dtstr
self.response.out.write(dump)
return
elif (self.request.POST.get('action')=='Delete Report'):
try:
crm = self.get_custom_report_metadata(report_name)
except Exception as err:
logging.error("Failed to find custom report named %s!" % report_name)
raise
crm.key.delete()
msg = "Deleted course report %s" % report_name
return self.get_custom_report(msg=msg)
elif (self.request.POST.get('action')=='Save Changes'):
fields = ['table_name', 'title', 'depends_on', 'html', 'sql', 'javascript', 'description', 'collection', 'group_tags', 'meta_info']
try:
crm = self.get_custom_report_metadata(report_name)
except Exception as err:
logging.error("Failed to find custom report named %s!" % report_name)
raise
for field in fields:
fval = self.request.POST.get(field)
if field=='group_tags':
fval = [x.strip() for x in fval.split(',')]
elif field=='meta_info':
fval = eval(fval) or {}
if fval is None:
logging.error("oops, expected value for field=%s, but got fval=%s" % (field, fval))
else:
setattr(crm, field, fval)
crm.put()
logging.info('saved crm = %s' % crm)
msg = "Saved custom report %s" % report_name
if not crm.table_name:
msg += "...Warning! table_name cannot be left empty"
try:
if not crm:
crm = self.get_custom_report_metadata(report_name)
except Exception as err:
logging.error("Cannot get custom report %s" % report_name)
raise
# backward compatability: to accommodate jhints, ignore lines in javascript with jinja2 template commands
if ('{{parameters}}' in crm.javascript) and not ('jshint ignore' in crm.javascript):
newjs = []
for k in crm.javascript.split('\n'):
if (("{% autoescape" in k)) and not ('jshint ignore' in k):
k += " // jshint ignore:line"
elif ('{{parameters}}' in k) and not ('jshint ignore' in k):
newjs.append("/* jshint ignore:start */")
newjs.append(k)
k = "/* jshint ignore:end */"
newjs.append(k)
crm.javascript = '\n'.join(newjs)
data = {'html': crm.html,
'js': crm.javascript,
'report_name': report_name,
'report': crm,
'msg': msg,
'parameter_values': parameter_values,
'meta_info': json.dumps(crm.meta_info),
'editor_heights': editor_heights or "{}",
}
data.update(self.common_data)
template = JINJA_ENVIRONMENT.get_template('edit_custom_report.html')
self.response.out.write(template.render(data))
def get_admin(self):
'''
Admin page: show authorized users, clear cache
'''
if not self.user in self.AUTHORIZED_USERS: # require superuser
return self.no_auth_sorry()
msg = ""
action = self.request.POST.get('action', None)
custom_reports_standard_source_dir = "ANALYTICS_STANDARD_REPORTS"
custom_reports_standard_source_file = "ANALYTICS_STANDARD_REPORTS.yaml"
crssd = 'data/%s' % custom_reports_standard_source_dir
crssf = 'data/%s' % custom_reports_standard_source_file
if os.path.exists(crssd):
custom_reports_standard_source = custom_reports_standard_source_dir
else:
custom_reports_standard_source = custom_reports_standard_source_file
try:
custom_reports_standard_source = local_config.CUSTOM_REPORTS_SOURCE
except Exception as err:
pass
if action == 'Flush cache':
memcache.flush_all()
msg = "Cache flushed"
elif action == 'Reload staff table':
self.get_staff_table(reload=True)
msg = "Staff table reloaded"
elif action == 'Reload course listings':
self.get_course_listings(ignore_cache=True)
msg = "Course listings reloaded"
elif action == 'List current course tags':
tags = self.get_course_listings_tags()
msg = "Current course tags = %s" % tags
elif action == 'Check access':
username = self.request.get('username')
course_id = self.request.get('course_id')
if self.is_user_authorized_for_course(course_id=course_id,
user=username):
msg = "User %s IS authorized for %s" % (username, course_id)
else:
msg = "User %s is NOT authorized for %s" % (username,
course_id)
elif action == 'Reload Course Listings':
collection = self.request.POST.get('collection')
self.get_course_listings(ignore_cache=True, collection=collection)
msg = "Course listings for '%s' reloaded" % collection
elif action == 'Reload Standard Reports':
crssd = 'data/%s' % custom_reports_standard_source
if os.path.exists(crssd):
if os.path.isdir(crssd):
files = glob.glob('%s/*.yaml' % crssd)
elif os.path.isfile(crssd):
files = [crssd]
msg = "<ul>"
for fn in files:
msg += "<li>Standard Reports loading from %s<br/>" % (fn)
report_file_data = open(fn).read()
try:
msg += self.import_custom_report_from_file_data(
report_file_data, overwrite=True)
except Exception as err:
logging.error(
"Oops! Error importing custom report from %s" %
fn)
raise
msg += "</li>"
msg += "</ul>"
else:
msg = "Error: cannot find file or directory %s" % crssd
elif action == 'Reload Custom Reports':
collection = self.request.POST.get('collection')
cnt = self.import_custom_report_metadata(ignore_cache=True,
collection=collection)
msg = "Custom Report Metadata for '%s' reloaded (%d reports)" % (
collection, cnt)
elif action == 'Export Custom Reports':
collection = self.request.POST.get('collection')
cnt, destination = self.export_custom_report_metadata(
ignore_cache=True, collection=collection)
msg = "Custom Report Metadata for '%s' exported to %s (%d reports)" % (
collection, destination, cnt)
elif action == 'Add staff':
fields = ['username', 'role', 'course_id', 'notes']
data = {x: (self.request.POST.get(x) or '') for x in fields}
self.add_staff_table_entry(data)
msg = "New staff %s added" % data
todelete = self.request.POST.get('do-delete', None)
if todelete is not None:
self.disable_staff_table_entry(int(todelete))
msg = "Deleted staff table row %s" % todelete
stable = self.get_staff_table()
stafftable = self.list2table([
DataTableField({
'icon': 'delete',
'field': 'sid',
'title': ' '
}), 'username', 'role', 'course_id', 'notes'
], stable)
data = self.common_data.copy()
course_listings_source = self.get_collection_metadata(
'COURSE_LISTINGS_TABLE')
data.update({
'superusers':
self.AUTHORIZED_USERS,
'table':
stafftable,
'msg':
msg,
'listings_source':
course_listings_source,
'staff_source':
local_config.STAFF_COURSE_TABLE,
'collections':
self.collections_available(asdict=True),
'custom_reports_standard_source':
custom_reports_standard_source,
})
template = JINJA_ENVIRONMENT.get_template('admin.html')
self.response.out.write(template.render(data))
def get_custom_report(self, msg=""):
'''
custom reports page
'''
if not self.user in self.AUTHORIZED_USERS: # require superuser
return self.no_auth_sorry()
if (self.request.POST.get('action')=='Create new Custom Report'):
title = self.request.POST.get('title')
name = self.request.POST.get('name')
existing_crm = self.get_custom_report_metadata(name, single=False)
if existing_crm.count():
msg = "Cannot create report '%s', already exists" % name
else:
crm = CustomReport(title=title, name=name)
#crm.html = """<div id="contain-{{report_name}}" style="min-width: 310px; height: 400px; margin: 0 auto">
crm.html = """<div id="contain-{{report_name}}" style="min-width: 310px; margin: 0 auto">
<img src="/images/loading_icon.gif"/>\n</div>"""
jstemp, jsfn, uptodate = JINJA_ENVIRONMENT.loader.get_source(JINJA_ENVIRONMENT, 'custom_report_default.js')
crm.javascript = str(jstemp)
#jstemp = JINJA_ENVIRONMENT.get_template('custom_report_default.js')
#crm.javascript = jstemp.render({})
logging.info("[cr] creating new custom report %s" % crm)
crm.put()
# return self.redirect('/custom/edit_report/%s' % name)
#
# because of how NDB may take awhile to store the new entry, we cannot go directly
# to the report edit page, but instead, must go to a transition page asking
# for the user to click on a button first.
data = self.common_data.copy()
data.update({'report': crm,
})
template = JINJA_ENVIRONMENT.get_template('edit_custom_report_transition.html')
self.response.out.write(template.render(data))
return
elif (self.request.POST.get('action')=='Edit this report'):
name = self.request.POST.get('name')
return self.redirect('/custom/edit_report/%s' % name)
elif (self.request.POST.get('action')=='Upload Custom Report(s)'):
report_file_data = self.request.get('file')
overwrite = (self.request.get('overwrite')=='yes')
msg += self.import_custom_report_from_file_data(report_file_data, overwrite)
cr_page_title = "Custom Reports"
try:
cr_page_title = local_config.CUSTOM_REPORTS_PAGE_TITLE
except Exception as err:
pass
data = self.common_data.copy()
data.update({'is_staff': self.is_superuser(),
'reports': self.get_custom_report_metadata(single=False),
'msg': msg,
'custom_report': self.custom_report_container(self.is_authorized_for_custom_report, staff=True,
group_tag = "{{group_tag}}",
),
'cr_page_title': cr_page_title,
})
template = JINJA_ENVIRONMENT.get_template('custom_reports.html')
self.response.out.write(template.render(data))
def __getitem__(self, report_name):
try:
crm = other.get_custom_report_metadata(report_name)
err = None
except Exception as err:
crm = None
if not crm:
logging.info("No custom report '%s' found, err=%s" % (report_name, err))
return "Missing custom report %s" % report_name
# check access authorization
# logging.info('[crc] checking auth for report %s, pdata=%s' % (crm.name, pdata))
auth_ok, msg = is_authorized_for_custom_report(crm, pdata)
if not auth_ok:
return "" # return empty string if not authorized
# logging.info('[cr] name=%s, title=%s' % (crm.name, crm.title)) # debugging
title = JINJA_ENVIRONMENT.from_string(crm.title)
try:
title_rendered = title.render(pdata)
except Exception as err:
logging.error('[cr] Failed to render report %s title %s' % (crm.name, crm.title))
title = crm.title
parameters = {x:v for x,v in pdata.items() if v is not None}
parameters['orgname'] = other.ORGNAME
parameters['dashboard_mode'] = other.MODE # 'mooc' or '' (empty meaning residential, non-mooc)
parameters['course_report'] = other.get_course_report_dataset()
parameters['course_report_org'] = other.get_course_report_dataset(force_use_org=True)
parameters['orgname'] = other.ORGNAME
if 'require_table' in (crm.meta_info or []):
dataset = None
table = crm.meta_info['require_table']
if '{' in table:
try:
table = table.format(**parameters)
except Exception as err:
logging.error("Cannot substitute for parameters in require_table=%s, err=%s" % (table, err))
if '.' in table:
(dataset, table) = table.split('.', 1)
else:
course_id = parameters.get('course_id')
if course_id:
try:
dataset = bqutil.course_id2dataset(course_id, use_dataset_latest=other.use_dataset_latest())
except Exception as err:
logging.error("failed to get dataset for course_id=%s" % course_id)
raise
else:
logging.info("Suppressing report %s because dataset not specifid in require_table %s" % (title, table))
dataset = None
if dataset is not None:
try:
tinfo = bqutil.get_bq_table_info(dataset, table) or None
except Exception as err:
tinfo = None
if not "Not Found" in str(err):
logging.error(err)
logging.error(traceback.format_exc())
if not tinfo:
logging.info("Suppressing report %s because %s.%s doesn't exist" % (title, dataset, table))
return ""
else:
logging.info("Skipping require_table check")
report_id = hashlib.sha224("%s %s" % (crm.name, json.dumps(pdata))).hexdigest()
if crm.description:
try:
crm.description = crm.description.format(**parameters)
except Exception as err:
logging.info('[cr] %s cannot format description %s' % (crm.name, crm.description))
if self.do_no_embed and 'embedded' in (crm.meta_info or {}):
crm.meta_info.pop('embedded')
if self.force_embed:
crm.meta_info['embedded'] = True
if self.do_always_show:
crm.meta_info['always_show'] = True
template = JINJA_ENVIRONMENT.get_template('custom_report_container.html')
data = {'is_staff': other.is_superuser(),
'report': crm,
'report_params': json.dumps(parameters),
'report_is_staff': pdata.get('staff'),
'report_meta_info': json.dumps(crm.meta_info or {}),
'immediate_view': json.dumps(self.immediate_view),
'do_embed' : (crm.meta_info or {}).get('embedded') or self.force_embed,
'always_show': self.do_always_show,
'title': title_rendered,
'id': report_id,
}
self.immediate_view = False # return to non-immediate view by default
self.do_no_embed = False # return to default
self.force_embed = False # return to default
return template.render(data)
def ajax_get_report_html(self, report_name=None):
'''
return HTML for specified custom report
'''
crm, pdata, auth_ok, msg = self.custom_report_auth_check(
report_name) # crm = CourseReport model
if not auth_ok:
return self.no_auth_sorry()
if self.request.get('save_parameters'):
self.session['edit_report_parameter_values'] = json.dumps(
{x: v
for x, v in pdata.items() if v is not None})
logging.info("Saved edit_report_parameter_values = %s" %
self.session['edit_report_parameter_values'])
parameters = {x: v for x, v in pdata.items() if v is not None}
parameters['orgname'] = self.ORGNAME
parameters[
'dashboard_mode'] = self.MODE # 'mooc' or '' (empty meaning residential, non-mooc)
parameters['feature_flags'] = self.FEATURE_FLAGS
uuid = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(10))
render_data = {
'report_name':
report_name,
'report_uuid':
"%s_%s" % (report_name, uuid),
'orgname':
self.ORGNAME,
'parameters':
json.dumps(parameters), # for js
'parameter_values':
parameters, # for html template variables
'custom_report':
self.custom_report_container(
self.is_authorized_for_custom_report, **parameters
), # pass pdata so children of page also get parameters
'is_staff':
self.is_superuser(),
'is_pm':
self.is_pm(),
'cr_js_library':
self.get_custom_report_library_javascript(),
}
render_data.update(pdata)
#html = crm.html
html = JINJA_ENVIRONMENT.from_string(crm.html).render(
render_data) # allows custom reports to be included
js_src = "<script type='text/javascript'>"
js_src += "$(document).ready( function () {%s} );" % crm.javascript # js goes in html
js_src += "</script>"
try:
js = Template(js_src).render(
render_data
) # render template variables, ideally w/o global context
except Exception as err:
logging.info(
"Warning: could not render js in custom_reports, err=%s" %
str(err))
js = JINJA_ENVIRONMENT.from_string(js_src).render(render_data)
#template = JINJA_ENVIRONMENT.from_string(html)
# logging.info("get_report_html name=%s, parameters=%s" % (report_name, parameters))
data = {
'html': html + js,
'js': crm.javascript,
}
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(data))
def get_custom_report(self, msg=""):
'''
custom reports page
'''
if not self.user in self.AUTHORIZED_USERS: # require superuser
return self.no_auth_sorry()
if (self.request.POST.get('action') == 'Create new Custom Report'):
title = self.request.POST.get('title')
name = self.request.POST.get('name')
existing_crm = self.get_custom_report_metadata(name, single=False)
if existing_crm.count():
msg = "Cannot create report '%s', already exists" % name
else:
crm = CustomReport(title=title, name=name)
#crm.html = """<div id="contain-{{report_name}}" style="min-width: 310px; height: 400px; margin: 0 auto">
crm.html = """<div id="contain-{{report_name}}" style="min-width: 310px; margin: 0 auto">
<img src="/images/loading_icon.gif"/>\n</div>"""
jstemp, jsfn, uptodate = JINJA_ENVIRONMENT.loader.get_source(
JINJA_ENVIRONMENT, 'custom_report_default.js')
crm.javascript = str(jstemp)
#jstemp = JINJA_ENVIRONMENT.get_template('custom_report_default.js')
#crm.javascript = jstemp.render({})
logging.info("[cr] creating new custom report %s" % crm)
crm.put()
# return self.redirect('/custom/edit_report/%s' % name)
#
# because of how NDB may take awhile to store the new entry, we cannot go directly
# to the report edit page, but instead, must go to a transition page asking
# for the user to click on a button first.
data = self.common_data.copy()
data.update({
'report': crm,
})
template = JINJA_ENVIRONMENT.get_template(
'edit_custom_report_transition.html')
self.response.out.write(template.render(data))
return
elif (self.request.POST.get('action') == 'Edit this report'):
name = self.request.POST.get('name')
return self.redirect('/custom/edit_report/%s' % name)
elif (self.request.POST.get('action') == 'Upload Custom Report(s)'):
report_file_data = self.request.get('file')
overwrite = (self.request.get('overwrite') == 'yes')
msg += self.import_custom_report_from_file_data(
report_file_data, overwrite)
cr_page_title = "Custom Reports"
try:
cr_page_title = local_config.CUSTOM_REPORTS_PAGE_TITLE
except Exception as err:
pass
data = self.common_data.copy()
data.update({
'is_staff':
self.is_superuser(),
'reports':
self.get_custom_report_metadata(single=False),
'msg':
msg,
'custom_report':
self.custom_report_container(
self.is_authorized_for_custom_report,
staff=True,
group_tag="{{group_tag}}",
),
'cr_page_title':
cr_page_title,
})
template = JINJA_ENVIRONMENT.get_template('custom_reports.html')
self.response.out.write(template.render(data))
