def sample_detail(request, sample_id):
context = {}
project = request.user.project
connection = Connection(get_bamboo_url(project))
main_dataset = CachedDataset(get_bamboo_dataset_id(project),
connection=connection)
try:
sample = main_dataset.get_data(query={'sample_id_sample_barcode_id':
sample_id},
cache=True,
cache_expiry=2592000)[-1]
except:
try:
sample = main_dataset.get_data(query={'sample_id_sample_manual_id':
sample_id},
cache=True, cache_expiry=2592000)[-1]
except:
raise Http404(u"Requested Sample (%(sample)s) does not exist."
% {'sample': sample_id})
from collections import OrderedDict
sorted_sample = OrderedDict([(key, sample[key])
for key in sorted(sample.iterkeys())])
results = soil_results(sample)
context.update({'sample': sorted_sample,
'results': results})
return render(request, 'sample_detail.html', context)
def list_submissions(request):
context = {'category': 'submissions'}
connection = Connection(get_bamboo_url(request.user.project))
main_dataset = CachedDataset(get_bamboo_dataset_id(request.user.project),
connection=connection)
submissions_list = main_dataset.get_data(cache=True, cache_expiry=VSHORT)
for submission in submissions_list:
submission.update(detailed_id_dict(submission, prefix='teacher_'))
paginator = FlynsarmyPaginator(submissions_list, 10, adjacent_pages=2)
page = request.GET.get('page')
try:
submissions = paginator.page(page)
except PageNotAnInteger:
submissions = paginator.page(1)
except EmptyPage:
submissions = paginator.page(paginator.num_pages)
context.update({'submissions': submissions})
return render(request, 'list_submissions.html', context)
def samples_list(request, search_string=None):
context = {}
lookup = request.GET.get('lookup', None)
# for now, look up will just forward to detail view
if lookup:
return sample_detail(request, lookup.strip())
# init bamboo with user's URL
project = request.user.project
connection = Connection(get_bamboo_url(project))
main_dataset = CachedDataset(get_bamboo_dataset_id(project),
connection=connection)
submissions_list = main_dataset.get_data(cache=True, cache_expiry=60,
order_by='-end')
paginator = FlynsarmyPaginator(submissions_list, 20, adjacent_pages=2)
page = request.GET.get('page')
try:
submissions = paginator.page(page)
except PageNotAnInteger:
submissions = paginator.page(1)
except EmptyPage:
submissions = paginator.page(paginator.num_pages)
context.update({'samples': submissions,
'lookup': lookup})
return render(request, 'samples_list.html', context)
def home(request):
context = {'category': 'home'}
connection = Connection(get_bamboo_url(request.user.project))
main_dataset = CachedDataset(get_bamboo_dataset_id(request.user.project),
connection=connection)
teacher_dataset = CachedDataset(get_bamboo_dataset_id(request.user.project,
is_registration=True),
connection=connection)
# total number of submissions
try:
nb_submissions = main_dataset.get_data(count=True, cache=True)
except (BambooError, ErrorParsingBambooData) as e:
print(e)
nb_submissions = None
# total number of registered teachers
try:
nb_teachers = teacher_dataset.get_data(count=True, cache=True)
except BambooError:
nb_teachers = None
context.update({'nb_submissions': nb_submissions,
'nb_teachers': nb_teachers})
return render(request, 'home.html', context)
def detail_teacher_bamboo(request, uid):
''' Report Card View leveraging bamboo aggregation '''
context = {'category': 'teachers',
'schoolcat': '%s|%s'
% (request.user.project.slug, 'school_names')}
# retrieve short ID
sid = request.GET.get('short', None)
if not sid:
sid = short_id_from(uid)
# build barcode (identifier on submissions) from UID param.
barcode = build_urlid_with(uid, sid)
connection = Connection(get_bamboo_url(request.user.project))
teacher_dataset = CachedDataset(get_bamboo_dataset_id(request.user.project,
is_registration=True),
connection=connection)
# Retrieve teacher data from bamboo (1req)
teacher = teacher_dataset.get_data(query={'barcode': barcode},
cache=True, cache_expiry=VLONG)[0]
context.update({'teacher': teacher})
teacher.update(detailed_id_dict(teacher))
return render(request, 'detail_teacher_bamboo.html', context)
def list_teachers(request):
context = {'category': 'teachers',
'schoolcat': '%s|%s'
% (request.user.project.slug, 'school_names')}
# redirect to a teacher's page if jump_to matches one.
jump_to = request.GET.get('jump_to', None)
connection = Connection(get_bamboo_url(request.user.project))
teacher_dataset = CachedDataset(get_bamboo_dataset_id(request.user.project,
is_registration=True),
connection=connection)
teachers_list = teacher_dataset.get_data(cache=True, cache_expiry=MEDIUM)
''' Not using the `group` method of bamboo here as it's innefiscient
for our use case: it would request multiple inner-loops and uglify
the templates.
Instead, we grab the raw list of teachers and then group them
by the school name resulting in a 2-dim list.
While it works, I believe the pagination is done at school level
making it useless for cases where there are many teachers per school '''
# school list container ; each key should contain list of teachers.
school_list = {}
for teacher in teachers_list:
if not teacher.get('barcode', None):
continue
teacher.update(detailed_id_dict(teacher))
# requested teacher exists ; redirect
if teacher.get('sid_') == jump_to:
return redirect(detail_teacher, uid=teacher.get('uid_'))
if teacher.get('school') in school_list:
school_list.get(teacher.get('school')).append(teacher)
else:
school_list[teacher.get('school')] = [teacher, ]
paginator = FlynsarmyPaginator(school_list.values(), 10, adjacent_pages=2)
page = request.GET.get('page', 1)
try:
schools = paginator.page(page)
except PageNotAnInteger:
schools = paginator.page(1)
except EmptyPage:
schools = paginator.page(paginator.num_pages)
# context.update({'teachers': teachers})
context.update({'schools': schools})
return render(request, 'list_teachers.html', context)
def retrieve_dataset(self):
connection = Connection(get_bamboo_url(self.project))
self.dataset = CachedDataset(get_option(self.project, 'ests_dataset'),
connection=connection)
self.data = self.dataset.get_data(query={u'barcode': self.sample_id},
cache=True)[0]
self.siblings = {}
for position in self.POSITIONS:
if position == self.position:
self.siblings[position] = self.sample_id
continue
try:
data = self.dataset.get_data(query={u'position': position,
u'block': self.data['block'],
u'quadrant': self.data['quadrant'],
u'cluster': self.data['cluster'],
u'plot': self.data['plot'],
},
select=['barcode'],
cache=True)[-1]
if isinstance(data, dict):
data = data.get(u'barcode', u'')
except:
# raise
data = None
self.siblings[position] = data
def card_teacher(request, uid):
context = {'category': 'teachers'}
connection = Connection(get_bamboo_url(request.user.project))
teacher_dataset = CachedDataset(get_bamboo_dataset_id(request.user.project,
is_registration=True),
connection=connection)
teacher = teacher_dataset.get_data(query={'barcode': uid},
cache=True, cache_expiry=VLONG)[0]
teacher.update(detailed_id_dict(teacher))
teacher.update({'qrcode': b64_qrcode(uid, scale=2.0)})
context.update({'teacher': teacher})
return render(request, 'card_teacher.html', context)
class ESTSSample(object):
STATUS_COLLECTED = 0
STATUS_SENT_TO_PC = 1
STATUS_ARRIVED_AT_PC = 2
STATUS_SENT_TO_NSTC = 3
STATUS_ARRIVED_AT_NSTC = 4
STATUS_SENT_TO_ARCHIVE = 5
STATUS_ARRIVED_AT_ARCHIVE = 6
STATUSES = {STATUS_COLLECTED: u"Collected",
STATUS_SENT_TO_PC: u"Sent to PC",
STATUS_ARRIVED_AT_PC: u"Arrived at PC",
STATUS_SENT_TO_NSTC: u"Sent to NSTC",
STATUS_ARRIVED_AT_NSTC: u"Arrived at NSTC",
STATUS_SENT_TO_ARCHIVE: u"Sent to Archive",
STATUS_ARRIVED_AT_ARCHIVE: u"Arrived at Archive"}
TOP_QR = 'top_qr'
SUB_QR = 'sub_qr'
QR_20_40 = 'qr_20_40'
QR_40_60 = 'qr_40_60'
QR_60_80 = 'qr_60_80'
QR_80_100 = 'qr_80_100'
POSITIONS = {
TOP_QR: u"top",
SUB_QR: u"0-20",
QR_20_40: u"20-40",
QR_40_60: u"40-60",
QR_60_80: u"60-80",
QR_80_100: u"80-100"
}
def __init__(self, sample_id, project):
self.sample_id = sample_id
self._position = None
self._status = None
self._status_date = None
self._events = {}
self.project = project
self.retrieve_dataset()
self.query_status()
def __str__(self):
return self.sample_id
def retrieve_dataset(self):
connection = Connection(get_bamboo_url(self.project))
self.dataset = CachedDataset(get_option(self.project, 'ests_dataset'),
connection=connection)
self.data = self.dataset.get_data(query={u'barcode': self.sample_id},
cache=True)[0]
self.siblings = {}
for position in self.POSITIONS:
if position == self.position:
self.siblings[position] = self.sample_id
continue
try:
data = self.dataset.get_data(query={u'position': position,
u'block': self.data['block'],
u'quadrant': self.data['quadrant'],
u'cluster': self.data['cluster'],
u'plot': self.data['plot'],
},
select=['barcode'],
cache=True)[-1]
if isinstance(data, dict):
data = data.get(u'barcode', u'')
except:
# raise
data = None
self.siblings[position] = data
def query_status(self):
for astatus in self.STATUSES.keys():
end = self.data.get('step%d_end_time' % astatus)
name = self.data.get('step%d_name' % astatus)
try:
if astatus > 1:
prev_end = self.data.get('step%d_end_time' % (astatus - 1))
else:
prev_end = self.data.get('end')
delay = (end - prev_end)
except:
delay = None
processing_center = self.data.get('step%d_processing_center' % astatus)
pc_destination = self.data.get('step%s_pc_destination' % astatus)
pc_name = processing_center if processing_center else pc_destination
if end is not None and end != u'null':
# create Event
self._events[astatus] = {'date': end,
'delay': delay,
'survey_day': end,
'pc_name': pc_name,
'name': name,
'status': astatus}
# Update Status
self._status = astatus
self._status_date = end
# mark as collected if none found
if not self.status:
self._status = self.STATUS_COLLECTED
self._status_date = self.data.get('end')
@property
def is_valid(self):
return self._status is not None
@property
def status(self):
return self._status
@property
def status_date(self):
return self._status_date
@property
def verbose_status(self):
return self.STATUSES.get(self.status)
@classmethod
def get_verbose_status(cls, status):
return cls.STATUSES.get(status)
@property
def position(self):
return self.data.get(u'position', u"n/a")
@property
def depth(self):
return self.position
def events(self):
return self._events
def ident(self):
return u'%s.%s.%s.%s' % self.plot_ident_data() + '.' + self.depth
def plot_ident_data(self):
return (self.iblock, self.iquadrant, self.icluster, self.iplot)
def plot_ident(self):
return u".".join([str(e) for e in self.plot_ident_data()])
@property
def iblock(self):
block = self.data.get(u'block', u'')
try:
return int(block)
except:
return block
@property
def iquadrant(self):
quadrant = self.data.get(u'quadrant', u'')
try:
return int(quadrant)
except:
return quadrant
@property
def icluster(self):
cluster = self.data.get(u'cluster', u'')
try:
return int(cluster)
except:
return cluster
@property
def iplot(self):
plot = self.data.get(u'plot', u'')
try:
return int(plot)
except:
return plot
@property
def gps(self):
# use only GPS1 for now
gpd = {}
for key, value in self.data.items():
if not key.startswith('_gps1'):
continue
gpd.update({key.replace('_gps1_', ''): value})
return gpd
@property
def sibling_top_qr(self):
return self.siblings.get('top_qr')
@property
def sibling_sub_qr(self):
return self.siblings.get('sub_qr')
@property
def sibling_qr_20_40(self):
return self.siblings.get('qr_20_40')
@property
def sibling_qr_40_60(self):
return self.siblings.get('qr_40_60')
@property
def sibling_qr_60_80(self):
return self.siblings.get('qr_60_80')
@property
def sibling_qr_80_100(self):
return self.siblings.get('qr_80_100')
def detail_teacher_django(request, uid):
''' Report Card View processing data from submissions list/data only
2 bamboo requests:
- teacher data from uuid
- list of submissions for that teacher
All processing/grouping done in python. '''
context = {'category': 'teachers',
'schoolcat': '%s|%s'
% (request.user.project.slug, 'school_names')}
connection = Connection(get_bamboo_url(request.user.project))
main_dataset = CachedDataset(get_bamboo_dataset_id(request.user.project),
connection=connection)
teacher_dataset = CachedDataset(get_bamboo_dataset_id(request.user.project,
is_registration=True),
connection=connection)
def get_age_group(sub):
''' return age group (key str) of a given submission based on age'''
for key, age_range in REPORTS_AGE_GROUPS.items():
if (sub.get(u'general_information_age', 0)
in range(age_range[0], age_range[1] + 1)):
return key
return u"all"
def init_reports():
''' A reports container with initial values '''
reports = {}
for age_group in REPORTS_AGE_GROUPS.keys():
reports[age_group] = {}
for sex in ('male', 'female', 'total'):
reports[age_group][sex] = {}
for level in REPORTS_READING_LEVELS.keys() \
+ REPORTS_NUMERACY_LEVELS.keys() + ['total']:
reports[age_group][sex][level] = {'nb': 0, 'percent': None}
return reports
def compute_report_for(reports, submission, is_numeracy=False):
''' increment counters on all categories for a submission '''
age_group = get_age_group(submission)
sex = submission.get('general_information_sex')
level = submission.get('learning_levels_numeracy_nothing' if is_numeracy
else 'learning_levels_reading_nothing')
# AGE GRP SEX LEVEL
reports['all']['total']['total']['nb'] += 1
reports['all']['total'][level]['nb'] += 1
reports['all'][sex]['total']['nb'] += 1
reports['all'][sex][level]['nb'] += 1
reports[age_group][sex][level]['nb'] += 1
reports[age_group]['total'][level]['nb'] += 1
reports[age_group][sex]['total']['nb'] += 1
reports[age_group]['total']['total']['nb'] += 1
def compute_percentages(reports):
''' calculates the percentages fields for the reports dict '''
def pc(num, denum):
try:
return float(num['nb']) / float(denum['nb'])
except ZeroDivisionError:
return 0
for age_group, ago in reports.items():
for sex, so in ago.items():
for level, lo in so.items():
reports[age_group][sex][level]['percent'] = \
pc(reports[age_group][sex][level],
reports[age_group][sex]['total'])
def sort_reports(reports_dict):
''' sort report by Age (asc) then gender keeping total/All at last '''
def cmp_rep(x, y):
if x == y:
return 0
# 'all' age group is last
if x == 'all' or y == 'all':
return 1 if x == 'all' else -1
# compare minimum age for group to sort
xa_min = REPORTS_AGE_GROUPS.get(x, (100, 0))[0]
ya_min = REPORTS_AGE_GROUPS.get(y, (100, 0))[0]
return 1 if xa_min > ya_min else -1
reports = []
# loop on dict and transform to:
# - dicts composed of {'name': x, 'data': y}
# - data is an ordered array of dicts
for age_group in sorted(reports_dict.keys(), cmp=cmp_rep):
age_group_data = reports_dict.get(age_group)
age_group_data.update({'name': age_group})
age_group_sex = []
for sex in sorted(age_group_data.keys()):
if sex == 'name':
continue
sex_data = age_group_data.get(sex)
sex_data.update({'name': sex})
age_group_sex.append(sex_data)
reports.append({'name': age_group,
'data': age_group_sex})
return reports
# retrieve short ID
sid = request.GET.get('short', None)
if not sid:
sid = short_id_from(uid)
# if we don't have an UID, find out teacher from SID.
if uid == sid:
barcodes = {}
for bc in teacher_dataset.get_data(select=['barcode'],
cache=True, cache_expiry=VSHORT):
barcodes[short_id_from(bc)] = bc
barcode = barcodes.get(sid, '')
else:
barcode = uid
# Retrieve teacher data from bamboo (1req)
teacher = teacher_dataset.get_data(query={'barcode': barcode},
cache=True, cache_expiry=VLONG)[0]
teacher.update(detailed_id_dict(teacher))
# retrieve list of submissions from bamboo (1req)
submissions = main_dataset.get_data(
query={'$or': [{'teacher_barcode': barcode},
{'teacher_fallback_id':
{'$regex':sid, '$options':'i'}}]},
select=['general_information_age',
'general_information_sex',
'learning_levels_numeracy_nothing',
'learning_levels_reading_nothing',
'school_junior_secondary',
'school_primary',
'school_senior_secondary',
'schooling_status_grades'])
# initialize containers for reading report and numeracy report.
reading_dict = init_reports()
numeracy_dict = init_reports()
# loop on submissions to fill reading/num reports
for submission in submissions:
compute_report_for(reading_dict, submission)
compute_report_for(numeracy_dict, submission, is_numeracy=True)
# compute percentages for reading/num reports
compute_percentages(reading_dict)
compute_percentages(numeracy_dict)
# sort/transform reports for template
reading_reports = sort_reports(reading_dict)
numeracy_reports = sort_reports(numeracy_dict)
context.update({'teacher': teacher,
'reading_reports': reading_reports,
'numeracy_reports': numeracy_reports})
return render(request, 'detail_teacher.html', context)
