def create_records_from(request, xls_data, filepath=None):
nb_errors = nb_updated = nb_created = 0
try:
payload = DataRecord.batch_create(xls_data, request.user.partner)
except Exception as e:
payload = xls_data
level = 'danger'
logger.exception(e)
message = _("Unable to record data: {exp}").format(exp=repr(e))
nb_errors = 1
else:
nb_errors = len(payload['errors'])
nb_created = sum([
1 for v in payload.values()
if isinstance(v, dict) and v.get('action') == 'created'
])
nb_updated = sum([
1 for v in payload.values()
if isinstance(v, dict) and v.get('action') == 'updated'
])
if nb_created or nb_updated:
message = _("Congratulations! Your data has been recorded.")
if nb_created:
message += "\n" + _("{nb_created} records were created.")
if nb_updated:
message += "\n" + _("{nb_updated} records were updated.")
if nb_errors:
message += "\n" + _("{nb_errors} errors "
"were found in your file.")
level = 'warning'
else:
level = 'success'
elif not nb_errors:
message = _("Thank You for submitting! "
"No new data to record though.")
level = 'info'
else:
message = _("Outch! Your file contains {nb_errors} errors!")
level = 'danger'
finally:
if filepath is not None:
os.unlink(filepath)
payload['feedback'] = {
'nb_errors':
nb_errors,
'nb_updated':
nb_updated,
'nb_created':
nb_created,
'text':
message.format(nb_errors=nb_errors,
nb_updated=nb_updated,
nb_created=nb_created),
'level':
level,
'redirect':
level in ('success', 'info'),
}
return payload
def raw_data_record(request, record_id, *args, **kwargs):
context = {"page": "raw_data"}
dr = DataRecord.get_by_id(record_id)
if dr is None:
raise Http404(_("Unable to find DataRecord with ID `{id}`").format(id=record_id))
context.update({"record": dr})
return render(request, kwargs.get("template_name", "raw_data_record.html"), context)
def create_records_from(request, xls_data, filepath=None):
nb_errors = nb_updated = nb_created = 0
try:
payload = DataRecord.batch_create(xls_data,
request.user.partner)
except Exception as e:
payload = xls_data
level = 'danger'
logger.exception(e)
message = _("Unable to record data: {exp}").format(exp=repr(e))
nb_errors = 1
else:
nb_errors = len(payload['errors'])
nb_created = sum([1 for v in payload.values()
if isinstance(v, dict)
and v.get('action') == 'created'])
nb_updated = sum([1 for v in payload.values()
if isinstance(v, dict)
and v.get('action') == 'updated'])
if nb_created or nb_updated:
message = _("Congratulations! Your data has been recorded.")
if nb_created:
message += "\n" + _("{nb_created} records were created.")
if nb_updated:
message += "\n" + _("{nb_updated} records were updated.")
if nb_errors:
message += "\n" + _("{nb_errors} errors "
"were found in your file.")
level = 'warning'
else:
level = 'success'
elif not nb_errors:
message = _("Thank You for submitting! "
"No new data to record though.")
level = 'info'
else:
message = _("Outch! Your file contains {nb_errors} errors!")
level = 'danger'
finally:
if filepath is not None:
os.unlink(filepath)
payload['feedback'] = {
'nb_errors': nb_errors,
'nb_updated': nb_updated,
'nb_created': nb_created,
'text': message.format(nb_errors=nb_errors,
nb_updated=nb_updated,
nb_created=nb_created),
'level': level,
'redirect': level in ('success', 'info'),
}
return payload
def raw_data_record(request, record_id, *args, **kwargs):
context = {'page': 'raw_data'}
dr = DataRecord.get_by_id(record_id)
if dr is None:
raise Http404(
_("Unable to find DataRecord with ID `{id}`").format(id=record_id))
context.update({'record': dr})
return render(request, kwargs.get('template_name', 'raw_data_record.html'),
context)
def value_text_for(indicator, periods, child):
if len(periods) == 1:
dr = DataRecord.get_or_none(
indicator=indicator,
period=periods[-1],
entity=child,
only_validated=True)
if dr is None:
return "-"
return dr.human()
else:
dr = indicator.data_for(periods=periods, entity=child)
return dr['human']
def handle_record(self, jsdata, entity, periods):
logger.info(periods)
missing = '0.0'
data = {}
# loop on rows
indic_data = {(indic_id, pid): val
for indic_id, pid, val in jsdata['rows']}
for period in periods:
pid = period.dhis_strid
for indicator in Indicator.get_all_dhis():
numerator = indic_data.get((indicator.dhis_numerator_id, pid))
denominator = indic_data.get(
(indicator.dhis_denominator_id, pid))
if numerator is None or numerator == missing:
logger.error("Missing numerator `{}` for `{}`".format(
indicator.dhis_numerator_id, indicator))
continue
if not indicator.is_number and (denominator is None
or denominator == missing):
logger.error("Missing denominator `{}` for `{}`".format(
indicator.dhis_denominator_id, indicator))
continue
logger.debug(data_ident_for(indicator, period, entity))
data.update({
data_ident_for(indicator, period, entity): {
'slug': indicator.slug,
'period': period,
'entity': entity,
'numerator': numerator,
'denominator': denominator
}
})
d = DataRecord.batch_create(data,
dhisbot,
source=DataRecord.DHIS,
arrival_status=DataRecord.ARRIVED,
auto_validate=True)
if self.debug:
pp(d)
return d
def json_data_record_for(request, period_str, entity_uuid, indicator_slug):
entity = Entity.get_or_none(entity_uuid)
if entity is None:
raise Http404(_("Unknown entity UUID `{u}`").format(u=entity_uuid))
period = process_period_filter(request, period_str, 'period').get('period')
if period is None:
raise Http404(_("Unknown period `{p}`").format(p=period_str))
indicator = Indicator.get_or_none(indicator_slug)
if indicator is None:
raise Http404(_("Unknown indicator `{s}`").format(s=indicator_slug))
return JsonResponse(DataRecord.get_for(period, entity, indicator),
safe=False)
def json_data_record_for(request, period_str, entity_uuid, indicator_slug):
entity = Entity.get_or_none(entity_uuid)
if entity is None:
raise Http404(_("Unknown entity UUID `{u}`").format(u=entity_uuid))
period = process_period_filter(request, period_str, 'period').get('period')
if period is None:
raise Http404(_("Unknown period `{p}`").format(p=period_str))
indicator = Indicator.get_or_none(indicator_slug)
if indicator is None:
raise Http404(_("Unknown indicator `{s}`").format(s=indicator_slug))
return JsonResponse(DataRecord.get_for(period, entity, indicator),
safe=False)
def handle_record(self, jsdata, entity, periods):
logger.info(periods)
missing = '0.0'
data = {}
# loop on rows
indic_data = {(indic_id, pid): val
for indic_id, pid, val in jsdata['rows']}
for period in periods:
pid = period.dhis_strid
for indicator in Indicator.get_all_dhis():
numerator = indic_data.get((indicator.dhis_numerator_id, pid))
denominator = indic_data.get(
(indicator.dhis_denominator_id, pid))
if numerator is None or numerator == missing:
logger.error("Missing numerator `{}` for `{}`"
.format(indicator.dhis_numerator_id,
indicator))
continue
if not indicator.is_number and (denominator is None
or denominator == missing):
logger.error("Missing denominator `{}` for `{}`"
.format(indicator.dhis_denominator_id,
indicator))
continue
logger.debug(data_ident_for(indicator, period, entity))
data.update({data_ident_for(indicator, period, entity): {
'slug': indicator.slug,
'period': period,
'entity': entity,
'numerator': numerator,
'denominator': denominator}})
d = DataRecord.batch_create(data, dhisbot,
source=DataRecord.DHIS,
arrival_status=DataRecord.ARRIVED,
auto_validate=True)
if self.debug:
pp(d)
return d
def data_point_for(indicator, entity, period):
dr = DataRecord.get_or_none(indicator=indicator,
period=period, entity=entity,
only_validated=True)
return {
'has_data': bool(getattr(dr, 'id', False)),
'id': getattr(dr, 'id', None),
'indicator': indicator,
'entity': entity,
'period': period,
'numerator': getattr(dr, 'numerator', None),
'denominator': getattr(dr, 'denominator', None),
'value': getattr(dr, 'value', None),
'formatted': getattr(dr, 'formatted', None),
'formatted_numerator': getattr(dr, 'formatted_numerator', None),
'formatted_denominator': getattr(dr, 'formatted_denominator', None),
'human': getattr(dr, 'human', lambda: None)(),
}
def data_point_for(indicator, entity, period):
dr = DataRecord.get_or_none(indicator=indicator,
period=period,
entity=entity,
only_validated=True)
return {
'has_data': bool(getattr(dr, 'id', False)),
'id': getattr(dr, 'id', None),
'indicator': indicator,
'entity': entity,
'period': period,
'numerator': getattr(dr, 'numerator', None),
'denominator': getattr(dr, 'denominator', None),
'value': getattr(dr, 'value', None),
'formatted': getattr(dr, 'formatted', None),
'formatted_numerator': getattr(dr, 'formatted_numerator', None),
'formatted_denominator': getattr(dr, 'formatted_denominator', None),
'human': getattr(dr, 'human', lambda: None)(),
}
def generate_validation_tally_for(entity, period, save_to=None):
from dmd.models.DataRecords import DataRecord
# colors
black = 'FF000000'
dark_gray = 'FFA6A6A6'
light_gray = 'FFDEDEDE'
# styles
header_font = Font(name='Calibri',
size=12,
bold=True,
italic=False,
vertAlign=None,
underline='none',
strike=False,
color=black)
std_font = Font(name='Calibri',
size=12,
bold=False,
italic=False,
vertAlign=None,
underline='none',
strike=False,
color=black)
header_fill = PatternFill(fill_type=FILL_SOLID, start_color=dark_gray)
thin_black_side = Side(style='thin', color='FF000000')
std_border = Border(
left=thin_black_side,
right=thin_black_side,
top=thin_black_side,
bottom=thin_black_side,
)
centered_alignment = Alignment(horizontal='center',
vertical='center',
text_rotation=0,
wrap_text=False,
shrink_to_fit=False,
indent=0)
left_alignment = Alignment(horizontal='left', vertical='center')
number_format = '# ### ### ##0'
header_style = {
'font': header_font,
'fill': header_fill,
'border': std_border,
'alignment': centered_alignment,
}
std_style = {
'font': std_font,
'border': std_border,
'alignment': centered_alignment,
'number_format': number_format,
}
name_style = {
'font': std_font,
'border': std_border,
'alignment': left_alignment,
'number_format': number_format,
}
odd_fill = PatternFill(fill_type=FILL_SOLID, start_color=light_gray)
col_number = 1
col_indicator = 2
col_numerator = 3
col_denominator = 4
col_human = 5
col_valid = 6
col_new_numerator = 7
col_new_denominator = 8
last_col = col_new_denominator
def apply_style(target, style):
for key, value in style.items():
setattr(target, key, value)
title = "Validation {dps} {period}".format(dps=entity.short_name,
period=period.strid)
logger.info(title)
wb = Workbook()
wb.remove_sheet(wb.active)
# write a sheet for each ZS
for zs in entity.get_children():
ws = wb.create_sheet()
ws.title = zs.short_name
def std_write(row, column, value, style=std_style):
cell = ws.cell(row=row, column=column)
cell.value = value
apply_style(cell, style)
# write header
row = 1
std_write(row, col_number, "#", header_style)
xl_set_col_width(ws, col_number, 1.6)
std_write(row, col_indicator, "Indicateur", header_style)
xl_set_col_width(ws, col_indicator, 37.3)
std_write(row, col_numerator, "Numérateur", header_style)
xl_set_col_width(ws, col_numerator, 2.8)
std_write(row, col_denominator, "Dénominateur", header_style)
xl_set_col_width(ws, col_denominator, 3)
std_write(row, col_human, "Valeur.", header_style)
std_write(row, col_valid, "OK ?.", header_style)
std_write(row, col_new_numerator, "Numérateur.", header_style)
xl_set_col_width(ws, col_new_numerator, 2.8)
std_write(row, col_new_denominator, "Dénominateur", header_style)
xl_set_col_width(ws, col_new_denominator, 3)
row += 1
# one row per indicator
for indicator in Indicator.objects.all():
dr = DataRecord.get_or_none(period=period,
entity=zs,
indicator=indicator,
only_validated=False)
std_write(row, col_number, indicator.number, std_style)
std_write(row, col_indicator, indicator.name, name_style)
std_write(row, col_numerator, "" if dr is None else dr.numerator,
std_style)
std_write(row, col_denominator,
"" if dr is None else dr.denominator, std_style)
std_write(row, col_human, "" if dr is None else dr.human(),
std_style)
std_write(row, col_valid, "", std_style)
std_write(row, col_new_numerator, "", std_style)
std_write(row, col_new_denominator, "", std_style)
row += 1
# apply even/odd style
for r in range(1, row):
if r % 2 == 0:
for c in range(1, last_col + 1):
ws.cell(row=r, column=c).fill = odd_fill
if save_to:
logger.info("saving to {}".format(save_to))
wb.save(save_to)
return
stream = StringIO.StringIO()
wb.save(stream)
return stream
def generate_validation_tally_for(entity, period, save_to=None):
from dmd.models.DataRecords import DataRecord
# colors
black = 'FF000000'
dark_gray = 'FFA6A6A6'
light_gray = 'FFDEDEDE'
# styles
header_font = Font(
name='Calibri',
size=12,
bold=True,
italic=False,
vertAlign=None,
underline='none',
strike=False,
color=black)
std_font = Font(
name='Calibri',
size=12,
bold=False,
italic=False,
vertAlign=None,
underline='none',
strike=False,
color=black)
header_fill = PatternFill(fill_type=FILL_SOLID, start_color=dark_gray)
thin_black_side = Side(style='thin', color='FF000000')
std_border = Border(
left=thin_black_side,
right=thin_black_side,
top=thin_black_side,
bottom=thin_black_side,
)
centered_alignment = Alignment(
horizontal='center',
vertical='center',
text_rotation=0,
wrap_text=False,
shrink_to_fit=False,
indent=0)
left_alignment = Alignment(
horizontal='left',
vertical='center')
number_format = '# ### ### ##0'
header_style = {
'font': header_font,
'fill': header_fill,
'border': std_border,
'alignment': centered_alignment,
}
std_style = {
'font': std_font,
'border': std_border,
'alignment': centered_alignment,
'number_format': number_format,
}
name_style = {
'font': std_font,
'border': std_border,
'alignment': left_alignment,
'number_format': number_format,
}
odd_fill = PatternFill(fill_type=FILL_SOLID, start_color=light_gray)
col_number = 1
col_indicator = 2
col_numerator = 3
col_denominator = 4
col_human = 5
col_valid = 6
col_new_numerator = 7
col_new_denominator = 8
last_col = col_new_denominator
def apply_style(target, style):
for key, value in style.items():
setattr(target, key, value)
title = "Validation {dps} {period}".format(
dps=entity.short_name, period=period.strid)
logger.info(title)
wb = Workbook()
wb.remove_sheet(wb.active)
# write a sheet for each ZS
for zs in entity.get_children():
ws = wb.create_sheet()
ws.title = zs.short_name
def std_write(row, column, value, style=std_style):
cell = ws.cell(row=row, column=column)
cell.value = value
apply_style(cell, style)
# write header
row = 1
std_write(row, col_number, "#", header_style)
xl_set_col_width(ws, col_number, 1.6)
std_write(row, col_indicator, "Indicateur", header_style)
xl_set_col_width(ws, col_indicator, 37.3)
std_write(row, col_numerator, "Numérateur", header_style)
xl_set_col_width(ws, col_numerator, 2.8)
std_write(row, col_denominator, "Dénominateur", header_style)
xl_set_col_width(ws, col_denominator, 3)
std_write(row, col_human, "Valeur.", header_style)
std_write(row, col_valid, "OK ?.", header_style)
std_write(row, col_new_numerator, "Numérateur.", header_style)
xl_set_col_width(ws, col_new_numerator, 2.8)
std_write(row, col_new_denominator, "Dénominateur", header_style)
xl_set_col_width(ws, col_new_denominator, 3)
row += 1
# one row per indicator
for indicator in Indicator.objects.all():
dr = DataRecord.get_or_none(
period=period,
entity=zs,
indicator=indicator,
only_validated=False)
std_write(row, col_number, indicator.number, std_style)
std_write(row, col_indicator, indicator.name, name_style)
std_write(row, col_numerator,
"" if dr is None else dr.numerator, std_style)
std_write(row, col_denominator,
"" if dr is None else dr.denominator, std_style)
std_write(row, col_human,
"" if dr is None else dr.human(), std_style)
std_write(row, col_valid, "", std_style)
std_write(row, col_new_numerator, "", std_style)
std_write(row, col_new_denominator, "", std_style)
row += 1
# apply even/odd style
for r in range(1, row):
if r % 2 == 0:
for c in range(1, last_col + 1):
ws.cell(row=r, column=c).fill = odd_fill
if save_to:
logger.info("saving to {}".format(save_to))
wb.save(save_to)
return
stream = StringIO.StringIO()
wb.save(stream)
return stream
def gen_map_for(entity, periods, indicator, save_as=None,
with_title=True, with_legend=True,
with_index=True, with_scale=True):
children = sorted(entity.get_children(), key=lambda x: x.short_name)
bbox = children_bounds(entity)
# data = indicator.data_for(periods=periods, entity=entity)
data = DataRecord.get_for(periods[-1] if periods else None,
entity, indicator)
# from pprint import pprint as pp ; pp(data.values())
scale = QuantileScale(values=[x['value'] for x in data.values()],
colors=MAP_COLORS)
def color_for(e):
if data is None:
return COLOR_INITIAL
try:
return scale.color_for_value(data[e.uuids]['value'])
except KeyError:
return COLOR_INITIAL
# setup coordinate system
xdist = bbox[2] - bbox[0]
ydist = bbox[3] - bbox[1]
iwidth = CANVAS_SIZE
iheight = CANVAS_SIZE
xratio = iwidth / xdist
yratio = iheight / ydist
def mmap(x, y):
return int(iwidth - ((bbox[2] - x) * xratio)), \
int((bbox[3] - y) * yratio)
# blank (transparent) sheet
image = get_image(CANVAS_SIZE, CANVAS_SIZE)
# draw each feature independently
for index, child in enumerate(children):
if not child.geojson.get('geometry'):
continue
pixels = []
if child.geojson['geometry']['type'] == 'MultiPolygon':
coordinates = child.geojson['geometry']['coordinates']
else:
coordinates = [child.geojson['geometry']['coordinates']]
pixels = [mmap(x, y)
for feature_coord in coordinates
for x, y in feature_coord[0]]
# draw polygon
feature_draw = ImageDraw.Draw(image)
feature_draw.polygon(pixels,
outline=OUTLINE_COLOR,
fill=color_for(child))
# draw thicker boundaries
feature_draw.line(pixels, fill=OUTLINE_COLOR, width=OUTLINE_WIDTH)
for pixel in pixels:
feature_draw.ellipse(
(pixel[0] - OUTLINE_WIDTH // 2,
pixel[1] - OUTLINE_WIDTH // 2,
pixel[0] + OUTLINE_WIDTH // 2,
pixel[1] + OUTLINE_WIDTH // 2),
fill=OUTLINE_COLOR)
# draw a letter
if with_index and child.gps:
coords = mmap(*child.gps)
negpad = CANVAS_SIZE // 80 // 2
index_font = get_font('Regular', CANVAS_SIZE // 80, mono=True)
feature_draw.text(
(coords[0] - negpad, coords[1] - negpad),
letter_for(index),
font=index_font, fill=INDEX_COLOR)
if with_legend:
legend = build_legend_for(scale)
image.paste(legend, (CANVAS_SIZE - LEGEND_PADDING - legend.size[0],
CANVAS_SIZE - LEGEND_PADDING - legend.size[1]),
mask=legend)
if with_title:
title = build_title_for(indicator, periods, entity)
export_width = image.size[0] + title.size[1]
export_height = CANVAS_SIZE + title.size[1]
export_image = get_image(export_width, export_height)
export_image.paste(title, (0, 0), mask=title)
export_image.paste(image, (title.size[1], title.size[1]), mask=image)
# replace main image with new one (larger with title)
image = export_image
if with_index:
index = build_index_for(indicator, periods, children)
ileft_coords = (INDEX_PADDING, title.size[1] + INDEX_PADDING)
iright_coords = (image.size[0] - index.size[0] - INDEX_PADDING,
title.size[1] + INDEX_PADDING)
index_coords = iright_coords if display_right(entity) else ileft_coords
image.paste(index, index_coords,
mask=index)
if with_scale:
visual_scale = build_scale_for(bbox)
image.paste(visual_scale,
(SCALE_PADDING,
image.size[1] - SCALE_PADDING - visual_scale.size[1]),
mask=visual_scale)
# save end result to file or buffer
if save_as:
image.save(save_as, "PNG", mode="RGBA")
else:
return image