class FormsInDateExpressionSpec(JsonObject): type = TypeProperty('icds_get_case_forms_in_date') case_id_expression = DefaultProperty(required=True) xmlns = ListProperty(required=False) from_date_expression = DictProperty(required=False) to_date_expression = DictProperty(required=False) count = BooleanProperty(default=False) def configure(self, case_id_expression, from_date_expression=None, to_date_expression=None): self._case_id_expression = case_id_expression self._from_date_expression = from_date_expression self._to_date_expression = to_date_expression def __call__(self, item, context=None): case_id = self._case_id_expression(item, context) if self._from_date_expression: from_date = self._from_date_expression(item, context) else: from_date = None if self._to_date_expression: to_date = self._to_date_expression(item, context) else: to_date = None if not case_id: return [] assert context.root_doc['domain'] return self._get_forms(case_id, from_date, to_date, context) def _get_forms(self, case_id, from_date, to_date, context): domain = context.root_doc['domain'] xmlns_tuple = tuple(self.xmlns) cache_key = (self.__class__.__name__, case_id, self.count, from_date, to_date, xmlns_tuple) if context.get_cache_value(cache_key) is not None: return context.get_cache_value(cache_key) xform_ids = self._get_case_form_ids(case_id, context) # TODO(Emord) this will eventually break down when cases have a lot of # forms associated with them. perhaps change to intersecting two sets xforms = self._get_filtered_forms_from_es(case_id, xform_ids, context) if self.xmlns: xforms = [x for x in xforms if x['xmlns'] in xmlns_tuple] if from_date: xforms = [x for x in xforms if x['timeEnd'] >= from_date] if to_date: xforms = [x for x in xforms if x['timeEnd'] <= to_date] if self.count: count = len(xforms) context.set_cache_value(cache_key, count) return count form_ids = [x['_id'] for x in xforms] xforms = FormAccessors(domain).get_forms(form_ids) xforms = [ self._get_form_json(f, context) for f in xforms if f.domain == domain ] context.set_cache_value(cache_key, xforms) return xforms def _get_filtered_forms_from_es(self, case_id, xform_ids, context): cache_key = (self.__class__.__name__, 'es_helper', case_id, tuple(xform_ids)) if context.get_cache_value(cache_key) is not None: return context.get_cache_value(cache_key) def _transform_time_end(xform): xform = xform.get('_source', {}) if not xform.get('xmlns', None): return None try: time = xform['form']['meta']['timeEnd'] except KeyError: return None xform['timeEnd'] = datetime.strptime( time, '%Y-%m-%dT%H:%M:%S.%fZ').date() return xform forms = mget_query('forms', xform_ids, ['form.meta.timeEnd', 'xmlns', '_id']) forms = filter(None, map(_transform_time_end, forms)) context.set_cache_value(cache_key, forms) return forms def _get_case_form_ids(self, case_id, context): cache_key = (self.__class__.__name__, 'helper', case_id) if context.get_cache_value(cache_key) is not None: return context.get_cache_value(cache_key) domain = context.root_doc['domain'] xform_ids = CaseAccessors(domain).get_case_xform_ids(case_id) context.set_cache_value(cache_key, xform_ids) return xform_ids def _get_form_json(self, form, context): cache_key = (XFORM_CACHE_KEY_PREFIX, form.get_id) if context.get_cache_value(cache_key) is not None: return context.get_cache_value(cache_key) form_json = form.to_json() context.set_cache_value(cache_key, form_json) return form_json
class OpenMaleResidentSpec(SumWhenTemplateSpec): type = TypeProperty("open_male_resident") expression = "closed_on IS NULL AND sex IN ('M', 'O') AND resident = 1"
class ReachedReferralHealthProblem5ProblemsSpec(SumWhenTemplateSpec): type = TypeProperty("reached_referral_health_problem_5_problems") expression = "referral_reached_facility = ? AND (referral_health_problem ~ ? OR referral_health_problem ~ ? OR referral_health_problem ~ ? OR referral_health_problem ~ ? OR referral_health_problem ~ ?)"
class OpenMaleHHCasteSpec(SumWhenTemplateSpec): type = TypeProperty("open_male_hh_caste") expression = "closed_on IS NULL AND sex IN ('M', 'O') and hh_caste = ?"
class OpenMaleHHMinoritySpec(SumWhenTemplateSpec): type = TypeProperty("open_male_hh_minority") expression = "closed_on IS NULL AND sex in ('M', 'O') and hh_minority = 1"
class OpenFemaleHHCasteNotSpec(SumWhenTemplateSpec): type = TypeProperty("open_female_hh_caste_not") expression = "closed_on IS NULL AND sex = 'F' and hh_caste NOT IN (?, ?)"
class OpenFemaleResidentSpec(SumWhenTemplateSpec): type = TypeProperty("open_female_resident") expression = "closed_on IS NULL AND sex = 'F' AND resident = 1"
class QuarterFilterSpec(FilterSpec): type = TypeProperty('quarter') show_all = BooleanProperty(default=False)
class ChoiceListFilterSpec(FilterSpec): type = TypeProperty('choice_list') show_all = BooleanProperty(default=True) datatype = DataTypeProperty(default='string') choices = ListProperty(FilterChoice)
class PreFilterSpec(FilterSpec): type = TypeProperty('pre') pre_value = DefaultProperty(required=True) pre_operator = StringProperty(default=None, required=False)
class NumericFilterSpec(FilterSpec): type = TypeProperty('numeric')
class LocationDrilldownFilterSpec(FilterSpec): type = TypeProperty('location_drilldown') include_descendants = BooleanProperty(default=False) # default to some random high number '99' max_drilldown_levels = IntegerProperty(default=99) ancestor_expression = DictProperty(default={}, required=False)
class MultiFieldDynamicChoiceFilterSpec(DynamicChoiceListFilterSpec): type = TypeProperty('multi_field_dynamic_choice_list') fields = ListProperty(default=[])
class AbtSupervisorExpressionSpec(JsonObject): type = TypeProperty('abt_supervisor') @property @memoized def _flag_specs(self): """ Return a dict where keys are form xmlns and values are lists of FlagSpecs """ path = os.path.join(os.path.dirname(__file__), 'flagspecs.yaml') with open(path) as f: return yaml.load(f) @classmethod def _get_val(cls, item, path): """ Return the answer in the given submitted form (item) to the question specified by path. Return empty tuple if no answer was given to the given question. """ if path: try: v = item['form'] for key in path: v = v[key] return v except KeyError: return () @classmethod def _question_answered(cls, value): """ Return true if the given value indicates that an answer was provided for its question. """ return value != () @classmethod def _raise_for_any_answer(cls, danger_value): """ Return true if the given danger_value indicates that any question answer should raise the flag. """ return danger_value == [] @classmethod @quickcache(['app_id', 'xmlns', 'lang']) def _get_questions(cls, app_id, xmlns, lang): questions = Application.get(app_id).get_questions(xmlns, [lang], include_groups=True) return {q['value']: q for q in questions} @classmethod def _get_question_options(cls, item, question_path): """ Return a list of option values for the given question path and item (which is a dict representation of an XFormInstance) """ questions = cls._get_questions(item['app_id'], item['xmlns'], cls._get_language(item)) question = questions.get('/data/' + "/".join(question_path), {}) return question.get("options", []) @classmethod def _get_unchecked(cls, xform_instance, question_path, answer, ignore=None): """ Return the unchecked options in the given question. Do not return any which appear in the option ignore parameter. answer should be a string ignore should be a list of strings. """ answer = answer or "" options = { o['value']: o['label'] for o in cls._get_question_options(xform_instance, question_path) } checked = set(answer.split(" ")) unchecked = set(options.keys()) - checked relevant_unchecked = unchecked - set(ignore) return [options[u] for u in relevant_unchecked] @classmethod def _get_comments(cls, item, spec): """ Return the comments for the question specified in the spec. If the spec does not contain a `comment` field, then the `question` field is used to build the path to the comment question. """ comments_question = spec.get('comment', False) question_path = spec["question"] if not comments_question: parts = question_path[-1].split("_") parts.insert(1, "comments") comments_question = question_path[:-1] + ["_".join(parts)] comments = cls._get_val(item, comments_question) return comments if comments != () else "" @classmethod def _get_language(cls, item): """ Return the language in which this row should be rendered. """ if item.get("domain", None) in ("airsmadagascar", "abtmali"): return "fra" country = cls._get_val(item, ["location_data", "country"]) if country in [ "Senegal", 'S\xe9n\xe9gal', "Benin", "Mali", "Madagascar" ]: return "fra" elif country in ["mozambique", "Mozambique"]: return "por" return "en" @classmethod def _get_warning(cls, spec, item): default = six.text_type(spec.get("warning", "")) language = cls._get_language(item) warning_key_map = { "fra": "warning_fr", "por": "warning_por", "en": "warning" } warning = six.text_type(spec.get(warning_key_map[language], default)) return warning if warning else default @classmethod def _get_inspector_names(cls, item): repeat_items = cls._get_val(item, ['employee_group', 'employee']) if repeat_items == (): return "" if type(repeat_items) != list: repeat_items = [repeat_items] repeat_items = [{'form': x} for x in repeat_items] names = [] for i in repeat_items: for q in [ 'other_abt_employee_name', 'abt_employee_name', 'other_non-abt_employee_name' ]: name = cls._get_val(i, ['abt_emp_list', q]) if name: names.append(name) break return ", ".join(names) @classmethod def _get_flag_name(cls, item, spec): """ Return value that should be in the flag column. Defaults to the question id if spec doesn't specify something else. """ default = spec['question'][-1] flag_name_key_map = { "fra": "flag_name_fr", "por": "flag_name_por", "en": "flag_name", } lang = cls._get_language(item) name = spec.get(flag_name_key_map[lang], None) return name if name else default def __call__(self, item, context=None): """ Given a document (item), return a list of documents representing each of the flagged questions. """ names = self._get_inspector_names(item) docs = [] for spec in self._flag_specs.get(item['xmlns'], []): if spec.get("base_path", False): repeat_items = self._get_val(item, spec['base_path']) if repeat_items == (): repeat_items = [] if type(repeat_items) != list: # bases will be a dict if the repeat only happened once. repeat_items = [repeat_items] # We have to add the 'form' key here so that _get_val works correctly. repeat_items = [{'form': x} for x in repeat_items] else: repeat_items = [item] # Iterate over the repeat items, or the single submission for partial in repeat_items: form_value = self._get_val(partial, spec['question']) warning_type = spec.get("warning_type", None) if warning_type == "unchecked" and form_value: # Don't raise flag if no answer given ignore = spec.get("ignore", []) unchecked = self._get_unchecked( item, spec.get('base_path', []) + spec['question'], form_value, ignore) if unchecked: # Raise a flag because there are unchecked answers. docs.append({ 'flag': self._get_flag_name(item, spec), 'warning': self._get_warning( spec, item).format(msg=", ".join(unchecked)), 'comments': self._get_comments(partial, spec), 'names': names, }) elif warning_type == "q3_special" and form_value: # One of the questions doesn't follow the same format as the # others, hence this special case. missing_items = "" if form_value == "only_license": missing_items = "cell" if form_value == "only_cell": missing_items = "license" if form_value == "none": missing_items = "cell, license" if missing_items: docs.append({ 'flag': self._get_flag_name(item, spec), 'warning': self._get_warning(spec, item).format(msg=missing_items), 'comments': self._get_comments(partial, spec), 'names': names, }) else: danger_value = spec.get('answer', []) if form_value == danger_value or ( self._question_answered(form_value) and self._raise_for_any_answer(danger_value)): docs.append({ 'flag': self._get_flag_name(item, spec), 'warning': self._get_warning( spec, item).format(msg=self._get_val( partial, spec.get('warning_question', None)) or ""), 'comments': self._get_comments(partial, spec), 'names': names, }) return docs
class MultibarAggregateChartSpec(ChartSpec): type = TypeProperty('multibar-aggregate') primary_aggregation = StringProperty(required=True) secondary_aggregation = StringProperty(required=True) value_column = StringProperty(required=True)
class FormsInDateExpressionSpec(JsonObject): type = TypeProperty('icds_get_case_forms_in_date') case_id_expression = DefaultProperty(required=True) xmlns = ListProperty(required=False) from_date_expression = DictProperty(required=False) to_date_expression = DictProperty(required=False) count = BooleanProperty(default=False) def configure(self, case_id_expression, from_date_expression=None, to_date_expression=None): self._case_id_expression = case_id_expression self._from_date_expression = from_date_expression self._to_date_expression = to_date_expression def __call__(self, item, context=None): case_id = self._case_id_expression(item, context) if self._from_date_expression: from_date = self._from_date_expression(item, context) else: from_date = None if self._to_date_expression: to_date = self._to_date_expression(item, context) else: to_date = None if not case_id: return [] assert context.root_doc['domain'] return self._get_forms(case_id, from_date, to_date, context) def _get_forms(self, case_id, from_date, to_date, context): domain = context.root_doc['domain'] xmlns_tuple = tuple(self.xmlns) cache_key = (self.__class__.__name__, case_id, self.count, from_date, to_date, xmlns_tuple) if context.get_cache_value(cache_key) is not None: return context.get_cache_value(cache_key) xform_ids = FormsInDateExpressionSpec._get_case_form_ids( case_id, context) # TODO(Emord) this will eventually break down when cases have a lot of # forms associated with them. perhaps change to intersecting two sets xforms = FormsInDateExpressionSpec._get_filtered_forms_from_es( case_id, xform_ids, context) if self.xmlns: xforms = [x for x in xforms if x['xmlns'] in xmlns_tuple] if from_date: xforms = [x for x in xforms if x['timeEnd'] >= from_date] if to_date: xforms = [x for x in xforms if x['timeEnd'] <= to_date] if self.count: count = len(xforms) context.set_cache_value(cache_key, count) return count if not ICDS_UCR_ELASTICSEARCH_DOC_LOADING.enabled( case_id, NAMESPACE_OTHER): form_ids = [x['_id'] for x in xforms] xforms = FormAccessors(domain).get_forms(form_ids) xforms = FormsInDateExpressionSpec._get_form_json_list( case_id, xforms, context, domain) context.set_cache_value(cache_key, xforms) return xforms @staticmethod def _get_filtered_forms_from_es(case_id, xform_ids, context): es_toggle_enabled = ICDS_UCR_ELASTICSEARCH_DOC_LOADING.enabled( case_id, NAMESPACE_OTHER) cache_key = (FormsInDateExpressionSpec.__name__, 'es_helper', case_id, tuple(xform_ids), es_toggle_enabled) if context.get_cache_value(cache_key) is not None: return context.get_cache_value(cache_key) source = True if es_toggle_enabled else [ 'form.meta.timeEnd', 'xmlns', '_id' ] forms = FormsInDateExpressionSpec._bulk_get_forms_from_elasticsearch( xform_ids, source) context.set_cache_value(cache_key, forms) return forms @staticmethod def _get_case_form_ids(case_id, context): cache_key = (FormsInDateExpressionSpec.__name__, 'helper', case_id) if context.get_cache_value(cache_key) is not None: return context.get_cache_value(cache_key) domain = context.root_doc['domain'] xform_ids = CaseAccessors(domain).get_case_xform_ids(case_id) context.set_cache_value(cache_key, xform_ids) return xform_ids @staticmethod def _get_form_json_list(case_id, xforms, context, domain): domain_filtered_forms = [f for f in xforms if f.domain == domain] return [ FormsInDateExpressionSpec._get_form_json(f, context) for f in domain_filtered_forms ] @staticmethod def _get_form_json(form, context): cached_form = FormsInDateExpressionSpec._get_cached_form_json( form, context) if cached_form is not None: return cached_form form_json = form.to_json() FormsInDateExpressionSpec._set_cached_form_json( form, form_json, context) return form_json @staticmethod def _bulk_get_form_json_from_es(forms): form_ids = [form.form_id for form in forms] es_forms = FormsInDateExpressionSpec._bulk_get_forms_from_elasticsearch( form_ids, source=True) return {f['_id']: f for f in es_forms} @staticmethod def _get_cached_form_json(form, context): return context.get_cache_value( FormsInDateExpressionSpec._get_form_json_cache_key(form)) @staticmethod def _set_cached_form_json(form, form_json, context): context.set_cache_value( FormsInDateExpressionSpec._get_form_json_cache_key(form), form_json) @staticmethod def _get_form_json_cache_key(form): return (XFORM_CACHE_KEY_PREFIX, form.form_id) @staticmethod def _bulk_get_forms_from_elasticsearch(form_ids, source): forms = mget_query('forms', form_ids, source) return list( filter(None, [ FormsInDateExpressionSpec. _transform_time_end_and_filter_bad_data(f) for f in forms ])) @staticmethod def _transform_time_end_and_filter_bad_data(xform): xform = xform.get('_source', {}) if not xform.get('xmlns', None): return None try: time = xform['form']['meta']['timeEnd'] except KeyError: return None xform['timeEnd'] = datetime.strptime(time, '%Y-%m-%dT%H:%M:%S.%fZ').date() return xform def __str__(self): value = "case_forms[{case_id}]".format( case_id=self._case_id_expression) if self.from_date_expression or self.to_date_expression: value = "{value}[date={start} to {end}]".format( value=value, start=self._from_date_expression, end=self._to_date_expression) if self.xmlns: value = "{value}[xmlns=\n{xmlns}\n]".format( value=value, xmlns=add_tabbed_text("\n".join(self.xmlns))) if self.count: value = "count({value})".format(value=value) return value
class OpenFemaleDisabledSpec(SumWhenTemplateSpec): type = TypeProperty("open_female_disabled") expression = "closed_on IS NULL AND sex = 'F' and disabled = 1"
class NotFilterSpec(BaseFilterSpec): type = TypeProperty('not') filter = DictProperty() # todo: validators=FilterFactory.validate_spec
class OpenFemaleHHMinoritySpec(SumWhenTemplateSpec): type = TypeProperty("open_female_hh_minority") expression = "closed_on IS NULL AND sex = 'F' and hh_minority = 1"
class NamedFilterSpec(BaseFilterSpec): type = TypeProperty('named') name = StringProperty(required=True)
class OpenMaleDisabledSpec(SumWhenTemplateSpec): type = TypeProperty("open_male_disabled") expression = "closed_on IS NULL AND sex IN ('M', 'O') and disabled = 1"
class FieldColumn(ReportColumn): type = TypeProperty('field') field = StringProperty(required=True) aggregation = StringProperty( choices=list(SQLAGG_COLUMN_MAP), required=True, ) format = StringProperty(default='default', choices=[ 'default', 'percent_of_total', ]) sortable = BooleanProperty(default=False) width = StringProperty(default=None, required=False) css_class = StringProperty(default=None, required=False) @classmethod def wrap(cls, obj): # lazy migrations for legacy data. # todo: remove once all reports are on new format # 1. set column_id to alias, or field if no alias found _add_column_id_if_missing(obj) # 2. if aggregation='expand' convert to ExpandedColumn if obj.get('aggregation') == 'expand': del obj['aggregation'] obj['type'] = 'expanded' return ExpandedColumn.wrap(obj) return super(FieldColumn, cls).wrap(obj) def format_data(self, data): if self.format == 'percent_of_total': column_name = self.column_id total = sum(row[column_name] for row in data) for row in data: row[column_name] = '{:.0%}'.format( row[column_name] / total ) def get_column_config(self, data_source_config, lang): return ColumnConfig(columns=[ DatabaseColumn( header=self.get_header(lang), agg_column=SQLAGG_COLUMN_MAP[self.aggregation](self.field, alias=self.column_id), sortable=self.sortable, data_slug=self.column_id, format_fn=self.get_format_fn(), help_text=self.description, visible=self.visible, width=self.width, css_class=self.css_class, ) ]) def get_fields(self, data_source_config=None, lang=None): return [self.field] def _data_source_col_config(self, data_source_config): return filter( lambda c: c['column_id'] == self.field, data_source_config.configured_indicators )[0] def _column_data_type(self, data_source_config): return self._data_source_col_config(data_source_config).get('datatype') def _use_terms_aggregation_for_max_min(self, data_source_config): return ( self.aggregation in ['max', 'min'] and self._column_data_type(data_source_config) and self._column_data_type(data_source_config) not in ['integer', 'decimal'] ) def get_query_column_ids(self): return [self.column_id]
class OpenMaleHHCasteNotSpec(SumWhenTemplateSpec): type = TypeProperty("open_male_hh_caste_not") expression = "closed_on IS NULL AND sex in ('M', 'O') and hh_caste NOT IN (?, ?)"
class AgeInMonthsBucketsColumn(IntegerBucketsColumn): type = TypeProperty('age_in_months_buckets') def _base_expression(self, bounds): return "extract(year from age({}))*12 + extract(month from age({})) BETWEEN {} and {}".format( self.field, self.field, bounds[0], bounds[1])
class OpenMaleMigrantDistinctFromSpec(SumWhenTemplateSpec): type = TypeProperty("open_male_migrant_distinct_from") expression = "closed_on IS NULL AND sex IN ('M', 'O') AND resident IS DISTINCT FROM 1"
class PercentageColumn(ReportColumn): type = TypeProperty('percent') numerator = ObjectProperty(FieldColumn, required=True) denominator = ObjectProperty(FieldColumn, required=True) format = StringProperty( choices=['percent', 'fraction', 'both', 'numeric_percent', 'decimal'], default='percent' ) def get_column_config(self, data_source_config, lang): # todo: better checks that fields are not expand num_config = self.numerator.get_column_config(data_source_config, lang) denom_config = self.denominator.get_column_config(data_source_config, lang) return ColumnConfig(columns=[ AggregateColumn( header=self.get_header(lang), aggregate_fn=lambda n, d: {'num': n, 'denom': d}, format_fn=self.get_format_fn(), columns=[c.view for c in num_config.columns + denom_config.columns], slug=self.column_id, data_slug=self.column_id, )], warnings=num_config.warnings + denom_config.warnings, ) def get_format_fn(self): NO_DATA_TEXT = '--' CANT_CALCULATE_TEXT = '?' class NoData(Exception): pass class BadData(Exception): pass def trap_errors(fn): def inner(*args, **kwargs): try: return fn(*args, **kwargs) except BadData: return CANT_CALCULATE_TEXT except NoData: return NO_DATA_TEXT return inner def _raw(data): if data['denom']: try: return float(round(data['num'] / data['denom'], 3)) except (ValueError, TypeError): raise BadData() else: raise NoData() def _raw_pct(data, round_type=float): return round_type(_raw(data) * 100) @trap_errors def _clean_raw(data): return _raw(data) @trap_errors def _numeric_pct(data): return _raw_pct(data, round_type=int) @trap_errors def _pct(data): return '{0:.0f}%'.format(_raw_pct(data)) _fraction = lambda data: '{num}/{denom}'.format(**data) return { 'percent': _pct, 'fraction': _fraction, 'both': lambda data: '{} ({})'.format(_pct(data), _fraction(data)), 'numeric_percent': _numeric_pct, 'decimal': _clean_raw, }[self.format] def get_column_ids(self): # override this to include the columns for the numerator and denominator as well return [self.column_id, self.numerator.column_id, self.denominator.column_id] def get_fields(self, data_source_config=None, lang=None): return self.numerator.get_fields() + self.denominator.get_fields()
class OpenPregnantResidentSpec(SumWhenTemplateSpec): type = TypeProperty("open_pregnant_resident") expression = "closed_on IS NULL AND is_pregnant = 1 and sex = 'F' AND resident = 1"
class PieChartSpec(ChartSpec): type = TypeProperty('pie') aggregation_column = StringProperty() value_column = StringProperty(required=True)
class ReferralHealthProblem2ProblemsSpec(SumWhenTemplateSpec): type = TypeProperty("referral_health_problem_2_problems") expression = "referral_health_problem ~ ? OR referral_health_problem ~ ?"
class LocationParentIdSpec(JsonObject): type = TypeProperty('location_parent_id') location_id_expression = DictProperty(required=True)