예제 #1
0
    def process_indicators(self, namespaces, domain, doc_dict):
        if not doc_dict.get('initial_processing_complete', False):
            return
        xmlns = doc_dict.get('xmlns')
        if not xmlns:
            return

        form_indicator_defs = []
        for namespace in namespaces:
            form_indicator_defs.extend(
                FormIndicatorDefinition.get_all(namespace, domain, xmlns=xmlns)
            )

        if not form_indicator_defs:
            return

        try:
            indicator_form = IndicatorXForm.get_or_create_from_dict(doc_dict)[0]
            indicator_form.update_indicators_in_bulk(
                form_indicator_defs, logger=pillow_eval_logging
            )
        except Exception as e:
            pillow_eval_logging.error(
                "Error creating for MVP Indicator for form %(form_id)s: "
                "%(error)s" % {
                    'form_id': doc_dict['_id'],
                    'error': e,
                },
            )
예제 #2
0
 def get_indicators_by_doc(self, doc_dict):
     indicators = []
     xmlns = doc_dict.get('xmlns')
     domain = doc_dict.get('domain')
     if xmlns and domain:
         namespaces = INDICATOR_CONFIG[domain]
         for namespace in namespaces:
             indicators.extend(FormIndicatorDefinition.get_all(namespace, domain, xmlns=xmlns))
     return indicators
예제 #3
0
 def get_indicators_by_doc(self, doc_dict):
     indicators = []
     xmlns = doc_dict.get('xmlns')
     domain = doc_dict.get('domain')
     if xmlns and domain:
         namespaces = INDICATOR_CONFIG[domain]
         for namespace in namespaces:
             indicators.extend(
                 FormIndicatorDefinition.get_all(namespace,
                                                 domain,
                                                 xmlns=xmlns))
     return indicators
예제 #4
0
    def process_indicators(self, doc_dict, domain, namespaces):
        if not doc_dict.get('initial_processing_complete', False):
            # Make sure we don't update the indicators
            # before the XFormPillows and CasePillows.
            pillow_eval_logging.info(
                "FormIndicatorPillow initial processing "
                "was false for form in %(domain)s, "
                "doc id: %(doc_id)s" % {
                    'domain': domain,
                    'doc_id': doc_dict['_id'],
                })
            return

        pillow_eval_logging.info(
            "FormIndicatorPillow processing started for form in %(domain)s, "
            "doc id: %(doc_id)s" % {
                'domain': domain,
                'doc_id': doc_dict['_id'],
            })

        xmlns = doc_dict.get('xmlns')
        if not xmlns:
            pillow_eval_logging.info(
                '[INDICATOR %(domain)s] Could not find XMLS while '
                'processing indicator for %(xform_id)s' % {
                    'domain': domain,
                    'xform_id': doc_dict['_id'],
                }
            )
            return

        indicators = []
        for namespace in namespaces:
            indicators.extend(
                FormIndicatorDefinition.get_all(namespace, domain, xmlns=xmlns)
            )

        if indicators:
            xform_doc = XFormInstance.get(doc_dict['_id'])
            xform_doc.update_indicators_in_bulk(indicators,
                                                logger=pillow_eval_logging)
        else:
            pillow_eval_logging.info(
                "No indicators could be found for form in %(domain)s, "
                "doc id: %(doc_id)s" % {
                    'domain': domain,
                    'doc_id': doc_dict['_id'],
                }
            )
예제 #5
0
    def update_indicators_for_xmlns(self, domain, form_label_filter=None):
        key = [MVP.NAMESPACE, domain]
        all_labels = (
            FormLabelIndicatorDefinition.get_db()
            .view("indicators/form_labels", reduce=False, startkey=key, endkey=key + [{}])
            .all()
        )
        for label in all_labels:
            label_name = label["value"]
            if form_label_filter is not None and form_label_filter != label_name:
                continue
            xmlns = label["key"][-2]
            print "\n\nGetting Forms of Type %s and XMLNS %s for domain %s" % (label_name, xmlns, domain)

            relevant_forms = (
                XFormInstance.get_db()
                .view(
                    "all_forms/view",
                    reduce=True,
                    startkey=["submission xmlns", domain, xmlns],
                    endkey=["submission xmlns", domain, xmlns, {}],
                )
                .first()
            )
            num_forms = relevant_forms["value"] if relevant_forms else 0

            form_ids = [
                r["id"]
                for r in XFormInstance.view(
                    "all_forms/view",
                    reduce=False,
                    include_docs=False,
                    startkey=["submission xmlns", domain, xmlns],
                    endkey=["submission xmlns", domain, xmlns, {}],
                ).all()
            ]

            print "Found %d forms with matching XMLNS %s" % (num_forms, xmlns)
            relevant_indicators = FormIndicatorDefinition.get_all(namespace=MVP.NAMESPACE, domain=domain, xmlns=xmlns)
            if relevant_indicators:
                self._throttle_updates(
                    "Forms (TYPE: %s, XMLNS %s, DOMAIN: %s)" % (label_name, xmlns, domain),
                    relevant_indicators,
                    num_forms,
                    domain,
                    form_ids,
                    XFormInstance,
                )
예제 #6
0
    def handle(self, *args, **options):
        xform_db = XFormInstance.get_db()

        for domain in get_indicator_domains():
            namespaces = get_namespaces(domain)
            indicators = []
            for namespace in namespaces:
                indicators.extend(FormIndicatorDefinition.get_all(namespace, domain))

            form_ids = get_form_ids_by_type(domain, 'XFormInstance',
                                            start=datetime.date(2013, 8, 1),
                                            end=datetime.date(2013, 10, 15))

            for doc in iter_docs(xform_db, form_ids):
                xfrom_doc = XFormInstance.wrap(doc)
                xfrom_doc.update_indicators_in_bulk(indicators, logger=logging)
예제 #7
0
    def process_indicators(self, doc_dict, domain, namespaces):
        if not doc_dict.get('initial_processing_complete', False):
            # Make sure we don't update the indicators
            # before the XFormPillows and CasePillows.
            pillow_eval_logging.info("FormIndicatorPillow initial processing "
                                     "was false for form in %(domain)s, "
                                     "doc id: %(doc_id)s" % {
                                         'domain': domain,
                                         'doc_id': doc_dict['_id'],
                                     })
            return

        pillow_eval_logging.info(
            "FormIndicatorPillow processing started for form in %(domain)s, "
            "doc id: %(doc_id)s" % {
                'domain': domain,
                'doc_id': doc_dict['_id'],
            })

        xmlns = doc_dict.get('xmlns')
        if not xmlns:
            pillow_eval_logging.info(
                '[INDICATOR %(domain)s] Could not find XMLS while '
                'processing indicator for %(xform_id)s' % {
                    'domain': domain,
                    'xform_id': doc_dict['_id'],
                })
            return

        indicators = []
        for namespace in namespaces:
            indicators.extend(
                FormIndicatorDefinition.get_all(namespace, domain,
                                                xmlns=xmlns))

        if indicators:
            xform_doc = XFormInstance.get(doc_dict['_id'])
            xform_doc.update_indicators_in_bulk(indicators,
                                                logger=pillow_eval_logging)
        else:
            pillow_eval_logging.info(
                "No indicators could be found for form in %(domain)s, "
                "doc id: %(doc_id)s" % {
                    'domain': domain,
                    'doc_id': doc_dict['_id'],
                })
예제 #8
0
    def update_indicators_for_xmlns(self, domain, form_label_filter=None):
        key = [MVP.NAMESPACE, domain]
        all_labels = FormLabelIndicatorDefinition.get_db().view(
            'indicators/form_labels',
            reduce=False,
            startkey=key,
            endkey=key + [{}],
        ).all()
        for label in all_labels:
            label_name = label['value']
            if form_label_filter is not None and form_label_filter != label_name:
                continue
            xmlns = label['key'][-2]
            print("\n\nGetting Forms of Type %s and XMLNS %s for domain %s" % (label_name, xmlns, domain))

            relevant_forms = XFormInstance.get_db().view(
                "all_forms/view",
                reduce=True,
                startkey=['submission xmlns', domain, xmlns],
                endkey=['submission xmlns', domain, xmlns, {}],
            ).first()
            num_forms = relevant_forms['value'] if relevant_forms else 0

            form_ids = [r['id'] for r in XFormInstance.view(
                "all_forms/view",
                reduce=False,
                include_docs=False,
                startkey=['submission xmlns', domain, xmlns],
                endkey=['submission xmlns', domain, xmlns, {}],
            ).all()]

            print("Found %d forms with matching XMLNS %s" % (num_forms, xmlns))
            relevant_indicators = FormIndicatorDefinition.get_all(
                namespace=MVP.NAMESPACE,
                domain=domain,
                xmlns=xmlns
            )
            if relevant_indicators:
                self._throttle_updates(
                    "Forms (TYPE: %s, XMLNS %s, DOMAIN: %s)" % (
                        label_name, xmlns, domain),
                    relevant_indicators, num_forms, domain,
                    form_ids, XFormInstance)
예제 #9
0
    def process_indicators(self, doc_dict, domain, namespaces):
        if not doc_dict.get('inital_processing_complete', False):
            # Make sure we don't update the indicators before the XFormPillows and CasePillows.
            return

        xmlns = doc_dict.get('xmlns')
        if not xmlns:
            pillow_logging.warning('[INDICATOR %(domain)s] Could not find XMLS while '
                                   'processing indicator for %(xform_id)s' % {
                                       'domain': domain,
                                       'xform_id': doc_dict['_id'],
                                   })
            return

        indicators = []
        for namespace in namespaces:
            indicators.extend(FormIndicatorDefinition.get_all(namespace, domain, xmlns=xmlns))

        if indicators:
            xform_doc = XFormInstance.get(doc_dict['_id'])
            xform_doc.update_indicators_in_bulk(indicators, logger=pillow_logging)
예제 #10
0
    def handle(self, *args, **options):
        xform_db = XFormInstance.get_db()

        for domain in get_indicator_domains():
            namespaces = get_namespaces(domain)
            indicators = []
            for namespace in namespaces:
                indicators.extend(
                    FormIndicatorDefinition.get_all(namespace, domain))

            key = [domain, "by_type", "XFormInstance"]
            data = xform_db.view('couchforms/all_submissions_by_domain',
                                 startkey=key + ["2013-08-01"],
                                 endkey=key + ["2013-10-15"],
                                 reduce=False,
                                 include_docs=False).all()
            form_ids = [d['id'] for d in data]

            for doc in iter_docs(xform_db, form_ids):
                xfrom_doc = XFormInstance.wrap(doc)
                xfrom_doc.update_indicators_in_bulk(indicators, logger=logging)
예제 #11
0
    def handle(self, *args, **options):
        xform_db = XFormInstance.get_db()

        for domain in get_indicator_domains():
            namespaces = get_namespaces(domain)
            indicators = []
            for namespace in namespaces:
                indicators.extend(FormIndicatorDefinition.get_all(namespace, domain))

            key = [domain, "by_type", "XFormInstance"]
            data = xform_db.view(
                'couchforms/all_submissions_by_domain',
                startkey=key+["2013-08-01"],
                endkey=key+["2013-10-15"],
                reduce=False,
                include_docs=False
            ).all()
            form_ids = [d['id'] for d in data]

            for doc in iter_docs(xform_db, form_ids):
                xfrom_doc = XFormInstance.wrap(doc)
                xfrom_doc.update_indicators_in_bulk(indicators, logger=logging)
예제 #12
0
    def update_indicators_for_xmlns(self, domain, form_label_filter=None):
        key = [MVP.NAMESPACE, domain]
        all_labels = get_db().view('indicators/form_labels',
            reduce=False,
            startkey=key,
            endkey=key + [{}],
        ).all()
        for label in all_labels:
            label_name = label['value']
            if form_label_filter is not None and form_label_filter != label_name:
                continue
            xmlns = label['key'][-2]
            print "\n\nGetting Forms of Type %s and XMLNS %s for domain %s" % (label_name, xmlns, domain)

            relevant_forms = get_db().view("reports_forms/all_forms",
                reduce=True,
                startkey=['submission xmlns', domain, xmlns],
                endkey=['submission xmlns', domain, xmlns, {}],
            ).first()
            num_forms = relevant_forms['value'] if relevant_forms else 0
            get_forms = lambda skip, limit: XFormInstance.view("reports_forms/all_forms",
                reduce=False,
                include_docs=True,
                startkey=['submission xmlns', domain, xmlns],
                endkey=['submission xmlns', domain, xmlns, {}],
                skip=skip,
                limit=limit
            ).all()

            print "Found %d forms with matching XMLNS %s" % (num_forms, xmlns)
            relevant_indicators = FormIndicatorDefinition.get_all(
                namespace=MVP.NAMESPACE,
                domain=domain,
                xmlns=xmlns
            )
            if relevant_indicators:
                self._throttle_updates("Forms (TYPE: %s, XMLNS %s, DOMAIN: %s)" % (label_name, xmlns, domain),
                                       relevant_indicators, num_forms, domain, get_forms)
예제 #13
0
def get_form_indicators(namespaces, domain, xmlns):
    return [
        indicator for namespace in namespaces
        for indicator in FormIndicatorDefinition.get_all(namespace, domain, xmlns=xmlns)
    ]
예제 #14
0
def get_form_indicators(namespaces, domain, xmlns):
    return [
        indicator for namespace in namespaces for indicator in
        FormIndicatorDefinition.get_all(namespace, domain, xmlns=xmlns)
    ]