def process_indicators(self, namespaces, domain, doc_dict):
if not doc_dict.get('initial_processing_complete', False):
return
xmlns = doc_dict.get('xmlns')
if not xmlns:
return
form_indicator_defs = []
for namespace in namespaces:
form_indicator_defs.extend(
FormIndicatorDefinition.get_all(namespace, domain, xmlns=xmlns)
)
if not form_indicator_defs:
return
try:
indicator_form = IndicatorXForm.get_or_create_from_dict(doc_dict)[0]
indicator_form.update_indicators_in_bulk(
form_indicator_defs, logger=pillow_eval_logging
)
except Exception as e:
pillow_eval_logging.error(
"Error creating for MVP Indicator for form %(form_id)s: "
"%(error)s" % {
'form_id': doc_dict['_id'],
'error': e,
},
)
def get_indicators_by_doc(self, doc_dict):
indicators = []
xmlns = doc_dict.get('xmlns')
domain = doc_dict.get('domain')
if xmlns and domain:
namespaces = INDICATOR_CONFIG[domain]
for namespace in namespaces:
indicators.extend(FormIndicatorDefinition.get_all(namespace, domain, xmlns=xmlns))
return indicators
def get_indicators_by_doc(self, doc_dict):
indicators = []
xmlns = doc_dict.get('xmlns')
domain = doc_dict.get('domain')
if xmlns and domain:
namespaces = INDICATOR_CONFIG[domain]
for namespace in namespaces:
indicators.extend(
FormIndicatorDefinition.get_all(namespace,
domain,
xmlns=xmlns))
return indicators
def process_indicators(self, doc_dict, domain, namespaces):
if not doc_dict.get('initial_processing_complete', False):
# Make sure we don't update the indicators
# before the XFormPillows and CasePillows.
pillow_eval_logging.info(
"FormIndicatorPillow initial processing "
"was false for form in %(domain)s, "
"doc id: %(doc_id)s" % {
'domain': domain,
'doc_id': doc_dict['_id'],
})
return
pillow_eval_logging.info(
"FormIndicatorPillow processing started for form in %(domain)s, "
"doc id: %(doc_id)s" % {
'domain': domain,
'doc_id': doc_dict['_id'],
})
xmlns = doc_dict.get('xmlns')
if not xmlns:
pillow_eval_logging.info(
'[INDICATOR %(domain)s] Could not find XMLS while '
'processing indicator for %(xform_id)s' % {
'domain': domain,
'xform_id': doc_dict['_id'],
}
)
return
indicators = []
for namespace in namespaces:
indicators.extend(
FormIndicatorDefinition.get_all(namespace, domain, xmlns=xmlns)
)
if indicators:
xform_doc = XFormInstance.get(doc_dict['_id'])
xform_doc.update_indicators_in_bulk(indicators,
logger=pillow_eval_logging)
else:
pillow_eval_logging.info(
"No indicators could be found for form in %(domain)s, "
"doc id: %(doc_id)s" % {
'domain': domain,
'doc_id': doc_dict['_id'],
}
)
def update_indicators_for_xmlns(self, domain, form_label_filter=None):
key = [MVP.NAMESPACE, domain]
all_labels = (
FormLabelIndicatorDefinition.get_db()
.view("indicators/form_labels", reduce=False, startkey=key, endkey=key + [{}])
.all()
)
for label in all_labels:
label_name = label["value"]
if form_label_filter is not None and form_label_filter != label_name:
continue
xmlns = label["key"][-2]
print "\n\nGetting Forms of Type %s and XMLNS %s for domain %s" % (label_name, xmlns, domain)
relevant_forms = (
XFormInstance.get_db()
.view(
"all_forms/view",
reduce=True,
startkey=["submission xmlns", domain, xmlns],
endkey=["submission xmlns", domain, xmlns, {}],
)
.first()
)
num_forms = relevant_forms["value"] if relevant_forms else 0
form_ids = [
r["id"]
for r in XFormInstance.view(
"all_forms/view",
reduce=False,
include_docs=False,
startkey=["submission xmlns", domain, xmlns],
endkey=["submission xmlns", domain, xmlns, {}],
).all()
]
print "Found %d forms with matching XMLNS %s" % (num_forms, xmlns)
relevant_indicators = FormIndicatorDefinition.get_all(namespace=MVP.NAMESPACE, domain=domain, xmlns=xmlns)
if relevant_indicators:
self._throttle_updates(
"Forms (TYPE: %s, XMLNS %s, DOMAIN: %s)" % (label_name, xmlns, domain),
relevant_indicators,
num_forms,
domain,
form_ids,
XFormInstance,
)
def handle(self, *args, **options):
xform_db = XFormInstance.get_db()
for domain in get_indicator_domains():
namespaces = get_namespaces(domain)
indicators = []
for namespace in namespaces:
indicators.extend(FormIndicatorDefinition.get_all(namespace, domain))
form_ids = get_form_ids_by_type(domain, 'XFormInstance',
start=datetime.date(2013, 8, 1),
end=datetime.date(2013, 10, 15))
for doc in iter_docs(xform_db, form_ids):
xfrom_doc = XFormInstance.wrap(doc)
xfrom_doc.update_indicators_in_bulk(indicators, logger=logging)
def process_indicators(self, doc_dict, domain, namespaces):
if not doc_dict.get('initial_processing_complete', False):
# Make sure we don't update the indicators
# before the XFormPillows and CasePillows.
pillow_eval_logging.info("FormIndicatorPillow initial processing "
"was false for form in %(domain)s, "
"doc id: %(doc_id)s" % {
'domain': domain,
'doc_id': doc_dict['_id'],
})
return
pillow_eval_logging.info(
"FormIndicatorPillow processing started for form in %(domain)s, "
"doc id: %(doc_id)s" % {
'domain': domain,
'doc_id': doc_dict['_id'],
})
xmlns = doc_dict.get('xmlns')
if not xmlns:
pillow_eval_logging.info(
'[INDICATOR %(domain)s] Could not find XMLS while '
'processing indicator for %(xform_id)s' % {
'domain': domain,
'xform_id': doc_dict['_id'],
})
return
indicators = []
for namespace in namespaces:
indicators.extend(
FormIndicatorDefinition.get_all(namespace, domain,
xmlns=xmlns))
if indicators:
xform_doc = XFormInstance.get(doc_dict['_id'])
xform_doc.update_indicators_in_bulk(indicators,
logger=pillow_eval_logging)
else:
pillow_eval_logging.info(
"No indicators could be found for form in %(domain)s, "
"doc id: %(doc_id)s" % {
'domain': domain,
'doc_id': doc_dict['_id'],
})
def update_indicators_for_xmlns(self, domain, form_label_filter=None):
key = [MVP.NAMESPACE, domain]
all_labels = FormLabelIndicatorDefinition.get_db().view(
'indicators/form_labels',
reduce=False,
startkey=key,
endkey=key + [{}],
).all()
for label in all_labels:
label_name = label['value']
if form_label_filter is not None and form_label_filter != label_name:
continue
xmlns = label['key'][-2]
print("\n\nGetting Forms of Type %s and XMLNS %s for domain %s" % (label_name, xmlns, domain))
relevant_forms = XFormInstance.get_db().view(
"all_forms/view",
reduce=True,
startkey=['submission xmlns', domain, xmlns],
endkey=['submission xmlns', domain, xmlns, {}],
).first()
num_forms = relevant_forms['value'] if relevant_forms else 0
form_ids = [r['id'] for r in XFormInstance.view(
"all_forms/view",
reduce=False,
include_docs=False,
startkey=['submission xmlns', domain, xmlns],
endkey=['submission xmlns', domain, xmlns, {}],
).all()]
print("Found %d forms with matching XMLNS %s" % (num_forms, xmlns))
relevant_indicators = FormIndicatorDefinition.get_all(
namespace=MVP.NAMESPACE,
domain=domain,
xmlns=xmlns
)
if relevant_indicators:
self._throttle_updates(
"Forms (TYPE: %s, XMLNS %s, DOMAIN: %s)" % (
label_name, xmlns, domain),
relevant_indicators, num_forms, domain,
form_ids, XFormInstance)
def process_indicators(self, doc_dict, domain, namespaces):
if not doc_dict.get('inital_processing_complete', False):
# Make sure we don't update the indicators before the XFormPillows and CasePillows.
return
xmlns = doc_dict.get('xmlns')
if not xmlns:
pillow_logging.warning('[INDICATOR %(domain)s] Could not find XMLS while '
'processing indicator for %(xform_id)s' % {
'domain': domain,
'xform_id': doc_dict['_id'],
})
return
indicators = []
for namespace in namespaces:
indicators.extend(FormIndicatorDefinition.get_all(namespace, domain, xmlns=xmlns))
if indicators:
xform_doc = XFormInstance.get(doc_dict['_id'])
xform_doc.update_indicators_in_bulk(indicators, logger=pillow_logging)
def handle(self, *args, **options):
xform_db = XFormInstance.get_db()
for domain in get_indicator_domains():
namespaces = get_namespaces(domain)
indicators = []
for namespace in namespaces:
indicators.extend(
FormIndicatorDefinition.get_all(namespace, domain))
key = [domain, "by_type", "XFormInstance"]
data = xform_db.view('couchforms/all_submissions_by_domain',
startkey=key + ["2013-08-01"],
endkey=key + ["2013-10-15"],
reduce=False,
include_docs=False).all()
form_ids = [d['id'] for d in data]
for doc in iter_docs(xform_db, form_ids):
xfrom_doc = XFormInstance.wrap(doc)
xfrom_doc.update_indicators_in_bulk(indicators, logger=logging)
def handle(self, *args, **options):
xform_db = XFormInstance.get_db()
for domain in get_indicator_domains():
namespaces = get_namespaces(domain)
indicators = []
for namespace in namespaces:
indicators.extend(FormIndicatorDefinition.get_all(namespace, domain))
key = [domain, "by_type", "XFormInstance"]
data = xform_db.view(
'couchforms/all_submissions_by_domain',
startkey=key+["2013-08-01"],
endkey=key+["2013-10-15"],
reduce=False,
include_docs=False
).all()
form_ids = [d['id'] for d in data]
for doc in iter_docs(xform_db, form_ids):
xfrom_doc = XFormInstance.wrap(doc)
xfrom_doc.update_indicators_in_bulk(indicators, logger=logging)
def update_indicators_for_xmlns(self, domain, form_label_filter=None):
key = [MVP.NAMESPACE, domain]
all_labels = get_db().view('indicators/form_labels',
reduce=False,
startkey=key,
endkey=key + [{}],
).all()
for label in all_labels:
label_name = label['value']
if form_label_filter is not None and form_label_filter != label_name:
continue
xmlns = label['key'][-2]
print "\n\nGetting Forms of Type %s and XMLNS %s for domain %s" % (label_name, xmlns, domain)
relevant_forms = get_db().view("reports_forms/all_forms",
reduce=True,
startkey=['submission xmlns', domain, xmlns],
endkey=['submission xmlns', domain, xmlns, {}],
).first()
num_forms = relevant_forms['value'] if relevant_forms else 0
get_forms = lambda skip, limit: XFormInstance.view("reports_forms/all_forms",
reduce=False,
include_docs=True,
startkey=['submission xmlns', domain, xmlns],
endkey=['submission xmlns', domain, xmlns, {}],
skip=skip,
limit=limit
).all()
print "Found %d forms with matching XMLNS %s" % (num_forms, xmlns)
relevant_indicators = FormIndicatorDefinition.get_all(
namespace=MVP.NAMESPACE,
domain=domain,
xmlns=xmlns
)
if relevant_indicators:
self._throttle_updates("Forms (TYPE: %s, XMLNS %s, DOMAIN: %s)" % (label_name, xmlns, domain),
relevant_indicators, num_forms, domain, get_forms)
def get_form_indicators(namespaces, domain, xmlns):
return [
indicator for namespace in namespaces
for indicator in FormIndicatorDefinition.get_all(namespace, domain, xmlns=xmlns)
]
def get_form_indicators(namespaces, domain, xmlns):
return [
indicator for namespace in namespaces for indicator in
FormIndicatorDefinition.get_all(namespace, domain, xmlns=xmlns)
]