def get_all_values(self, doc, eval_context=None):
if not eval_context:
eval_context = EvaluationContext(doc)
rows = []
for item in self.get_items(doc, eval_context):
indicators = self.indicators.get_values(item, eval_context)
rows.append(indicators)
eval_context.increment_iteration()
return rows
def setUpClass(cls):
super(TestLocationParentIdExpression, cls).setUpClass()
cls.evaluation_context = EvaluationContext({"domain": cls.domain})
cls.expression_spec = {
"type": "location_parent_id",
"location_id_expression": {
"type": "property_name",
"property_name": "location_id",
}
}
cls.expression = ExpressionFactory.from_spec(cls.expression_spec)
def _save_document_helper(indicator, doc):
eval_context = EvaluationContext(doc)
something_failed = False
configs_to_remove = []
configs = dict()
for config_id in indicator.indicator_config_ids:
try:
configs[config_id] = _get_config(config_id)
except (ResourceNotFound, StaticDataSourceConfigurationNotFoundError):
celery_task_logger.info(
"{} no longer exists, skipping".format(config_id))
configs_to_remove.append(config_id)
continue
except ESError:
celery_task_logger.info(
"ES errored when trying to retrieve config")
something_failed = True
continue
for config_id, config in six.iteritems(configs):
adapter = None
try:
adapter = get_indicator_adapter(config, can_handle_laboratory=True)
adapter.save(doc, eval_context)
eval_context.reset_iteration()
except (ProtocolError, ReadTimeout):
celery_task_logger.info(
"Riak error when saving config: {}".format(config_id))
something_failed = True
except RequestError:
celery_task_logger.info(
"Couch error when saving config: {}".format(config_id))
something_failed = True
except (ESError, ConnectionTimeout):
# a database had an issue so log it and go on to the next document
celery_task_logger.info(
"ES error when saving config: {}".format(config_id))
something_failed = True
except (DatabaseError, InternalError):
# a database had an issue so log it and go on to the next document
celery_task_logger.info(
"psql error when saving config: {}".format(config_id))
something_failed = True
except Exception as e:
# getting the config could fail before the adapter is set
if adapter:
adapter.handle_exception(doc, e)
something_failed = True
else:
configs_to_remove.append(config_id)
rebuild_related_docs = any(config.icds_rebuild_related_docs
for config in six.itervalues(configs) if config)
return (not something_failed, configs_to_remove, rebuild_related_docs)
def validate_document(self, document, eval_context=None):
if eval_context is None:
eval_context = EvaluationContext(document)
errors = []
for validation in self._validations():
if validation.validation_function(document, eval_context) is False:
errors.append((validation.name, validation.error_message))
if errors:
raise ValidationError(errors)
def _process_chunk_for_domain(self, domain, changes_chunk):
adapters = list(self.table_adapters_by_domain[domain])
changes_by_id = {change.id: change for change in changes_chunk}
to_delete_by_adapter = defaultdict(list)
rows_to_save_by_adapter = defaultdict(list)
async_configs_by_doc_id = defaultdict(list)
to_update = {change for change in changes_chunk if not change.deleted}
retry_changes, docs = self.get_docs_for_changes(to_update, domain)
change_exceptions = []
for doc in docs:
eval_context = EvaluationContext(doc)
for adapter in adapters:
if adapter.config.filter(doc, eval_context):
if adapter.run_asynchronous:
async_configs_by_doc_id[doc['_id']].append(
adapter.config._id)
else:
try:
rows_to_save_by_adapter[adapter].extend(
adapter.get_all_values(doc, eval_context))
except Exception as e:
change_exceptions.append(
(changes_by_id[doc["_id"]], e))
eval_context.reset_iteration()
elif adapter.config.deleted_filter(doc) or adapter.doc_exists(
doc):
to_delete_by_adapter[adapter].append(doc['_id'])
# bulk delete by adapter
to_delete = [c.id for c in changes_chunk if c.deleted]
for adapter in adapters:
delete_ids = to_delete_by_adapter[adapter] + to_delete
try:
adapter.bulk_delete(delete_ids)
except Exception:
retry_changes.update(
[c for c in changes_chunk if c.id in delete_ids])
# bulk update by adapter
for adapter, rows in six.iteritems(rows_to_save_by_adapter):
try:
adapter.save_rows(rows)
except Exception:
retry_changes.update(to_update)
if async_configs_by_doc_id:
doc_type_by_id = {
_id: changes_by_id[_id].metadata.document_type
for _id in async_configs_by_doc_id.keys()
}
AsyncIndicator.bulk_update_records(async_configs_by_doc_id, domain,
doc_type_by_id)
return retry_changes, change_exceptions
def test_dmc_referring_facility_id(self):
test_case = {
'_id': 'test_case_id',
'domain': 'enikshay-test',
'referring_facility_id': 'facility_id',
}
expression = self.get_expression('dmc_referring_facility_id', 'string')
self.assertEqual(
expression(test_case, EvaluationContext(test_case, 0)),
'facility_id')
def test_no_datatype(self):
expression = ExpressionFactory.from_spec({
"type": "ext_root_property_name",
"property_name": "foo",
})
self.assertEqual(
"foo_value",
expression(
{"some_item": "item_value"},
context=EvaluationContext({"foo": "foo_value"}, 0)
)
)
def setUp(self):
super(TestGetReportingGroupsExpression, self).setUp()
self.domain = uuid.uuid4().hex
self.second_domain = uuid.uuid4().hex
self.expression = ExpressionFactory.from_spec({
"type": "get_reporting_groups",
"user_id_expression": {
"type": "property_name",
"property_name": "user_id"
},
})
self.context = EvaluationContext({"domain": self.domain})
def setUp(self):
super(TestGetSubcasesExpression, self).setUp()
self.domain = uuid.uuid4().hex
self.factory = CaseFactory(domain=self.domain)
self.expression = ExpressionFactory.from_spec({
"type": "get_subcases",
"case_id_expression": {
"type": "property_name",
"property_name": "_id"
},
})
self.context = EvaluationContext({"domain": self.domain})
def test_expression_when_referral_does_not_pass_filters(self):
referral_2_case_id = uuid.uuid4().hex
self.create_referral_case(referral_2_case_id)
self.update_referral(referral_2_case_id, {
"referral_closed_reason": "duplicate_referral_reconciliation",
})
context = EvaluationContext({"domain": self.domain})
expression = ExpressionFactory.from_spec({
"type": "enikshay_most_recent_referral_from_person",
"person_id_expression": self.person_id,
})
self.assertEqual(expression({}, context), None)
def test_presumptives_found_positive(self):
test_case = {
'_id': 'test_case_id',
'domain': 'enikshay-test',
'episode_type_at_request': 'presumptive_tb',
'rft_general': 'diagnosis_dstb',
'result': 'tb_detected'
}
expression = self.get_expression('presumptives_found_positive',
'integer')
self.assertEqual(
expression(test_case, EvaluationContext(test_case, 0)), 1)
test_case['rft_general'] = 'diagnosis_drtb'
self.assertEqual(
expression(test_case, EvaluationContext(test_case, 0)), 1)
test_case['result'] = 'other'
self.assertEqual(
expression(test_case, EvaluationContext(test_case, 0)), 0)
def get_value(self, doc_id, context):
try:
assert context.root_doc['domain']
document_store = get_document_store(context.root_doc['domain'],
self.related_doc_type)
doc = document_store.get_document(doc_id)
# ensure no cross-domain lookups of different documents
if context.root_doc['domain'] != doc.get('domain'):
return None
# explicitly use a new evaluation context since this is a new document
return self._value_expression(doc, EvaluationContext(doc, 0))
except DocumentNotFoundError:
return None
def test_month_end_date(self):
expression = ExpressionFactory.from_spec({
"type": "ext_month_end",
})
self.assertEqual(
date(2015, 7, 31),
expression(
{"some_item": "item_value"},
context=EvaluationContext(
{
"opened_on": "2015-06-03T01:10:15.241903Z",
}, 1),
))
def test_ledger_balances_indicator(self):
indicator = IndicatorFactory.from_spec(self.spec)
doc = {'_id': 'case1', 'domain': 'domain'}
with patch('corehq.apps.userreports.indicators.get_values_by_product',
return_value=self.stock_states):
values = indicator.get_values(doc, EvaluationContext(doc, 0))
self.assertEqual([(val.column.id, val.value) for val in values],
[('immun_dates_tt_1', date(2016, 7, 18)),
('immun_dates_tt_2', date(2016, 8, 7)),
('immun_dates_hpv', date(2019, 4, 14)),
('immun_dates_non_exist', date(1970, 1, 1))])
def test_ancestor_location_exists(self):
context = EvaluationContext({})
expression = ExpressionFactory.from_spec(
{
'type': 'ancestor_location',
'location_id': self.city.location_id,
'location_type': "continent",
}, context)
ancestor_location = expression({}, context)
self.assertIsNotNone(ancestor_location)
self.assertEqual(ancestor_location.get("location_id"),
self.continent.location_id)
def test_regimen_property_previously_treated(self):
person_case = {
'_id': 'person-case-id',
'domain': 'enikshay-test',
}
episode_case = {
'_id': 'episode-case-id',
'adherence_schedule_id': '',
'case_definition': '',
'date_of_diagnosis': '',
'domain': 'enikshay-test',
'enikshay_enabled': '',
'episode_type': 'confirmed_tb',
'patient_type_choice': 'other_previously_treated',
'treatment_regimen': 'test_regimen',
'treatment_status': ''
}
self.database.mock_docs = {
'episode-case-id': episode_case,
'person-case-id': person_case
}
column = self.get_column('regimen')
self.assertEqual(column['datatype'], 'string')
expression = ExpressionFactory.from_spec(
column['expression'],
context=FactoryContext(self.named_expressions, {})
)
with mock.patch.object(MostRecentEpisodeCaseFromPerson, '__call__', lambda *args: episode_case):
self.assertEqual(expression(episode_case, EvaluationContext(episode_case, 0)), '')
episode_case['treatment_initiated'] = 'yes_phi'
with mock.patch.object(MostRecentEpisodeCaseFromPerson, '__call__', lambda *args: episode_case):
self.assertEqual(expression(episode_case, EvaluationContext(episode_case, 0)), 'Previously Treated')
def test_all_child_cases(self):
context = EvaluationContext({"domain": self.domain}, 0)
expression = ExpressionFactory.from_spec({
"type": "reduce_items",
"aggregation_fn": "count",
"items_expression": {
"type": "icds_get_child_cases",
"case_id_expression": {
"type": "constant",
"constant": self.test_case_id
}
}
})
self.assertEqual(2, expression({"some_field", "some_value"}, context))
def save_document(doc_ids):
lock_keys = []
for doc_id in doc_ids:
lock_keys.append(get_async_indicator_modify_lock_key(doc_id))
with CriticalSection(lock_keys):
indicators = AsyncIndicator.objects.filter(doc_id__in=doc_ids)
if not indicators:
return
first_indicator = indicators[0]
processed_indicators = []
failed_indicators = []
for i in indicators:
assert i.domain == first_indicator.domain
assert i.doc_type == first_indicator.doc_type
indicator_by_doc_id = {i.doc_id: i for i in indicators}
doc_store = get_document_store(first_indicator.domain,
first_indicator.doc_type)
for doc in doc_store.iter_documents(doc_ids):
indicator = indicator_by_doc_id[doc['_id']]
eval_context = EvaluationContext(doc)
for config_id in indicator.indicator_config_ids:
adapter = None
try:
config = _get_config(config_id)
adapter = get_indicator_adapter(config,
can_handle_laboratory=True)
adapter.save(doc, eval_context)
eval_context.reset_iteration()
except (ESError, RequestError, ConnectionTimeout):
# couch or es had an issue so don't log it and go on to the next doc
failed_indicators.append(indicator.pk)
break
except Exception as e:
# getting the config could fail before the adapter is set
if adapter:
adapter.handle_exception(doc, e)
failed_indicators.append(indicator.pk)
break
else:
processed_indicators.append(indicator.pk)
AsyncIndicator.objects.filter(pk__in=processed_indicators).delete()
AsyncIndicator.objects.filter(pk__in=failed_indicators).update(
date_queued=None,
unsuccessful_attempts=F('unsuccessful_attempts') + 1)
def test_not_closed(self):
expression = ExpressionFactory.from_spec({
"type": "ext_open_in_month",
})
self.assertEqual(
"yes",
expression(
{"some_item": "item_value"},
context=EvaluationContext(
{
"opened_on": "2015-06-03T01:10:15.241903Z",
"closed": False,
}, 3),
))
def test_key_populations(self):
episode_case = {
'_id': 'episode_case_id',
'domain': 'enikshay-test',
'treatment_initiation_date': '2017-09-28',
'archive_reason': None,
'treatment_outcome': None,
'indices': [{
'referenced_id': 'occurrence_case_id'
}]
}
occurrence_case = {
'_id': 'occurrence_case_id',
'domain': 'enikshay-test',
'indices': [{
'referenced_id': 'person_case_id'
}],
'key_populations': 'test test2 test3'
}
person_case = {
'_id': 'person_case_id',
'domain': 'enikshay-test',
'owner_id': 'owner-id'
}
test_case = {
'_id': 'test_case_id',
'domain': 'enikshay-test',
'episode_case_id': 'episode_case_id',
'rft_general': 'follow_up_dstb',
'date_reported': '2017-10-01',
'indices': [{
'referenced_id': 'occurrence_case_id'
}]
}
self.database.mock_docs = {
'episode_case_id': episode_case,
'occurrence_case_id': occurrence_case,
'person_case_id': person_case,
'test_case_id': test_case
}
expression = self.get_expression('key_populations', 'string')
self.assertEqual(
expression(test_case, EvaluationContext(test_case, 0)),
'test, test2, test3')
def test_cross_domain_lookups(self):
related_id = 'cross-domain-id'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {
'domain': 'wrong-domain',
'related_property': 'foo'
}
self.database.mock_docs = {
'my-id': my_doc,
related_id: related_doc
}
self.assertEqual(None, self.expression(my_doc, EvaluationContext(my_doc, 0)))
def get_referral_expression_case(self, person_id):
"""
Evaluate the ReferralTestExpression against the given person_id
"""
context = EvaluationContext({"domain": self.domain})
expression = referral_test_expression({
# "domain": self.domain,
'person_id_expression': {
"type": "property_name",
"property_name": "person_id"
}
}, context)
referral_or_trail = expression({"person_id": person_id}, context)
return referral_or_trail
def test_not_null_filter_root_doc(self):
indicator = IndicatorFactory.from_spec({
"filter": {
"filter": {
"operator": "in",
"expression": {
"expression": {
"datatype": None,
"type": "property_name",
"property_name": "ccs_opened_date"
},
"type": "root_doc"
},
"type": "boolean_expression",
"property_value": ["", None]
},
"type": "not"
},
"type": "boolean",
"display_name": None,
"column_id": "prop1_not_null"
})
self._check_result(
indicator, {}, 1,
EvaluationContext(root_doc=dict(ccs_opened_date='not_null')))
self._check_result(
indicator, {}, 1,
EvaluationContext(root_doc=dict(ccs_opened_date=' ')))
self._check_result(
indicator, {}, 0,
EvaluationContext(root_doc=dict(ccs_opened_date='')))
self._check_result(
indicator, {}, 0,
EvaluationContext(root_doc=dict(ccs_opened_date=None)))
self._check_result(indicator, {}, 0,
EvaluationContext(root_doc=dict()))
def test_simple_lookup(self):
related_id = 'related-id'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {
'domain': 'test-domain',
'related_property': 'foo'
}
self.database.mock_docs = {
'my-id': my_doc,
related_id: related_doc
}
self.assertEqual('foo', self.expression(my_doc, EvaluationContext(my_doc, 0)))
def test_ancestor_location_property(self):
context = EvaluationContext({'domain': self.domain})
expression = ExpressionFactory.from_spec({
'type': 'ancestor_location',
'location_id': self.city.location_id,
'location_type': "continent",
'location_property': "_id"
}, context)
ancestor_location_property = expression({}, context)
self.assertIsNotNone(ancestor_location_property)
self.assertEqual(
ancestor_location_property,
self.continent.location_id
)
def test_last_update_none(self):
expression = ExpressionFactory.from_spec({
"type": "ext_get_last_case_property_update",
"case_id_expression": {
"type": "constant",
"constant": self.test_case_id
},
"case_property": "bar",
})
self.assertEqual(
None,
expression(
{"some_item": "item_value"},
context=EvaluationContext({"domain": self.domain}, 0),
))
def test_xmlns_multiple(self):
context = EvaluationContext({"domain": self.domain}, 0)
expression = ExpressionFactory.from_spec({
"type": "reduce_items",
"aggregation_fn": "count",
"items_expression": {
"type": "ext_get_case_forms_by_date",
"case_id_expression": {
"type": "constant",
"constant": self.test_case_id
},
"xmlns": ["xmlns_a", "xmlns_b"]
}
})
self.assertEqual(5, expression({"some_field", "some_value"}, context))
def test_key_populations(self):
episode_case = {
'_id': 'episode_case_id',
'domain': 'enikshay-test',
'treatment_initiation_date': '2017-07-01',
'archive_reason': None,
'treatment_outcome': None,
'indices': [{
'referenced_id': 'occurrence_case_id'
}]
}
occurrence_case = {
'_id': 'occurrence_case_id',
'domain': 'enikshay-test',
'indices': [{
'referenced_id': 'person_case_id'
}],
'key_populations': 'test test2 test3'
}
test_case = {
'_id': 'test_case_id',
'domain': 'enikshay-test',
'episode_case_id': 'episode_case_id',
'rft_general': 'follow_up_dstb',
'date_reported': '2017-10-01',
'indices': [{
'referenced_id': 'occurrence_case_id'
}]
}
self.database.mock_docs = {
'episode_case_id': episode_case,
'test_case_id': test_case,
'occurrence_case_id': occurrence_case
}
column = self._get_column('key_populations_from_occurrence')
self.assertEqual(column['datatype'], 'string')
expression = ExpressionFactory.from_spec(column['expression'],
context=FactoryContext(
self.named_expressions,
{}))
self.assertEqual(
expression(test_case, EvaluationContext(test_case, 0)),
'test, test2, test3')
def test_no_parent_id(self):
expression = ExpressionFactory.from_spec({
"type": "ext_parent_id",
})
self.assertEqual(
None,
expression({"some_item": "item_value"},
context=EvaluationContext(
{
"indices": [
{
"identifier": "mother",
"referenced_id": "m_id",
},
],
}, 0)))
def test_disease_classification_extra_pulmonary_site_choice_other(self):
episode_case = {
'_id': 'episode_case_id',
'domain': 'enikshay-test',
'treatment_initiation_date': '2017-09-28',
'archive_reason': None,
'treatment_outcome': 'test',
'indices': [
{'referenced_id': 'occurrence_case_id'}
]
}
occurrence_case = {
'_id': 'occurrence_case_id',
'domain': 'enikshay-test',
'disease_classification': 'extra_pulmonary',
'site_choice': 'other',
'site_detail': 'test detail',
'indices': [
{'referenced_id': 'person_case_id'}
]
}
person_case = {
'_id': 'person_case_id',
'domain': 'enikshay-test',
'owner_id': 'owner-id'
}
self.database.mock_docs = {
'episode_case_id': episode_case,
'occurrence_case_id': occurrence_case,
'person_case_id': person_case
}
column = self._get_column('disease_classification')
self.assertEqual(column['datatype'], 'string')
expression = ExpressionFactory.from_spec(
column['expression'],
context=FactoryContext(self.named_expressions, {})
)
self.assertEqual(
expression(episode_case, EvaluationContext(episode_case, 0)),
'Extra Pulmonary, Other, test detail'
)
