class Spam(DocumentSchema):
ham = Ham(required=False)
ham_prop = SchemaProperty(Ham, required=False)
ham_dict_prop = SchemaDictProperty(Ham, required=False)
class IndexTree(DocumentSchema):
"""
Document type representing a case dependency tree (which is flattened to a single dict)
"""
# a flat mapping of cases to dicts of their indices. The keys in each dict are the index identifiers
# and the values are the referenced case IDs
indices = SchemaDictProperty()
@property
@memoized
def reverse_indices(self):
return _reverse_index_map(self.indices)
def __repr__(self):
return json.dumps(self.indices, indent=2)
@staticmethod
def get_all_dependencies(case_id, child_index_tree, extension_index_tree):
"""Takes a child and extension index tree and returns returns a set of all dependencies of <case_id>
Traverse each incoming index, return each touched case.
Traverse each outgoing index in the extension tree, return each touched case
"""
all_cases = set()
cases_to_check = set([case_id])
while cases_to_check:
case_to_check = cases_to_check.pop()
all_cases.add(case_to_check)
incoming_extension_indices = extension_index_tree.get_cases_that_directly_depend_on_case(
case_to_check)
incoming_child_indices = child_index_tree.get_cases_that_directly_depend_on_case(
case_to_check)
all_incoming_indices = incoming_extension_indices | incoming_child_indices
new_outgoing_cases_to_check = set(
extension_index_tree.indices.get(case_to_check, {}).values())
new_cases_to_check = (new_outgoing_cases_to_check
| all_incoming_indices) - all_cases
cases_to_check |= new_cases_to_check
return all_cases
@staticmethod
@memoized
def get_all_outgoing_cases(case_id, child_index_tree,
extension_index_tree):
"""traverse all outgoing child and extension indices"""
all_cases = set([case_id])
new_cases = set([case_id])
while new_cases:
case_to_check = new_cases.pop()
parent_cases = set(
child_index_tree.indices.get(case_to_check, {}).values())
host_cases = set(
extension_index_tree.indices.get(case_to_check, {}).values())
new_cases = (new_cases | parent_cases | host_cases) - all_cases
all_cases = all_cases | parent_cases | host_cases
return all_cases
@staticmethod
@memoized
def traverse_incoming_extensions(case_id, extension_index_tree,
closed_cases):
"""traverse open incoming extensions"""
all_cases = set([case_id])
new_cases = set([case_id])
while new_cases:
case_to_check = new_cases.pop()
open_incoming_extension_indices = {
case
for case in extension_index_tree.
get_cases_that_directly_depend_on_case(case_to_check)
if case not in closed_cases
}
for incoming_case in open_incoming_extension_indices:
new_cases.add(incoming_case)
all_cases.add(incoming_case)
return all_cases
def get_cases_that_directly_depend_on_case(self, case_id):
return self.reverse_indices.get(case_id, set([]))
def delete_index(self, from_case_id, index_name):
prior_ids = self.indices.pop(from_case_id, {})
prior_ids.pop(index_name, None)
if prior_ids:
self.indices[from_case_id] = prior_ids
self._clear_index_caches()
def set_index(self, from_case_id, index_name, to_case_id):
prior_ids = self.indices.get(from_case_id, {})
prior_ids[index_name] = to_case_id
self.indices[from_case_id] = prior_ids
self._clear_index_caches()
def _clear_index_caches(self):
try:
# self.reverse_indices is a memoized property, so we can't just call self.reverse_indices.reset_cache
self._reverse_indices_cache.clear()
except AttributeError:
pass
self.get_all_outgoing_cases.reset_cache()
self.traverse_incoming_extensions.reset_cache()
def apply_updates(self, other_tree):
"""
Apply updates from another IndexTree and return a copy with those applied.
If an id is found in the new one, use that id's indices, otherwise, use this ones,
(defaulting to nothing).
"""
assert isinstance(other_tree, IndexTree)
new = IndexTree(indices=copy(self.indices), )
new.indices.update(other_tree.indices)
return new
def _reverse_index_map(index_map):
reverse_indices = defaultdict(set)
for case_id, indices in index_map.items():
for indexed_case_id in indices.values():
reverse_indices[indexed_case_id].add(case_id)
return dict(reverse_indices)
class SimplifiedSyncLog(AbstractSyncLog):
"""
New, simplified sync log class that is used by ownership cleanliness restore.
Just maintains a flat list of case IDs on the phone rather than the case/dependent state
lists from the SyncLog class.
"""
log_format = StringProperty(default=LOG_FORMAT_SIMPLIFIED)
case_ids_on_phone = SetProperty(six.text_type)
# this is a subset of case_ids_on_phone used to flag that a case is only around because it has dependencies
# this allows us to purge it if possible from other actions
dependent_case_ids_on_phone = SetProperty(six.text_type)
owner_ids_on_phone = SetProperty(six.text_type)
index_tree = SchemaProperty(IndexTree) # index tree of subcases / children
extension_index_tree = SchemaProperty(
IndexTree) # index tree of extensions
closed_cases = SetProperty(six.text_type)
extensions_checked = BooleanProperty(default=False)
device_id = StringProperty()
_purged_cases = None
@property
def purged_cases(self):
if self._purged_cases is None:
self._purged_cases = set()
return self._purged_cases
def case_count(self):
return len(self.case_ids_on_phone)
def phone_is_holding_case(self, case_id):
"""
Whether the phone currently has a case, according to this sync log
"""
return case_id in self.case_ids_on_phone
def get_footprint_of_cases_on_phone(self):
return list(self.case_ids_on_phone)
@property
def primary_case_ids(self):
return self.case_ids_on_phone - self.dependent_case_ids_on_phone
def purge(self, case_id, xform_id=None):
"""
This happens in 3 phases, and recursively tries to purge outgoing indices of purged cases.
Definitions:
-----------
A case is *relevant* if:
- it is open and owned or,
- it has a relevant child or,
- it has a relevant extension or,
- it is the extension of a relevant case.
A case is *available* if:
- it is open and not an extension case or,
- it is open and is the extension of an available case.
A case is *live* if:
- it is owned and available or,
- it has a live child or,
- it has a live extension or,
- it is the exension of a live case.
Algorithm:
----------
1. Mark *relevant* cases
Mark all open cases owned by the user relevant. Traversing all outgoing child
and extension indexes, as well as all incoming extension indexes, mark all
touched cases relevant.
2. Mark *available* cases
Mark all relevant cases that are open and have no outgoing extension indexes
as available. Traverse incoming extension indexes which don't lead to closed
cases, mark all touched cases as available.
3. Mark *live* cases
Mark all relevant, owned, available cases as live. Traverse incoming
extension indexes which don't lead to closed cases, mark all touched
cases as live.
"""
_get_logger().debug("purging: {}".format(case_id))
self.dependent_case_ids_on_phone.add(case_id)
relevant = self._get_relevant_cases(case_id)
available = self._get_available_cases(relevant)
live = self._get_live_cases(available)
to_remove = (relevant - self.purged_cases) - live
self._remove_cases_purge_indices(to_remove, case_id, xform_id)
def _get_relevant_cases(self, case_id):
"""
Mark all open cases owned by the user relevant. Traversing all outgoing child
and extension indexes, as well as all incoming extension indexes,
mark all touched cases relevant.
"""
relevant = IndexTree.get_all_dependencies(
case_id,
child_index_tree=self.index_tree,
extension_index_tree=self.extension_index_tree,
)
_get_logger().debug("Relevant cases of {}: {}".format(
case_id, relevant))
return relevant
def _get_available_cases(self, relevant):
"""
Mark all relevant cases that are open and have no outgoing extension indexes
as available. Traverse incoming extension indexes which don't lead to closed
cases, mark all touched cases as available
"""
incoming_extensions = self.extension_index_tree.reverse_indices
available = {
case
for case in relevant if case not in self.closed_cases and (
not self.extension_index_tree.indices.get(case)
or self.index_tree.indices.get(case))
}
new_available = set() | available
while new_available:
case_to_check = new_available.pop()
for incoming_extension in incoming_extensions.get(
case_to_check, []):
closed = incoming_extension in self.closed_cases
purged = incoming_extension in self.purged_cases
if not closed and not purged:
new_available.add(incoming_extension)
available = available | new_available
_get_logger().debug("Available cases: {}".format(available))
return available
def _get_live_cases(self, available):
"""
Mark all relevant, owned, available cases as live. Traverse incoming
extension indexes which don't lead to closed cases, mark all touched
cases as available.
"""
primary_case_ids = self.primary_case_ids
live = available & primary_case_ids
new_live = set() | live
checked = set()
while new_live:
case_to_check = new_live.pop()
checked.add(case_to_check)
new_live = new_live | IndexTree.get_all_outgoing_cases(
case_to_check, self.index_tree,
self.extension_index_tree) - self.purged_cases
new_live = new_live | IndexTree.traverse_incoming_extensions(
case_to_check,
self.extension_index_tree,
frozenset(self.closed_cases),
) - self.purged_cases
new_live = new_live - checked
live = live | new_live
_get_logger().debug("live cases: {}".format(live))
return live
def _remove_cases_purge_indices(self, all_to_remove, checked_case_id,
xform_id):
"""Remove all cases marked for removal. Traverse child cases and try to purge those too."""
_get_logger().debug("cases to to_remove: {}".format(all_to_remove))
for to_remove in all_to_remove:
indices = self.index_tree.indices.get(to_remove, {})
self._remove_case(to_remove, all_to_remove, checked_case_id,
xform_id)
for referenced_case in indices.values():
is_dependent_case = referenced_case in self.dependent_case_ids_on_phone
already_primed_for_removal = referenced_case in all_to_remove
if is_dependent_case and not already_primed_for_removal and referenced_case != checked_case_id:
self.purge(referenced_case, xform_id)
def _remove_case(self, to_remove, all_to_remove, checked_case_id,
xform_id):
"""Removes case from index trees, case_ids_on_phone and dependent_case_ids_on_phone if pertinent"""
_get_logger().debug('removing: {}'.format(to_remove))
deleted_indices = self.index_tree.indices.pop(to_remove, {})
deleted_indices.update(
self.extension_index_tree.indices.pop(to_remove, {}))
self._validate_case_removal(to_remove, all_to_remove, deleted_indices,
checked_case_id, xform_id)
try:
self.case_ids_on_phone.remove(to_remove)
except KeyError:
should_fail_softly = not xform_id or _domain_has_legacy_toggle_set(
)
if should_fail_softly:
pass
else:
# this is only a soft assert for now because of http://manage.dimagi.com/default.asp?181443
# we should convert back to a real Exception when we stop getting any of these
_assert = soft_assert(notify_admins=True,
exponential_backoff=False)
_assert(
False, 'case already remove from synclog', {
'case_id': to_remove,
'synclog_id': self._id,
'form_id': xform_id
})
else:
self.purged_cases.add(to_remove)
if to_remove in self.dependent_case_ids_on_phone:
self.dependent_case_ids_on_phone.remove(to_remove)
def _validate_case_removal(self, case_to_remove, all_to_remove,
deleted_indices, checked_case_id, xform_id):
"""Traverse immediate outgoing indices. Validate that these are also candidates for removal."""
if case_to_remove == checked_case_id:
return
# Logging removed temporarily: https://github.com/dimagi/commcare-hq/pull/16259#issuecomment-303176217
# for index in deleted_indices.values():
# if xform_id and not _domain_has_legacy_toggle_set():
# # unblocking http://manage.dimagi.com/default.asp?185850
# _assert = soft_assert(send_to_ops=False, log_to_file=True, exponential_backoff=True,
# fail_if_debug=True)
# _assert(index in (all_to_remove | set([checked_case_id])),
# "expected {} in {} but wasn't".format(index, all_to_remove))
def _add_primary_case(self, case_id):
self.case_ids_on_phone.add(case_id)
if case_id in self.dependent_case_ids_on_phone:
self.dependent_case_ids_on_phone.remove(case_id)
def _add_index(self, index, case_update):
_get_logger().debug('adding index {} --<{}>--> {} ({}).'.format(
index.case_id, index.relationship, index.referenced_id,
index.identifier))
if index.relationship == const.CASE_INDEX_EXTENSION:
self._add_extension_index(index, case_update)
else:
self._add_child_index(index)
def _add_extension_index(self, index, case_update):
assert index.relationship == const.CASE_INDEX_EXTENSION
self.extension_index_tree.set_index(index.case_id, index.identifier,
index.referenced_id)
if index.referenced_id not in self.case_ids_on_phone:
self.case_ids_on_phone.add(index.referenced_id)
self.dependent_case_ids_on_phone.add(index.referenced_id)
case_child_indices = [
idx for idx in case_update.indices_to_add
if idx.relationship == const.CASE_INDEX_CHILD
and idx.referenced_id == index.referenced_id
]
if not case_child_indices and not case_update.is_live:
# this case doesn't also have child indices, and it is not owned, so it is dependent
self.dependent_case_ids_on_phone.add(index.case_id)
def _add_child_index(self, index):
assert index.relationship == const.CASE_INDEX_CHILD
self.index_tree.set_index(index.case_id, index.identifier,
index.referenced_id)
if index.referenced_id not in self.case_ids_on_phone:
self.case_ids_on_phone.add(index.referenced_id)
self.dependent_case_ids_on_phone.add(index.referenced_id)
def _delete_index(self, index):
self.index_tree.delete_index(index.case_id, index.identifier)
self.extension_index_tree.delete_index(index.case_id, index.identifier)
def update_phone_lists(self, xform, case_list):
made_changes = False
_get_logger().debug('updating sync log for {}'.format(self.user_id))
_get_logger().debug('case ids before update: {}'.format(', '.join(
self.case_ids_on_phone)))
_get_logger().debug('dependent case ids before update: {}'.format(
', '.join(self.dependent_case_ids_on_phone)))
_get_logger().debug('index tree before update: {}'.format(
self.index_tree))
_get_logger().debug('extension index tree before update: {}'.format(
self.extension_index_tree))
class CaseUpdate(object):
def __init__(self, case_id, owner_ids_on_phone):
self.case_id = case_id
self.owner_ids_on_phone = owner_ids_on_phone
self.was_live_previously = True
self.final_owner_id = None
self.is_closed = None
self.indices_to_add = []
self.indices_to_delete = []
@property
def extension_indices_to_add(self):
return [
index for index in self.indices_to_add
if index.relationship == const.CASE_INDEX_EXTENSION
]
def has_extension_indices_to_add(self):
return len(self.extension_indices_to_add) > 0
@property
def is_live(self):
"""returns whether an update is live for a specifc set of owner_ids"""
if self.is_closed:
return False
elif self.final_owner_id is None:
# we likely didn't touch owner_id so just default to whatever it was previously
return self.was_live_previously
else:
return self.final_owner_id in self.owner_ids_on_phone
ShortIndex = namedtuple(
'ShortIndex',
['case_id', 'identifier', 'referenced_id', 'relationship'])
# this is a variable used via closures in the function below
owner_id_map = {}
def get_latest_owner_id(case_id, action=None):
# "latest" just means as this forms actions are played through
if action is not None:
owner_id_from_action = action.updated_known_properties.get(
"owner_id")
if owner_id_from_action is not None:
owner_id_map[case_id] = owner_id_from_action
return owner_id_map.get(case_id, None)
all_updates = {}
for case in case_list:
if case.case_id not in all_updates:
_get_logger().debug('initializing update for case {}'.format(
case.case_id))
all_updates[case.case_id] = CaseUpdate(
case_id=case.case_id,
owner_ids_on_phone=self.owner_ids_on_phone)
case_update = all_updates[case.case_id]
case_update.was_live_previously = case.case_id in self.primary_case_ids
actions = case.get_actions_for_form(xform)
for action in actions:
_get_logger().debug('{}: {}'.format(case.case_id,
action.action_type))
owner_id = get_latest_owner_id(case.case_id, action)
if owner_id is not None:
case_update.final_owner_id = owner_id
if action.action_type == const.CASE_ACTION_INDEX:
for index in action.indices:
if index.referenced_id:
case_update.indices_to_add.append(
ShortIndex(case.case_id, index.identifier,
index.referenced_id,
index.relationship))
else:
case_update.indices_to_delete.append(
ShortIndex(case.case_id, index.identifier,
None, None))
elif action.action_type == const.CASE_ACTION_CLOSE:
case_update.is_closed = True
non_live_updates = []
for case in case_list:
case_update = all_updates[case.case_id]
if case_update.is_live:
_get_logger().debug('case {} is live.'.format(
case_update.case_id))
if case.case_id not in self.case_ids_on_phone:
self._add_primary_case(case.case_id)
made_changes = True
elif case.case_id in self.dependent_case_ids_on_phone:
self.dependent_case_ids_on_phone.remove(case.case_id)
made_changes = True
for index in case_update.indices_to_add:
self._add_index(index, case_update)
made_changes = True
for index in case_update.indices_to_delete:
self._delete_index(index)
made_changes = True
else:
class OpenmrsRepeater(CaseRepeater):
"""
``OpenmrsRepeater`` is responsible for updating OpenMRS patients
with changes made to cases in CommCare. It is also responsible for
creating OpenMRS "visits", "encounters" and "observations" when a
corresponding visit form is submitted in CommCare.
The ``OpenmrsRepeater`` class is different from most repeater
classes in three details:
1. It has a case type and it updates the OpenMRS equivalent of cases
like the ``CaseRepeater`` class, but it reads forms like the
``FormRepeater`` class. So it subclasses ``CaseRepeater`` but its
payload format is ``form_json``.
2. It makes many API calls for each payload.
3. It can have a location.
"""
class Meta(object):
app_label = 'repeaters'
include_app_id_param = False
friendly_name = _("Forward to OpenMRS")
payload_generator_classes = (FormRepeaterJsonPayloadGenerator, )
location_id = StringProperty(default='')
openmrs_config = SchemaProperty(OpenmrsConfig)
_has_config = True
# self.white_listed_case_types must have exactly one case type set
# for Atom feed integration to add cases for OpenMRS patients.
# self.location_id must be set to determine their case owner. The
# owner is set to the first CommCareUser instance found at that
# location.
atom_feed_enabled = BooleanProperty(default=False)
atom_feed_status = SchemaDictProperty(AtomFeedStatus)
def __init__(self, *args, **kwargs):
super(OpenmrsRepeater, self).__init__(*args, **kwargs)
def __eq__(self, other):
return (isinstance(other, self.__class__)
and self.get_id == other.get_id)
def __str__(self):
return Repeater.__str__(self)
@classmethod
def wrap(cls, data):
if 'atom_feed_last_polled_at' in data:
data['atom_feed_status'] = {
ATOM_FEED_NAME_PATIENT: {
'last_polled_at': data.pop('atom_feed_last_polled_at'),
'last_page': data.pop('atom_feed_last_page', None),
}
}
return super(OpenmrsRepeater, cls).wrap(data)
@cached_property
def requests(self):
# Used by atom_feed module and views that don't have a payload
# associated with the request
return self.get_requests()
def get_requests(self, payload_id=None):
return Requests(
self.domain,
self.url,
self.username,
self.plaintext_password,
verify=self.verify,
notify_addresses=self.notify_addresses,
payload_id=payload_id,
)
@cached_property
def first_user(self):
return get_one_commcare_user_at_location(self.domain, self.location_id)
@memoized
def payload_doc(self, repeat_record):
return FormAccessors(repeat_record.domain).get_form(
repeat_record.payload_id)
@property
def form_class_name(self):
"""
The class name used to determine which edit form to use
"""
return self.__class__.__name__
@classmethod
def available_for_domain(cls, domain):
return OPENMRS_INTEGRATION.enabled(domain)
def allowed_to_forward(self, payload):
"""
Forward the payload if ...
* it did not come from OpenMRS, and
* CaseRepeater says it's OK for the case types and users of any
of the payload's cases, and
* this repeater forwards to the right OpenMRS server for any of
the payload's cases.
:param payload: An XFormInstance (not a case)
"""
if payload.xmlns == XMLNS_OPENMRS:
# payload came from OpenMRS. Don't send it back.
return False
case_blocks = extract_case_blocks(payload)
case_ids = [case_block['@case_id'] for case_block in case_blocks]
cases = CaseAccessors(payload.domain).get_cases(case_ids, ordered=True)
if not any(
CaseRepeater.allowed_to_forward(self, case) for case in cases):
# If none of the case updates in the payload are allowed to
# be forwarded, drop it.
return False
if not self.location_id:
# If this repeater does not have a location, all payloads
# should go to it.
return True
repeaters = [
repeater for case in cases
for repeater in get_case_location_ancestor_repeaters(case)
]
# If this repeater points to the wrong OpenMRS server for this
# payload then let the right repeater handle it.
return self in repeaters
def get_payload(self, repeat_record):
payload = super(OpenmrsRepeater, self).get_payload(repeat_record)
return json.loads(payload)
def send_request(self, repeat_record, payload):
value_source_configs: Iterable[JsonDict] = chain(
self.openmrs_config.case_config.patient_identifiers.values(),
self.openmrs_config.case_config.person_properties.values(),
self.openmrs_config.case_config.person_preferred_name.values(),
self.openmrs_config.case_config.person_preferred_address.values(),
self.openmrs_config.case_config.person_attributes.values(),
)
case_trigger_infos = get_relevant_case_updates_from_form_json(
self.domain,
payload,
case_types=self.white_listed_case_types,
extra_fields=[
conf["case_property"] for conf in value_source_configs
if "case_property" in conf
],
form_question_values=get_form_question_values(payload),
)
requests = self.get_requests(payload_id=repeat_record.payload_id)
try:
response = send_openmrs_data(
requests,
self.domain,
payload,
self.openmrs_config,
case_trigger_infos,
)
except Exception as err:
requests.notify_exception(str(err))
return OpenmrsResponse(400, 'Bad Request', pformat_json(str(err)))
return response
class OpenmrsCaseConfig(DocumentSchema):
id_matchers = SchemaListProperty(IdMatcher)
person_properties = SchemaDictProperty(ValueSource)
person_attributes = SchemaDictProperty(ValueSource)
class FormQuestionSchema(Document):
"""
Contains information about the questions for a specific form
specifically the options that are available (or have ever been available) for
any multi-select questions.
Calling `update_schema` will load the app and any saved versions of the app
that have not already been processed and update the question schema with
any new options.
"""
domain = StringProperty(required=True)
app_id = StringProperty(required=True)
xmlns = StringProperty(required=True)
last_processed_version = IntegerProperty(default=0)
processed_apps = SetProperty(str)
apps_with_errors = SetProperty(str)
question_schema = SchemaDictProperty(QuestionMeta)
class Meta(object):
app_label = 'export'
@classmethod
def _get_id(cls, domain, app_id, xmlns):
def _none_to_empty_string(str):
return str if str is not None else ''
key = list(map(_none_to_empty_string, [domain, app_id, xmlns]))
return hashlib.sha1(':'.join(key).encode('utf-8')).hexdigest()
@classmethod
def get_by_key(cls, domain, app_id, xmlns):
_id = cls._get_id(domain, app_id, xmlns)
return cls.get(_id)
@classmethod
def get_or_create(cls, domain, app_id, xmlns):
try:
schema = cls.get_by_key(domain, app_id, xmlns)
except ResourceNotFound:
old_schemas = FormQuestionSchema.view(
'form_question_schema/by_xmlns',
key=[domain, app_id, xmlns],
include_docs=True).all()
if old_schemas:
doc = old_schemas[0].to_json()
del doc['_id']
del doc['_rev']
schema = FormQuestionSchema.wrap(doc)
schema.save()
for old in old_schemas:
old.delete()
else:
schema = FormQuestionSchema(domain=domain,
app_id=app_id,
xmlns=xmlns)
schema.save()
return schema
def validate(self, required=True):
# this isn't always set, so set to empty strings if not found
if self.app_id is None:
self.app_id = ''
super(FormQuestionSchema, self).validate(required=required)
if not self.get_id:
self._id = self._get_id(self.domain, self.app_id, self.xmlns)
def update_schema(self):
all_app_ids = get_build_ids_after_version(self.domain, self.app_id,
self.last_processed_version)
all_seen_apps = self.apps_with_errors | self.processed_apps
to_process = [
app_id for app_id in all_app_ids if app_id not in all_seen_apps
]
if self.app_id not in all_seen_apps:
to_process.append(self.app_id)
for app_doc in iter_docs(Application.get_db(), to_process):
if is_remote_app(app_doc):
continue
app = Application.wrap(app_doc)
try:
self.update_for_app(app)
except AppManagerException:
self.apps_with_errors.add(app.get_id)
self.last_processed_version = app.version
if to_process:
self.save()
def update_for_app(self, app):
xform = app.get_xform_by_xmlns(self.xmlns, log_missing=False)
if xform:
prefix = '/{}/'.format(xform.data_node.tag_name)
def to_json_path(xml_path):
if not xml_path:
return
if xml_path.startswith(prefix):
xml_path = xml_path[len(prefix):]
return 'form.{}'.format(xml_path.replace('/', '.'))
for question in xform.get_questions(app.langs):
question_path = to_json_path(question['value'])
if question['tag'] == 'select':
meta = self.question_schema.get(
question_path,
QuestionMeta(
repeat_context=to_json_path(question['repeat'])))
for opt in question['options']:
if opt['value'] not in meta.options:
meta.options.append(opt['value'])
self.question_schema[question_path] = meta
else:
# In the event that a question was previously a multi-select and not one any longer,
# we need to clear the question schema
self.question_schema.pop(question_path, None)
self.processed_apps.add(app.get_id)
self.last_processed_version = app.version
class OpenmrsCaseConfig(DocumentSchema):
# "patient_identifiers": {
# "e2b966d0-1d5f-11e0-b929-000c29ad1d07": {
# "doc_type": "CaseProperty",
# "case_property": "nid"
# },
# "uuid": {
# "doc_type": "CaseProperty",
# "case_property": "openmrs_uuid",
# }
# }
patient_identifiers = SchemaDictProperty(ValueSource)
# The patient_identifiers that are considered reliable
# "match_on_ids": ["uuid", "e2b966d0-1d5f-11e0-b929-000c29ad1d07",
match_on_ids = ListProperty()
# "person_properties": {
# "gender": {
# "doc_type": "CaseProperty",
# "case_property": "gender"
# },
# "birthdate": {
# "doc_type": "CaseProperty",
# "case_property": "dob"
# }
# }
person_properties = SchemaDictProperty(ValueSource)
# "patient_finder": {
# "doc_type": "WeightedPropertyPatientFinder",
# "searchable_properties": ["nid", "family_name"],
# "property_weights": [
# {"case_property": "nid", "weight": 0.9},
# // if "match_type" is not given it defaults to "exact"
# {"case_property": "family_name", "weight": 0.4},
# {
# "case_property": "given_name",
# "weight": 0.3,
# "match_type": "levenshtein",
# // levenshtein function takes edit_distance / len
# "match_params": [0.2]
# // i.e. 0.2 (20%) is one edit for every 5 characters
# // e.g. "Riyaz" matches "Riaz" but not "Riazz"
# },
# {"case_property": "city", "weight": 0.2},
# {
# "case_property": "dob",
# "weight": 0.3,
# "match_type": "days_diff",
# // days_diff matches based on days difference from given date
# "match_params": [364]
# }
# ]
# }
patient_finder = PatientFinder(required=False)
# "person_preferred_name": {
# "givenName": {
# "doc_type": "CaseProperty",
# "case_property": "given_name"
# },
# "middleName": {
# "doc_type": "CaseProperty",
# "case_property": "middle_name"
# },
# "familyName": {
# "doc_type": "CaseProperty",
# "case_property": "family_name"
# }
# }
person_preferred_name = SchemaDictProperty(ValueSource)
# "person_preferred_address": {
# "address1": {
# "doc_type": "CaseProperty",
# "case_property": "address_1"
# },
# "address2": {
# "doc_type": "CaseProperty",
# "case_property": "address_2"
# },
# "cityVillage": {
# "doc_type": "CaseProperty",
# "case_property": "city"
# }
# }
person_preferred_address = SchemaDictProperty(ValueSource)
# "person_attributes": {
# "c1f4239f-3f10-11e4-adec-0800271c1b75": {
# "doc_type": "CaseProperty",
# "case_property": "caste"
# },
# "c1f455e7-3f10-11e4-adec-0800271c1b75": {
# "doc_type": "CasePropertyMap",
# "case_property": "class",
# "value_map": {
# "sc": "c1fcd1c6-3f10-11e4-adec-0800271c1b75",
# "general": "c1fc20ab-3f10-11e4-adec-0800271c1b75",
# "obc": "c1fb51cc-3f10-11e4-adec-0800271c1b75",
# "other_caste": "c207073d-3f10-11e4-adec-0800271c1b75",
# "st": "c20478b6-3f10-11e4-adec-0800271c1b75"
# }
# }
# }
person_attributes = SchemaDictProperty(ValueSource)
@classmethod
def wrap(cls, data):
if 'id_matchers' in data:
# Convert legacy id_matchers to patient_identifiers. e.g.
# [{'doc_type': 'IdMatcher'
# 'identifier_type_id': 'e2b966d0-1d5f-11e0-b929-000c29ad1d07',
# 'case_property': 'nid'}]
# to
# {'e2b966d0-1d5f-11e0-b929-000c29ad1d07': {'doc_type': 'CaseProperty', 'case_property': 'nid'}},
patient_identifiers = {
m['identifier_type_id']: {
'doc_type': 'CaseProperty',
'case_property': m['case_property']
}
for m in data['id_matchers']
}
data['patient_identifiers'] = patient_identifiers
data['match_on_ids'] = list(patient_identifiers)
data.pop('id_matchers')
# Set default data types for known properties
for property_, value_source in chain(
data.get('person_properties', {}).items(),
data.get('person_preferred_name', {}).items(),
data.get('person_preferred_address', {}).items(),
):
data_type = OPENMRS_PROPERTIES[property_]
value_source.setdefault('external_data_type', data_type)
return super(OpenmrsCaseConfig, cls).wrap(data)
class OpenmrsCaseConfig(DocumentSchema):
# "patient_identifiers": {
# "e2b966d0-1d5f-11e0-b929-000c29ad1d07": {
# "doc_type": "CaseProperty",
# "case_property": "nid"
# },
# "uuid": {
# "doc_type": "CaseProperty",
# "case_property": "openmrs_uuid",
# }
# }
patient_identifiers = SchemaDictProperty(ValueSource)
# The patient_identifiers that are considered reliable
# "match_on_ids": ["uuid", "e2b966d0-1d5f-11e0-b929-000c29ad1d07",
match_on_ids = ListProperty()
# "person_properties": {
# "gender": {
# "doc_type": "CaseProperty",
# "case_property": "gender"
# },
# "birthdate": {
# "doc_type": "CaseProperty",
# "case_property": "dob"
# }
# }
person_properties = SchemaDictProperty(ValueSource)
# "patient_finder": {
# "doc_type": "WeightedPropertyPatientFinder",
# "searchable_properties": ["nid", "family_name"],
# "property_weights": [
# {"case_property": "nid", "weight": 0.9},
# {"case_property": "family_name", "weight": 0.4},
# {"case_property": "given_name", "weight": 0.3},
# {"case_property": "city", "weight": 0.2},
# {"case_property": "dob", "weight": 0.3}
# ]
# }
patient_finder = PatientFinder(required=False)
# "person_preferred_name": {
# "givenName": {
# "doc_type": "CaseProperty",
# "case_property": "given_name"
# },
# "middleName": {
# "doc_type": "CaseProperty",
# "case_property": "middle_name"
# },
# "familyName": {
# "doc_type": "CaseProperty",
# "case_property": "family_name"
# }
# }
person_preferred_name = SchemaDictProperty(ValueSource)
# "person_preferred_address": {
# "address1": {
# "doc_type": "CaseProperty",
# "case_property": "address_1"
# },
# "address2": {
# "doc_type": "CaseProperty",
# "case_property": "address_2"
# },
# "cityVillage": {
# "doc_type": "CaseProperty",
# "case_property": "city"
# }
# }
person_preferred_address = SchemaDictProperty(ValueSource)
# "person_attributes": {
# "c1f4239f-3f10-11e4-adec-0800271c1b75": {
# "doc_type": "CaseProperty",
# "case_property": "caste"
# },
# "c1f455e7-3f10-11e4-adec-0800271c1b75": {
# "doc_type": "CasePropertyMap",
# "case_property": "class",
# "value_map": {
# "sc": "c1fcd1c6-3f10-11e4-adec-0800271c1b75",
# "general": "c1fc20ab-3f10-11e4-adec-0800271c1b75",
# "obc": "c1fb51cc-3f10-11e4-adec-0800271c1b75",
# "other_caste": "c207073d-3f10-11e4-adec-0800271c1b75",
# "st": "c20478b6-3f10-11e4-adec-0800271c1b75"
# }
# }
# }
person_attributes = SchemaDictProperty(ValueSource)
@classmethod
def wrap(cls, data):
if 'id_matchers' in data:
# Convert id_matchers to patient_identifiers. e.g.
# [{'doc_type': 'IdMatcher'
# 'identifier_type_id': 'e2b966d0-1d5f-11e0-b929-000c29ad1d07',
# 'case_property': 'nid'}]
# to
# {'e2b966d0-1d5f-11e0-b929-000c29ad1d07': {'doc_type': 'CaseProperty', 'case_property': 'nid'}},
patient_identifiers = {
m['identifier_type_id']: {
'doc_type': 'CaseProperty',
'case_property': m['case_property']
}
for m in data['id_matchers']
}
data['patient_identifiers'] = patient_identifiers
data['match_on_ids'] = list(patient_identifiers)
data.pop('id_matchers')
return super(OpenmrsCaseConfig, cls).wrap(data)
class ApplicationMediaMixin(Document, MediaMixin):
"""
Manages multimedia for itself and sub-objects.
"""
# keys are the paths to each file in the final application media zip
multimedia_map = SchemaDictProperty(HQMediaMapItem)
# paths to custom logos
logo_refs = DictProperty()
archived_media = DictProperty(
) # where we store references to the old logos (or other multimedia) on a downgrade, so that information is not lost
@memoized
def all_media(self, lang=None):
"""
Somewhat counterituitively, this contains all media in the app EXCEPT app-level media (logos).
"""
media = []
self.media_form_errors = False
for module in [m for m in self.get_modules() if m.uses_media()]:
media.extend(module.all_media(lang=lang))
for form in module.get_forms():
try:
form.validate_form()
except (XFormValidationError, XFormException):
self.media_form_errors = True
else:
media.extend(form.all_media(lang=lang))
return media
def multimedia_map_for_build(self,
build_profile=None,
remove_unused=False):
if self.multimedia_map is None:
self.multimedia_map = {}
if self.multimedia_map and remove_unused:
self.remove_unused_mappings()
if not build_profile or not domain_has_privilege(
self.domain, privileges.BUILD_PROFILES):
return self.multimedia_map
requested_media = copy(
self.logo_paths) # logos aren't language-specific
for lang in build_profile.langs:
requested_media |= self.all_media_paths(lang=lang)
return {
path: self.multimedia_map[path]
for path in requested_media if path in self.multimedia_map
}
# The following functions (get_menu_media, get_case_list_form_media, get_case_list_menu_item_media,
# get_case_list_lookup_image, _get_item_media, and get_media_ref_kwargs) are used to set up context
# for app manager settings pages. Ideally, they'd be moved into ModuleMediaMixin and FormMediaMixin
# and perhaps share logic with those mixins' versions of all_media.
def get_menu_media(self,
module,
form=None,
form_index=None,
to_language=None):
if not module:
# user_registration isn't a real module, for instance
return {}
media_kwargs = self.get_media_ref_kwargs(module,
form=form,
form_index=form_index,
is_menu_media=True)
media_kwargs.update(to_language=to_language or self.default_language)
item = form or module
return self._get_item_media(item, media_kwargs)
def get_case_list_form_media(self, module, to_language=None):
if not module:
# user_registration isn't a real module, for instance
return {}
media_kwargs = self.get_media_ref_kwargs(module)
media_kwargs.update(to_language=to_language or self.default_language)
return self._get_item_media(module.case_list_form, media_kwargs)
def get_case_list_menu_item_media(self, module, to_language=None):
if not module or not module.uses_media() or not hasattr(
module, 'case_list'):
# user_registration isn't a real module, for instance
return {}
media_kwargs = self.get_media_ref_kwargs(module)
media_kwargs.update(to_language=to_language or self.default_language)
return self._get_item_media(module.case_list, media_kwargs)
def get_case_list_lookup_image(self, module, type='case'):
if not module:
return {}
media_kwargs = self.get_media_ref_kwargs(module)
details_name = '{}_details'.format(type)
if not hasattr(module, details_name):
return {}
image = ApplicationMediaReference(
module[details_name].short.lookup_image,
media_class=CommCareImage,
**media_kwargs).as_dict()
return {'image': image}
def _get_item_media(self, item, media_kwargs):
menu_media = {}
to_language = media_kwargs.pop('to_language', self.default_language)
image_ref = ApplicationMediaReference(
item.icon_by_language(to_language),
media_class=CommCareImage,
use_default_media=item.use_default_image_for_all,
**media_kwargs)
image_ref = image_ref.as_dict()
menu_media['image'] = image_ref
audio_ref = ApplicationMediaReference(
item.audio_by_language(to_language),
media_class=CommCareAudio,
use_default_media=item.use_default_audio_for_all,
**media_kwargs)
audio_ref = audio_ref.as_dict()
menu_media['audio'] = audio_ref
return menu_media
def get_media_ref_kwargs(self,
module,
form=None,
form_index=None,
is_menu_media=False):
return {
'app_lang': self.default_language,
'module_name': module.name,
'module_unique_id': module.unique_id,
'form_name': form.name if form else None,
'form_unique_id': form.unique_id if form else None,
'form_order': form_index,
'is_menu_media': is_menu_media,
}
@property
@memoized
def logo_paths(self):
return set(value['path'] for value in self.logo_refs.values())
def remove_unused_mappings(self):
"""
This checks to see if the paths specified in the multimedia map still exist in the Application.
If not, then that item is removed from the multimedia map.
"""
map_changed = False
if self.check_media_state()['has_form_errors']:
return
paths = list(self.multimedia_map) if self.multimedia_map else []
permitted_paths = self.all_media_paths() | self.logo_paths
for path in paths:
if path not in permitted_paths:
map_item = self.multimedia_map[path]
map_changed = True
del self.multimedia_map[path]
if map_changed:
self.save()
def create_mapping(self, multimedia, path, save=True):
"""
This creates the mapping of a path to the multimedia in an application to the media object stored in couch.
"""
path = path.strip()
map_item = HQMediaMapItem()
map_item.multimedia_id = multimedia._id
map_item.unique_id = HQMediaMapItem.gen_unique_id(
map_item.multimedia_id, path)
map_item.media_type = multimedia.doc_type
self.multimedia_map[path] = map_item
if save:
try:
self.save()
except ResourceConflict:
# Attempt to fetch the document again.
updated_doc = self.get(self._id)
updated_doc.create_mapping(multimedia, form_path)
def get_media_objects(self,
build_profile_id=None,
remove_unused=False,
multimedia_map=None):
"""
Gets all the media objects stored in the multimedia map.
If passed a profile, will only get those that are used
in a language in the profile.
Returns a generator of tuples, where the first item in the tuple is
the path (jr://...) and the second is the object (CommCareMultimedia or a subclass)
"""
found_missing_mm = False
# preload all the docs to avoid excessive couch queries.
# these will all be needed in memory anyway so this is ok.
build_profile = self.build_profiles[
build_profile_id] if build_profile_id else None
if not multimedia_map:
multimedia_map = self.multimedia_map_for_build(
build_profile=build_profile, remove_unused=remove_unused)
expected_ids = [
map_item.multimedia_id for map_item in multimedia_map.values()
]
raw_docs = dict(
(d["_id"], d)
for d in iter_docs(CommCareMultimedia.get_db(), expected_ids))
for path, map_item in multimedia_map.items():
media_item = raw_docs.get(map_item.multimedia_id)
if media_item:
media_cls = CommCareMultimedia.get_doc_class(
map_item.media_type)
yield path, media_cls.wrap(media_item)
else:
# Re-attempt to fetch media directly from couch
media = CommCareMultimedia.get_doc_class(map_item.media_type)
try:
media = media.get(map_item.multimedia_id)
yield path, media
except ResourceNotFound as e:
if toggles.CAUTIOUS_MULTIMEDIA.enabled(self.domain):
raise e
def get_object_map(self, multimedia_map=None):
object_map = {}
for path, media_obj in self.get_media_objects(
remove_unused=False, multimedia_map=multimedia_map):
object_map[path] = media_obj.get_media_info(path)
return object_map
def get_reference_totals(self):
"""
Returns a list of totals of each type of media in the application and total matches.
"""
totals = []
for mm in [
CommCareMultimedia.get_doc_class(t)
for t in CommCareMultimedia.get_doc_types()
]:
paths = self.get_all_paths_of_type(mm.__name__)
matched_paths = [
p for p in self.multimedia_map.keys() if p in paths
]
if len(paths) > 0:
totals.append({
'media_type': mm.get_nice_name(),
'totals': len(paths),
'matched': len(matched_paths),
'icon_class': mm.get_icon_class(),
'paths': matched_paths,
})
return totals
def check_media_state(self):
has_missing_refs = False
for media in self.all_media():
try:
self.multimedia_map[media.path]
except KeyError:
has_missing_refs = True
return {
"has_media": bool(self.all_media()),
"has_form_errors": self.media_form_errors,
"has_missing_refs": has_missing_refs,
}
def archive_logos(self):
"""
Archives any uploaded logos in the application.
"""
has_archived = False
if LOGO_ARCHIVE_KEY not in self.archived_media:
self.archived_media[LOGO_ARCHIVE_KEY] = {}
for slug, logo_data in list(self.logo_refs.items()):
self.archived_media[LOGO_ARCHIVE_KEY][slug] = logo_data
has_archived = True
del self.logo_refs[slug]
return has_archived
def restore_logos(self):
"""
Restores any uploaded logos in the application.
"""
has_restored = False
if hasattr(
self,
'archived_media') and LOGO_ARCHIVE_KEY in self.archived_media:
for slug, logo_data in list(
self.archived_media[LOGO_ARCHIVE_KEY].items()):
self.logo_refs[slug] = logo_data
has_restored = True
del self.archived_media[LOGO_ARCHIVE_KEY][slug]
return has_restored
class OpenmrsCaseConfig(DocumentSchema):
# "id_matchers": [
# {
# "case_property": "nid",
# "identifier_type_id": "e2b966d0-1d5f-11e0-b929-000c29ad1d07",
# "doc_type": "IdMatcher"
# },
# {
# "case_property": "openmrs_uuid",
# "identifier_type_id": "uuid",
# "doc_type": "IdMatcher"
# }
# ]
id_matchers = SchemaListProperty(IdMatcher)
# "person_properties": {
# "gender": {
# "doc_type": "CaseProperty",
# "case_property": "gender"
# },
# "birthdate": {
# "doc_type": "CaseProperty",
# "case_property": "dob"
# }
# }
person_properties = SchemaDictProperty(ValueSource)
# "person_preferred_name": {
# "givenName": {
# "doc_type": "CaseProperty",
# "case_property": "given_name"
# },
# "middleName": {
# "doc_type": "CaseProperty",
# "case_property": "middle_name"
# },
# "familyName": {
# "doc_type": "CaseProperty",
# "case_property": "family_name"
# }
# }
person_preferred_name = SchemaDictProperty(ValueSource)
# "person_preferred_address": {
# "address1": {
# "doc_type": "CaseProperty",
# "case_property": "address_1"
# },
# "address2": {
# "doc_type": "CaseProperty",
# "case_property": "address_2"
# },
# "cityVillage": {
# "doc_type": "CaseProperty",
# "case_property": "city"
# }
# }
person_preferred_address = SchemaDictProperty(ValueSource)
# "person_attributes": {
# "e2b966d0-1d5f-11e0-b929-000c29ad1d07": {
# "doc_type": "CaseProperty",
# "case_property": "hiv_status"
# }
# }
person_attributes = SchemaDictProperty(ValueSource)
class OpenmrsRepeater(CaseRepeater):
class Meta(object):
app_label = 'repeaters'
include_app_id_param = False
friendly_name = _("Forward to OpenMRS")
payload_generator_classes = (FormRepeaterJsonPayloadGenerator, )
location_id = StringProperty(default='')
openmrs_config = SchemaProperty(OpenmrsConfig)
_has_config = True
# self.white_listed_case_types must have exactly one case type set
# for Atom feed integration to add cases for OpenMRS patients.
# self.location_id must be set to determine their case owner. The
# owner is set to the first CommCareUser instance found at that
# location.
atom_feed_enabled = BooleanProperty(default=False)
atom_feed_status = SchemaDictProperty(AtomFeedStatus)
def __init__(self, *args, **kwargs):
super(OpenmrsRepeater, self).__init__(*args, **kwargs)
def __eq__(self, other):
return (isinstance(other, self.__class__)
and self.get_id == other.get_id)
@classmethod
def wrap(cls, data):
if 'atom_feed_last_polled_at' in data:
data['atom_feed_status'] = {
ATOM_FEED_NAME_PATIENT: {
'last_polled_at': data.pop('atom_feed_last_polled_at'),
'last_page': data.pop('atom_feed_last_page', None),
}
}
return super(OpenmrsRepeater, cls).wrap(data)
@cached_property
def requests(self):
return Requests(self.domain,
self.url,
self.username,
self.plaintext_password,
verify=self.verify)
@cached_property
def observation_mappings(self):
obs_mappings = defaultdict(list)
for form_config in self.openmrs_config.form_configs:
for obs_mapping in form_config.openmrs_observations:
if obs_mapping.value.check_direction(
DIRECTION_IMPORT) and obs_mapping.case_property:
obs_mappings[obs_mapping.concept].append(obs_mapping)
return obs_mappings
@memoized
def payload_doc(self, repeat_record):
return FormAccessors(repeat_record.domain).get_form(
repeat_record.payload_id)
@property
def form_class_name(self):
"""
The class name used to determine which edit form to use
"""
return self.__class__.__name__
@classmethod
def available_for_domain(cls, domain):
return OPENMRS_INTEGRATION.enabled(domain)
def allowed_to_forward(self, payload):
"""
Forward the payload if ...
* it did not come from OpenMRS, and
* CaseRepeater says it's OK for the case types and users of any
of the payload's cases, and
* this repeater forwards to the right OpenMRS server for any of
the payload's cases.
:param payload: An XFormInstance (not a case)
"""
if payload.xmlns == XMLNS_OPENMRS:
# payload came from OpenMRS. Don't send it back.
return False
case_blocks = extract_case_blocks(payload)
case_ids = [case_block['@case_id'] for case_block in case_blocks]
cases = CaseAccessors(payload.domain).get_cases(case_ids, ordered=True)
if not any(
CaseRepeater.allowed_to_forward(self, case) for case in cases):
# If none of the case updates in the payload are allowed to
# be forwarded, drop it.
return False
if not self.location_id:
# If this repeater does not have a location, all payloads
# should go to it.
return True
repeaters = [
repeater for case in cases
for repeater in get_case_location_ancestor_repeaters(case)
]
# If this repeater points to the wrong OpenMRS server for this
# payload then let the right repeater handle it.
return self in repeaters
def get_payload(self, repeat_record):
payload = super(OpenmrsRepeater, self).get_payload(repeat_record)
return json.loads(payload)
def send_request(self, repeat_record, payload):
value_sources = chain(
self.openmrs_config.case_config.patient_identifiers.values(),
self.openmrs_config.case_config.person_properties.values(),
self.openmrs_config.case_config.person_preferred_name.values(),
self.openmrs_config.case_config.person_preferred_address.values(),
self.openmrs_config.case_config.person_attributes.values(),
)
case_trigger_infos = get_relevant_case_updates_from_form_json(
self.domain,
payload,
case_types=self.white_listed_case_types,
extra_fields=[
vs.case_property for vs in value_sources
if hasattr(vs, 'case_property')
])
form_question_values = get_form_question_values(payload)
return send_openmrs_data(self.requests, self.domain, payload,
self.openmrs_config, case_trigger_infos,
form_question_values)
