class Spam(DocumentSchema): ham = Ham(required=False) ham_prop = SchemaProperty(Ham, required=False) ham_dict_prop = SchemaDictProperty(Ham, required=False)
class IndexTree(DocumentSchema): """ Document type representing a case dependency tree (which is flattened to a single dict) """ # a flat mapping of cases to dicts of their indices. The keys in each dict are the index identifiers # and the values are the referenced case IDs indices = SchemaDictProperty() @property @memoized def reverse_indices(self): return _reverse_index_map(self.indices) def __repr__(self): return json.dumps(self.indices, indent=2) @staticmethod def get_all_dependencies(case_id, child_index_tree, extension_index_tree): """Takes a child and extension index tree and returns returns a set of all dependencies of <case_id> Traverse each incoming index, return each touched case. Traverse each outgoing index in the extension tree, return each touched case """ all_cases = set() cases_to_check = set([case_id]) while cases_to_check: case_to_check = cases_to_check.pop() all_cases.add(case_to_check) incoming_extension_indices = extension_index_tree.get_cases_that_directly_depend_on_case( case_to_check) incoming_child_indices = child_index_tree.get_cases_that_directly_depend_on_case( case_to_check) all_incoming_indices = incoming_extension_indices | incoming_child_indices new_outgoing_cases_to_check = set( extension_index_tree.indices.get(case_to_check, {}).values()) new_cases_to_check = (new_outgoing_cases_to_check | all_incoming_indices) - all_cases cases_to_check |= new_cases_to_check return all_cases @staticmethod @memoized def get_all_outgoing_cases(case_id, child_index_tree, extension_index_tree): """traverse all outgoing child and extension indices""" all_cases = set([case_id]) new_cases = set([case_id]) while new_cases: case_to_check = new_cases.pop() parent_cases = set( child_index_tree.indices.get(case_to_check, {}).values()) host_cases = set( extension_index_tree.indices.get(case_to_check, {}).values()) new_cases = (new_cases | parent_cases | host_cases) - all_cases all_cases = all_cases | parent_cases | host_cases return all_cases @staticmethod @memoized def traverse_incoming_extensions(case_id, extension_index_tree, closed_cases): """traverse open incoming extensions""" all_cases = set([case_id]) new_cases = set([case_id]) while new_cases: case_to_check = new_cases.pop() open_incoming_extension_indices = { case for case in extension_index_tree. get_cases_that_directly_depend_on_case(case_to_check) if case not in closed_cases } for incoming_case in open_incoming_extension_indices: new_cases.add(incoming_case) all_cases.add(incoming_case) return all_cases def get_cases_that_directly_depend_on_case(self, case_id): return self.reverse_indices.get(case_id, set([])) def delete_index(self, from_case_id, index_name): prior_ids = self.indices.pop(from_case_id, {}) prior_ids.pop(index_name, None) if prior_ids: self.indices[from_case_id] = prior_ids self._clear_index_caches() def set_index(self, from_case_id, index_name, to_case_id): prior_ids = self.indices.get(from_case_id, {}) prior_ids[index_name] = to_case_id self.indices[from_case_id] = prior_ids self._clear_index_caches() def _clear_index_caches(self): try: # self.reverse_indices is a memoized property, so we can't just call self.reverse_indices.reset_cache self._reverse_indices_cache.clear() except AttributeError: pass self.get_all_outgoing_cases.reset_cache() self.traverse_incoming_extensions.reset_cache() def apply_updates(self, other_tree): """ Apply updates from another IndexTree and return a copy with those applied. If an id is found in the new one, use that id's indices, otherwise, use this ones, (defaulting to nothing). """ assert isinstance(other_tree, IndexTree) new = IndexTree(indices=copy(self.indices), ) new.indices.update(other_tree.indices) return new def _reverse_index_map(index_map): reverse_indices = defaultdict(set) for case_id, indices in index_map.items(): for indexed_case_id in indices.values(): reverse_indices[indexed_case_id].add(case_id) return dict(reverse_indices) class SimplifiedSyncLog(AbstractSyncLog): """ New, simplified sync log class that is used by ownership cleanliness restore. Just maintains a flat list of case IDs on the phone rather than the case/dependent state lists from the SyncLog class. """ log_format = StringProperty(default=LOG_FORMAT_SIMPLIFIED) case_ids_on_phone = SetProperty(six.text_type) # this is a subset of case_ids_on_phone used to flag that a case is only around because it has dependencies # this allows us to purge it if possible from other actions dependent_case_ids_on_phone = SetProperty(six.text_type) owner_ids_on_phone = SetProperty(six.text_type) index_tree = SchemaProperty(IndexTree) # index tree of subcases / children extension_index_tree = SchemaProperty( IndexTree) # index tree of extensions closed_cases = SetProperty(six.text_type) extensions_checked = BooleanProperty(default=False) device_id = StringProperty() _purged_cases = None @property def purged_cases(self): if self._purged_cases is None: self._purged_cases = set() return self._purged_cases def case_count(self): return len(self.case_ids_on_phone) def phone_is_holding_case(self, case_id): """ Whether the phone currently has a case, according to this sync log """ return case_id in self.case_ids_on_phone def get_footprint_of_cases_on_phone(self): return list(self.case_ids_on_phone) @property def primary_case_ids(self): return self.case_ids_on_phone - self.dependent_case_ids_on_phone def purge(self, case_id, xform_id=None): """ This happens in 3 phases, and recursively tries to purge outgoing indices of purged cases. Definitions: ----------- A case is *relevant* if: - it is open and owned or, - it has a relevant child or, - it has a relevant extension or, - it is the extension of a relevant case. A case is *available* if: - it is open and not an extension case or, - it is open and is the extension of an available case. A case is *live* if: - it is owned and available or, - it has a live child or, - it has a live extension or, - it is the exension of a live case. Algorithm: ---------- 1. Mark *relevant* cases Mark all open cases owned by the user relevant. Traversing all outgoing child and extension indexes, as well as all incoming extension indexes, mark all touched cases relevant. 2. Mark *available* cases Mark all relevant cases that are open and have no outgoing extension indexes as available. Traverse incoming extension indexes which don't lead to closed cases, mark all touched cases as available. 3. Mark *live* cases Mark all relevant, owned, available cases as live. Traverse incoming extension indexes which don't lead to closed cases, mark all touched cases as live. """ _get_logger().debug("purging: {}".format(case_id)) self.dependent_case_ids_on_phone.add(case_id) relevant = self._get_relevant_cases(case_id) available = self._get_available_cases(relevant) live = self._get_live_cases(available) to_remove = (relevant - self.purged_cases) - live self._remove_cases_purge_indices(to_remove, case_id, xform_id) def _get_relevant_cases(self, case_id): """ Mark all open cases owned by the user relevant. Traversing all outgoing child and extension indexes, as well as all incoming extension indexes, mark all touched cases relevant. """ relevant = IndexTree.get_all_dependencies( case_id, child_index_tree=self.index_tree, extension_index_tree=self.extension_index_tree, ) _get_logger().debug("Relevant cases of {}: {}".format( case_id, relevant)) return relevant def _get_available_cases(self, relevant): """ Mark all relevant cases that are open and have no outgoing extension indexes as available. Traverse incoming extension indexes which don't lead to closed cases, mark all touched cases as available """ incoming_extensions = self.extension_index_tree.reverse_indices available = { case for case in relevant if case not in self.closed_cases and ( not self.extension_index_tree.indices.get(case) or self.index_tree.indices.get(case)) } new_available = set() | available while new_available: case_to_check = new_available.pop() for incoming_extension in incoming_extensions.get( case_to_check, []): closed = incoming_extension in self.closed_cases purged = incoming_extension in self.purged_cases if not closed and not purged: new_available.add(incoming_extension) available = available | new_available _get_logger().debug("Available cases: {}".format(available)) return available def _get_live_cases(self, available): """ Mark all relevant, owned, available cases as live. Traverse incoming extension indexes which don't lead to closed cases, mark all touched cases as available. """ primary_case_ids = self.primary_case_ids live = available & primary_case_ids new_live = set() | live checked = set() while new_live: case_to_check = new_live.pop() checked.add(case_to_check) new_live = new_live | IndexTree.get_all_outgoing_cases( case_to_check, self.index_tree, self.extension_index_tree) - self.purged_cases new_live = new_live | IndexTree.traverse_incoming_extensions( case_to_check, self.extension_index_tree, frozenset(self.closed_cases), ) - self.purged_cases new_live = new_live - checked live = live | new_live _get_logger().debug("live cases: {}".format(live)) return live def _remove_cases_purge_indices(self, all_to_remove, checked_case_id, xform_id): """Remove all cases marked for removal. Traverse child cases and try to purge those too.""" _get_logger().debug("cases to to_remove: {}".format(all_to_remove)) for to_remove in all_to_remove: indices = self.index_tree.indices.get(to_remove, {}) self._remove_case(to_remove, all_to_remove, checked_case_id, xform_id) for referenced_case in indices.values(): is_dependent_case = referenced_case in self.dependent_case_ids_on_phone already_primed_for_removal = referenced_case in all_to_remove if is_dependent_case and not already_primed_for_removal and referenced_case != checked_case_id: self.purge(referenced_case, xform_id) def _remove_case(self, to_remove, all_to_remove, checked_case_id, xform_id): """Removes case from index trees, case_ids_on_phone and dependent_case_ids_on_phone if pertinent""" _get_logger().debug('removing: {}'.format(to_remove)) deleted_indices = self.index_tree.indices.pop(to_remove, {}) deleted_indices.update( self.extension_index_tree.indices.pop(to_remove, {})) self._validate_case_removal(to_remove, all_to_remove, deleted_indices, checked_case_id, xform_id) try: self.case_ids_on_phone.remove(to_remove) except KeyError: should_fail_softly = not xform_id or _domain_has_legacy_toggle_set( ) if should_fail_softly: pass else: # this is only a soft assert for now because of http://manage.dimagi.com/default.asp?181443 # we should convert back to a real Exception when we stop getting any of these _assert = soft_assert(notify_admins=True, exponential_backoff=False) _assert( False, 'case already remove from synclog', { 'case_id': to_remove, 'synclog_id': self._id, 'form_id': xform_id }) else: self.purged_cases.add(to_remove) if to_remove in self.dependent_case_ids_on_phone: self.dependent_case_ids_on_phone.remove(to_remove) def _validate_case_removal(self, case_to_remove, all_to_remove, deleted_indices, checked_case_id, xform_id): """Traverse immediate outgoing indices. Validate that these are also candidates for removal.""" if case_to_remove == checked_case_id: return # Logging removed temporarily: https://github.com/dimagi/commcare-hq/pull/16259#issuecomment-303176217 # for index in deleted_indices.values(): # if xform_id and not _domain_has_legacy_toggle_set(): # # unblocking http://manage.dimagi.com/default.asp?185850 # _assert = soft_assert(send_to_ops=False, log_to_file=True, exponential_backoff=True, # fail_if_debug=True) # _assert(index in (all_to_remove | set([checked_case_id])), # "expected {} in {} but wasn't".format(index, all_to_remove)) def _add_primary_case(self, case_id): self.case_ids_on_phone.add(case_id) if case_id in self.dependent_case_ids_on_phone: self.dependent_case_ids_on_phone.remove(case_id) def _add_index(self, index, case_update): _get_logger().debug('adding index {} --<{}>--> {} ({}).'.format( index.case_id, index.relationship, index.referenced_id, index.identifier)) if index.relationship == const.CASE_INDEX_EXTENSION: self._add_extension_index(index, case_update) else: self._add_child_index(index) def _add_extension_index(self, index, case_update): assert index.relationship == const.CASE_INDEX_EXTENSION self.extension_index_tree.set_index(index.case_id, index.identifier, index.referenced_id) if index.referenced_id not in self.case_ids_on_phone: self.case_ids_on_phone.add(index.referenced_id) self.dependent_case_ids_on_phone.add(index.referenced_id) case_child_indices = [ idx for idx in case_update.indices_to_add if idx.relationship == const.CASE_INDEX_CHILD and idx.referenced_id == index.referenced_id ] if not case_child_indices and not case_update.is_live: # this case doesn't also have child indices, and it is not owned, so it is dependent self.dependent_case_ids_on_phone.add(index.case_id) def _add_child_index(self, index): assert index.relationship == const.CASE_INDEX_CHILD self.index_tree.set_index(index.case_id, index.identifier, index.referenced_id) if index.referenced_id not in self.case_ids_on_phone: self.case_ids_on_phone.add(index.referenced_id) self.dependent_case_ids_on_phone.add(index.referenced_id) def _delete_index(self, index): self.index_tree.delete_index(index.case_id, index.identifier) self.extension_index_tree.delete_index(index.case_id, index.identifier) def update_phone_lists(self, xform, case_list): made_changes = False _get_logger().debug('updating sync log for {}'.format(self.user_id)) _get_logger().debug('case ids before update: {}'.format(', '.join( self.case_ids_on_phone))) _get_logger().debug('dependent case ids before update: {}'.format( ', '.join(self.dependent_case_ids_on_phone))) _get_logger().debug('index tree before update: {}'.format( self.index_tree)) _get_logger().debug('extension index tree before update: {}'.format( self.extension_index_tree)) class CaseUpdate(object): def __init__(self, case_id, owner_ids_on_phone): self.case_id = case_id self.owner_ids_on_phone = owner_ids_on_phone self.was_live_previously = True self.final_owner_id = None self.is_closed = None self.indices_to_add = [] self.indices_to_delete = [] @property def extension_indices_to_add(self): return [ index for index in self.indices_to_add if index.relationship == const.CASE_INDEX_EXTENSION ] def has_extension_indices_to_add(self): return len(self.extension_indices_to_add) > 0 @property def is_live(self): """returns whether an update is live for a specifc set of owner_ids""" if self.is_closed: return False elif self.final_owner_id is None: # we likely didn't touch owner_id so just default to whatever it was previously return self.was_live_previously else: return self.final_owner_id in self.owner_ids_on_phone ShortIndex = namedtuple( 'ShortIndex', ['case_id', 'identifier', 'referenced_id', 'relationship']) # this is a variable used via closures in the function below owner_id_map = {} def get_latest_owner_id(case_id, action=None): # "latest" just means as this forms actions are played through if action is not None: owner_id_from_action = action.updated_known_properties.get( "owner_id") if owner_id_from_action is not None: owner_id_map[case_id] = owner_id_from_action return owner_id_map.get(case_id, None) all_updates = {} for case in case_list: if case.case_id not in all_updates: _get_logger().debug('initializing update for case {}'.format( case.case_id)) all_updates[case.case_id] = CaseUpdate( case_id=case.case_id, owner_ids_on_phone=self.owner_ids_on_phone) case_update = all_updates[case.case_id] case_update.was_live_previously = case.case_id in self.primary_case_ids actions = case.get_actions_for_form(xform) for action in actions: _get_logger().debug('{}: {}'.format(case.case_id, action.action_type)) owner_id = get_latest_owner_id(case.case_id, action) if owner_id is not None: case_update.final_owner_id = owner_id if action.action_type == const.CASE_ACTION_INDEX: for index in action.indices: if index.referenced_id: case_update.indices_to_add.append( ShortIndex(case.case_id, index.identifier, index.referenced_id, index.relationship)) else: case_update.indices_to_delete.append( ShortIndex(case.case_id, index.identifier, None, None)) elif action.action_type == const.CASE_ACTION_CLOSE: case_update.is_closed = True non_live_updates = [] for case in case_list: case_update = all_updates[case.case_id] if case_update.is_live: _get_logger().debug('case {} is live.'.format( case_update.case_id)) if case.case_id not in self.case_ids_on_phone: self._add_primary_case(case.case_id) made_changes = True elif case.case_id in self.dependent_case_ids_on_phone: self.dependent_case_ids_on_phone.remove(case.case_id) made_changes = True for index in case_update.indices_to_add: self._add_index(index, case_update) made_changes = True for index in case_update.indices_to_delete: self._delete_index(index) made_changes = True else:
class OpenmrsRepeater(CaseRepeater): """ ``OpenmrsRepeater`` is responsible for updating OpenMRS patients with changes made to cases in CommCare. It is also responsible for creating OpenMRS "visits", "encounters" and "observations" when a corresponding visit form is submitted in CommCare. The ``OpenmrsRepeater`` class is different from most repeater classes in three details: 1. It has a case type and it updates the OpenMRS equivalent of cases like the ``CaseRepeater`` class, but it reads forms like the ``FormRepeater`` class. So it subclasses ``CaseRepeater`` but its payload format is ``form_json``. 2. It makes many API calls for each payload. 3. It can have a location. """ class Meta(object): app_label = 'repeaters' include_app_id_param = False friendly_name = _("Forward to OpenMRS") payload_generator_classes = (FormRepeaterJsonPayloadGenerator, ) location_id = StringProperty(default='') openmrs_config = SchemaProperty(OpenmrsConfig) _has_config = True # self.white_listed_case_types must have exactly one case type set # for Atom feed integration to add cases for OpenMRS patients. # self.location_id must be set to determine their case owner. The # owner is set to the first CommCareUser instance found at that # location. atom_feed_enabled = BooleanProperty(default=False) atom_feed_status = SchemaDictProperty(AtomFeedStatus) def __init__(self, *args, **kwargs): super(OpenmrsRepeater, self).__init__(*args, **kwargs) def __eq__(self, other): return (isinstance(other, self.__class__) and self.get_id == other.get_id) def __str__(self): return Repeater.__str__(self) @classmethod def wrap(cls, data): if 'atom_feed_last_polled_at' in data: data['atom_feed_status'] = { ATOM_FEED_NAME_PATIENT: { 'last_polled_at': data.pop('atom_feed_last_polled_at'), 'last_page': data.pop('atom_feed_last_page', None), } } return super(OpenmrsRepeater, cls).wrap(data) @cached_property def requests(self): # Used by atom_feed module and views that don't have a payload # associated with the request return self.get_requests() def get_requests(self, payload_id=None): return Requests( self.domain, self.url, self.username, self.plaintext_password, verify=self.verify, notify_addresses=self.notify_addresses, payload_id=payload_id, ) @cached_property def first_user(self): return get_one_commcare_user_at_location(self.domain, self.location_id) @memoized def payload_doc(self, repeat_record): return FormAccessors(repeat_record.domain).get_form( repeat_record.payload_id) @property def form_class_name(self): """ The class name used to determine which edit form to use """ return self.__class__.__name__ @classmethod def available_for_domain(cls, domain): return OPENMRS_INTEGRATION.enabled(domain) def allowed_to_forward(self, payload): """ Forward the payload if ... * it did not come from OpenMRS, and * CaseRepeater says it's OK for the case types and users of any of the payload's cases, and * this repeater forwards to the right OpenMRS server for any of the payload's cases. :param payload: An XFormInstance (not a case) """ if payload.xmlns == XMLNS_OPENMRS: # payload came from OpenMRS. Don't send it back. return False case_blocks = extract_case_blocks(payload) case_ids = [case_block['@case_id'] for case_block in case_blocks] cases = CaseAccessors(payload.domain).get_cases(case_ids, ordered=True) if not any( CaseRepeater.allowed_to_forward(self, case) for case in cases): # If none of the case updates in the payload are allowed to # be forwarded, drop it. return False if not self.location_id: # If this repeater does not have a location, all payloads # should go to it. return True repeaters = [ repeater for case in cases for repeater in get_case_location_ancestor_repeaters(case) ] # If this repeater points to the wrong OpenMRS server for this # payload then let the right repeater handle it. return self in repeaters def get_payload(self, repeat_record): payload = super(OpenmrsRepeater, self).get_payload(repeat_record) return json.loads(payload) def send_request(self, repeat_record, payload): value_source_configs: Iterable[JsonDict] = chain( self.openmrs_config.case_config.patient_identifiers.values(), self.openmrs_config.case_config.person_properties.values(), self.openmrs_config.case_config.person_preferred_name.values(), self.openmrs_config.case_config.person_preferred_address.values(), self.openmrs_config.case_config.person_attributes.values(), ) case_trigger_infos = get_relevant_case_updates_from_form_json( self.domain, payload, case_types=self.white_listed_case_types, extra_fields=[ conf["case_property"] for conf in value_source_configs if "case_property" in conf ], form_question_values=get_form_question_values(payload), ) requests = self.get_requests(payload_id=repeat_record.payload_id) try: response = send_openmrs_data( requests, self.domain, payload, self.openmrs_config, case_trigger_infos, ) except Exception as err: requests.notify_exception(str(err)) return OpenmrsResponse(400, 'Bad Request', pformat_json(str(err))) return response
class OpenmrsCaseConfig(DocumentSchema): id_matchers = SchemaListProperty(IdMatcher) person_properties = SchemaDictProperty(ValueSource) person_attributes = SchemaDictProperty(ValueSource)
class FormQuestionSchema(Document): """ Contains information about the questions for a specific form specifically the options that are available (or have ever been available) for any multi-select questions. Calling `update_schema` will load the app and any saved versions of the app that have not already been processed and update the question schema with any new options. """ domain = StringProperty(required=True) app_id = StringProperty(required=True) xmlns = StringProperty(required=True) last_processed_version = IntegerProperty(default=0) processed_apps = SetProperty(str) apps_with_errors = SetProperty(str) question_schema = SchemaDictProperty(QuestionMeta) class Meta(object): app_label = 'export' @classmethod def _get_id(cls, domain, app_id, xmlns): def _none_to_empty_string(str): return str if str is not None else '' key = list(map(_none_to_empty_string, [domain, app_id, xmlns])) return hashlib.sha1(':'.join(key).encode('utf-8')).hexdigest() @classmethod def get_by_key(cls, domain, app_id, xmlns): _id = cls._get_id(domain, app_id, xmlns) return cls.get(_id) @classmethod def get_or_create(cls, domain, app_id, xmlns): try: schema = cls.get_by_key(domain, app_id, xmlns) except ResourceNotFound: old_schemas = FormQuestionSchema.view( 'form_question_schema/by_xmlns', key=[domain, app_id, xmlns], include_docs=True).all() if old_schemas: doc = old_schemas[0].to_json() del doc['_id'] del doc['_rev'] schema = FormQuestionSchema.wrap(doc) schema.save() for old in old_schemas: old.delete() else: schema = FormQuestionSchema(domain=domain, app_id=app_id, xmlns=xmlns) schema.save() return schema def validate(self, required=True): # this isn't always set, so set to empty strings if not found if self.app_id is None: self.app_id = '' super(FormQuestionSchema, self).validate(required=required) if not self.get_id: self._id = self._get_id(self.domain, self.app_id, self.xmlns) def update_schema(self): all_app_ids = get_build_ids_after_version(self.domain, self.app_id, self.last_processed_version) all_seen_apps = self.apps_with_errors | self.processed_apps to_process = [ app_id for app_id in all_app_ids if app_id not in all_seen_apps ] if self.app_id not in all_seen_apps: to_process.append(self.app_id) for app_doc in iter_docs(Application.get_db(), to_process): if is_remote_app(app_doc): continue app = Application.wrap(app_doc) try: self.update_for_app(app) except AppManagerException: self.apps_with_errors.add(app.get_id) self.last_processed_version = app.version if to_process: self.save() def update_for_app(self, app): xform = app.get_xform_by_xmlns(self.xmlns, log_missing=False) if xform: prefix = '/{}/'.format(xform.data_node.tag_name) def to_json_path(xml_path): if not xml_path: return if xml_path.startswith(prefix): xml_path = xml_path[len(prefix):] return 'form.{}'.format(xml_path.replace('/', '.')) for question in xform.get_questions(app.langs): question_path = to_json_path(question['value']) if question['tag'] == 'select': meta = self.question_schema.get( question_path, QuestionMeta( repeat_context=to_json_path(question['repeat']))) for opt in question['options']: if opt['value'] not in meta.options: meta.options.append(opt['value']) self.question_schema[question_path] = meta else: # In the event that a question was previously a multi-select and not one any longer, # we need to clear the question schema self.question_schema.pop(question_path, None) self.processed_apps.add(app.get_id) self.last_processed_version = app.version
class OpenmrsCaseConfig(DocumentSchema): # "patient_identifiers": { # "e2b966d0-1d5f-11e0-b929-000c29ad1d07": { # "doc_type": "CaseProperty", # "case_property": "nid" # }, # "uuid": { # "doc_type": "CaseProperty", # "case_property": "openmrs_uuid", # } # } patient_identifiers = SchemaDictProperty(ValueSource) # The patient_identifiers that are considered reliable # "match_on_ids": ["uuid", "e2b966d0-1d5f-11e0-b929-000c29ad1d07", match_on_ids = ListProperty() # "person_properties": { # "gender": { # "doc_type": "CaseProperty", # "case_property": "gender" # }, # "birthdate": { # "doc_type": "CaseProperty", # "case_property": "dob" # } # } person_properties = SchemaDictProperty(ValueSource) # "patient_finder": { # "doc_type": "WeightedPropertyPatientFinder", # "searchable_properties": ["nid", "family_name"], # "property_weights": [ # {"case_property": "nid", "weight": 0.9}, # // if "match_type" is not given it defaults to "exact" # {"case_property": "family_name", "weight": 0.4}, # { # "case_property": "given_name", # "weight": 0.3, # "match_type": "levenshtein", # // levenshtein function takes edit_distance / len # "match_params": [0.2] # // i.e. 0.2 (20%) is one edit for every 5 characters # // e.g. "Riyaz" matches "Riaz" but not "Riazz" # }, # {"case_property": "city", "weight": 0.2}, # { # "case_property": "dob", # "weight": 0.3, # "match_type": "days_diff", # // days_diff matches based on days difference from given date # "match_params": [364] # } # ] # } patient_finder = PatientFinder(required=False) # "person_preferred_name": { # "givenName": { # "doc_type": "CaseProperty", # "case_property": "given_name" # }, # "middleName": { # "doc_type": "CaseProperty", # "case_property": "middle_name" # }, # "familyName": { # "doc_type": "CaseProperty", # "case_property": "family_name" # } # } person_preferred_name = SchemaDictProperty(ValueSource) # "person_preferred_address": { # "address1": { # "doc_type": "CaseProperty", # "case_property": "address_1" # }, # "address2": { # "doc_type": "CaseProperty", # "case_property": "address_2" # }, # "cityVillage": { # "doc_type": "CaseProperty", # "case_property": "city" # } # } person_preferred_address = SchemaDictProperty(ValueSource) # "person_attributes": { # "c1f4239f-3f10-11e4-adec-0800271c1b75": { # "doc_type": "CaseProperty", # "case_property": "caste" # }, # "c1f455e7-3f10-11e4-adec-0800271c1b75": { # "doc_type": "CasePropertyMap", # "case_property": "class", # "value_map": { # "sc": "c1fcd1c6-3f10-11e4-adec-0800271c1b75", # "general": "c1fc20ab-3f10-11e4-adec-0800271c1b75", # "obc": "c1fb51cc-3f10-11e4-adec-0800271c1b75", # "other_caste": "c207073d-3f10-11e4-adec-0800271c1b75", # "st": "c20478b6-3f10-11e4-adec-0800271c1b75" # } # } # } person_attributes = SchemaDictProperty(ValueSource) @classmethod def wrap(cls, data): if 'id_matchers' in data: # Convert legacy id_matchers to patient_identifiers. e.g. # [{'doc_type': 'IdMatcher' # 'identifier_type_id': 'e2b966d0-1d5f-11e0-b929-000c29ad1d07', # 'case_property': 'nid'}] # to # {'e2b966d0-1d5f-11e0-b929-000c29ad1d07': {'doc_type': 'CaseProperty', 'case_property': 'nid'}}, patient_identifiers = { m['identifier_type_id']: { 'doc_type': 'CaseProperty', 'case_property': m['case_property'] } for m in data['id_matchers'] } data['patient_identifiers'] = patient_identifiers data['match_on_ids'] = list(patient_identifiers) data.pop('id_matchers') # Set default data types for known properties for property_, value_source in chain( data.get('person_properties', {}).items(), data.get('person_preferred_name', {}).items(), data.get('person_preferred_address', {}).items(), ): data_type = OPENMRS_PROPERTIES[property_] value_source.setdefault('external_data_type', data_type) return super(OpenmrsCaseConfig, cls).wrap(data)
class OpenmrsCaseConfig(DocumentSchema): # "patient_identifiers": { # "e2b966d0-1d5f-11e0-b929-000c29ad1d07": { # "doc_type": "CaseProperty", # "case_property": "nid" # }, # "uuid": { # "doc_type": "CaseProperty", # "case_property": "openmrs_uuid", # } # } patient_identifiers = SchemaDictProperty(ValueSource) # The patient_identifiers that are considered reliable # "match_on_ids": ["uuid", "e2b966d0-1d5f-11e0-b929-000c29ad1d07", match_on_ids = ListProperty() # "person_properties": { # "gender": { # "doc_type": "CaseProperty", # "case_property": "gender" # }, # "birthdate": { # "doc_type": "CaseProperty", # "case_property": "dob" # } # } person_properties = SchemaDictProperty(ValueSource) # "patient_finder": { # "doc_type": "WeightedPropertyPatientFinder", # "searchable_properties": ["nid", "family_name"], # "property_weights": [ # {"case_property": "nid", "weight": 0.9}, # {"case_property": "family_name", "weight": 0.4}, # {"case_property": "given_name", "weight": 0.3}, # {"case_property": "city", "weight": 0.2}, # {"case_property": "dob", "weight": 0.3} # ] # } patient_finder = PatientFinder(required=False) # "person_preferred_name": { # "givenName": { # "doc_type": "CaseProperty", # "case_property": "given_name" # }, # "middleName": { # "doc_type": "CaseProperty", # "case_property": "middle_name" # }, # "familyName": { # "doc_type": "CaseProperty", # "case_property": "family_name" # } # } person_preferred_name = SchemaDictProperty(ValueSource) # "person_preferred_address": { # "address1": { # "doc_type": "CaseProperty", # "case_property": "address_1" # }, # "address2": { # "doc_type": "CaseProperty", # "case_property": "address_2" # }, # "cityVillage": { # "doc_type": "CaseProperty", # "case_property": "city" # } # } person_preferred_address = SchemaDictProperty(ValueSource) # "person_attributes": { # "c1f4239f-3f10-11e4-adec-0800271c1b75": { # "doc_type": "CaseProperty", # "case_property": "caste" # }, # "c1f455e7-3f10-11e4-adec-0800271c1b75": { # "doc_type": "CasePropertyMap", # "case_property": "class", # "value_map": { # "sc": "c1fcd1c6-3f10-11e4-adec-0800271c1b75", # "general": "c1fc20ab-3f10-11e4-adec-0800271c1b75", # "obc": "c1fb51cc-3f10-11e4-adec-0800271c1b75", # "other_caste": "c207073d-3f10-11e4-adec-0800271c1b75", # "st": "c20478b6-3f10-11e4-adec-0800271c1b75" # } # } # } person_attributes = SchemaDictProperty(ValueSource) @classmethod def wrap(cls, data): if 'id_matchers' in data: # Convert id_matchers to patient_identifiers. e.g. # [{'doc_type': 'IdMatcher' # 'identifier_type_id': 'e2b966d0-1d5f-11e0-b929-000c29ad1d07', # 'case_property': 'nid'}] # to # {'e2b966d0-1d5f-11e0-b929-000c29ad1d07': {'doc_type': 'CaseProperty', 'case_property': 'nid'}}, patient_identifiers = { m['identifier_type_id']: { 'doc_type': 'CaseProperty', 'case_property': m['case_property'] } for m in data['id_matchers'] } data['patient_identifiers'] = patient_identifiers data['match_on_ids'] = list(patient_identifiers) data.pop('id_matchers') return super(OpenmrsCaseConfig, cls).wrap(data)
class ApplicationMediaMixin(Document, MediaMixin): """ Manages multimedia for itself and sub-objects. """ # keys are the paths to each file in the final application media zip multimedia_map = SchemaDictProperty(HQMediaMapItem) # paths to custom logos logo_refs = DictProperty() archived_media = DictProperty( ) # where we store references to the old logos (or other multimedia) on a downgrade, so that information is not lost @memoized def all_media(self, lang=None): """ Somewhat counterituitively, this contains all media in the app EXCEPT app-level media (logos). """ media = [] self.media_form_errors = False for module in [m for m in self.get_modules() if m.uses_media()]: media.extend(module.all_media(lang=lang)) for form in module.get_forms(): try: form.validate_form() except (XFormValidationError, XFormException): self.media_form_errors = True else: media.extend(form.all_media(lang=lang)) return media def multimedia_map_for_build(self, build_profile=None, remove_unused=False): if self.multimedia_map is None: self.multimedia_map = {} if self.multimedia_map and remove_unused: self.remove_unused_mappings() if not build_profile or not domain_has_privilege( self.domain, privileges.BUILD_PROFILES): return self.multimedia_map requested_media = copy( self.logo_paths) # logos aren't language-specific for lang in build_profile.langs: requested_media |= self.all_media_paths(lang=lang) return { path: self.multimedia_map[path] for path in requested_media if path in self.multimedia_map } # The following functions (get_menu_media, get_case_list_form_media, get_case_list_menu_item_media, # get_case_list_lookup_image, _get_item_media, and get_media_ref_kwargs) are used to set up context # for app manager settings pages. Ideally, they'd be moved into ModuleMediaMixin and FormMediaMixin # and perhaps share logic with those mixins' versions of all_media. def get_menu_media(self, module, form=None, form_index=None, to_language=None): if not module: # user_registration isn't a real module, for instance return {} media_kwargs = self.get_media_ref_kwargs(module, form=form, form_index=form_index, is_menu_media=True) media_kwargs.update(to_language=to_language or self.default_language) item = form or module return self._get_item_media(item, media_kwargs) def get_case_list_form_media(self, module, to_language=None): if not module: # user_registration isn't a real module, for instance return {} media_kwargs = self.get_media_ref_kwargs(module) media_kwargs.update(to_language=to_language or self.default_language) return self._get_item_media(module.case_list_form, media_kwargs) def get_case_list_menu_item_media(self, module, to_language=None): if not module or not module.uses_media() or not hasattr( module, 'case_list'): # user_registration isn't a real module, for instance return {} media_kwargs = self.get_media_ref_kwargs(module) media_kwargs.update(to_language=to_language or self.default_language) return self._get_item_media(module.case_list, media_kwargs) def get_case_list_lookup_image(self, module, type='case'): if not module: return {} media_kwargs = self.get_media_ref_kwargs(module) details_name = '{}_details'.format(type) if not hasattr(module, details_name): return {} image = ApplicationMediaReference( module[details_name].short.lookup_image, media_class=CommCareImage, **media_kwargs).as_dict() return {'image': image} def _get_item_media(self, item, media_kwargs): menu_media = {} to_language = media_kwargs.pop('to_language', self.default_language) image_ref = ApplicationMediaReference( item.icon_by_language(to_language), media_class=CommCareImage, use_default_media=item.use_default_image_for_all, **media_kwargs) image_ref = image_ref.as_dict() menu_media['image'] = image_ref audio_ref = ApplicationMediaReference( item.audio_by_language(to_language), media_class=CommCareAudio, use_default_media=item.use_default_audio_for_all, **media_kwargs) audio_ref = audio_ref.as_dict() menu_media['audio'] = audio_ref return menu_media def get_media_ref_kwargs(self, module, form=None, form_index=None, is_menu_media=False): return { 'app_lang': self.default_language, 'module_name': module.name, 'module_unique_id': module.unique_id, 'form_name': form.name if form else None, 'form_unique_id': form.unique_id if form else None, 'form_order': form_index, 'is_menu_media': is_menu_media, } @property @memoized def logo_paths(self): return set(value['path'] for value in self.logo_refs.values()) def remove_unused_mappings(self): """ This checks to see if the paths specified in the multimedia map still exist in the Application. If not, then that item is removed from the multimedia map. """ map_changed = False if self.check_media_state()['has_form_errors']: return paths = list(self.multimedia_map) if self.multimedia_map else [] permitted_paths = self.all_media_paths() | self.logo_paths for path in paths: if path not in permitted_paths: map_item = self.multimedia_map[path] map_changed = True del self.multimedia_map[path] if map_changed: self.save() def create_mapping(self, multimedia, path, save=True): """ This creates the mapping of a path to the multimedia in an application to the media object stored in couch. """ path = path.strip() map_item = HQMediaMapItem() map_item.multimedia_id = multimedia._id map_item.unique_id = HQMediaMapItem.gen_unique_id( map_item.multimedia_id, path) map_item.media_type = multimedia.doc_type self.multimedia_map[path] = map_item if save: try: self.save() except ResourceConflict: # Attempt to fetch the document again. updated_doc = self.get(self._id) updated_doc.create_mapping(multimedia, form_path) def get_media_objects(self, build_profile_id=None, remove_unused=False, multimedia_map=None): """ Gets all the media objects stored in the multimedia map. If passed a profile, will only get those that are used in a language in the profile. Returns a generator of tuples, where the first item in the tuple is the path (jr://...) and the second is the object (CommCareMultimedia or a subclass) """ found_missing_mm = False # preload all the docs to avoid excessive couch queries. # these will all be needed in memory anyway so this is ok. build_profile = self.build_profiles[ build_profile_id] if build_profile_id else None if not multimedia_map: multimedia_map = self.multimedia_map_for_build( build_profile=build_profile, remove_unused=remove_unused) expected_ids = [ map_item.multimedia_id for map_item in multimedia_map.values() ] raw_docs = dict( (d["_id"], d) for d in iter_docs(CommCareMultimedia.get_db(), expected_ids)) for path, map_item in multimedia_map.items(): media_item = raw_docs.get(map_item.multimedia_id) if media_item: media_cls = CommCareMultimedia.get_doc_class( map_item.media_type) yield path, media_cls.wrap(media_item) else: # Re-attempt to fetch media directly from couch media = CommCareMultimedia.get_doc_class(map_item.media_type) try: media = media.get(map_item.multimedia_id) yield path, media except ResourceNotFound as e: if toggles.CAUTIOUS_MULTIMEDIA.enabled(self.domain): raise e def get_object_map(self, multimedia_map=None): object_map = {} for path, media_obj in self.get_media_objects( remove_unused=False, multimedia_map=multimedia_map): object_map[path] = media_obj.get_media_info(path) return object_map def get_reference_totals(self): """ Returns a list of totals of each type of media in the application and total matches. """ totals = [] for mm in [ CommCareMultimedia.get_doc_class(t) for t in CommCareMultimedia.get_doc_types() ]: paths = self.get_all_paths_of_type(mm.__name__) matched_paths = [ p for p in self.multimedia_map.keys() if p in paths ] if len(paths) > 0: totals.append({ 'media_type': mm.get_nice_name(), 'totals': len(paths), 'matched': len(matched_paths), 'icon_class': mm.get_icon_class(), 'paths': matched_paths, }) return totals def check_media_state(self): has_missing_refs = False for media in self.all_media(): try: self.multimedia_map[media.path] except KeyError: has_missing_refs = True return { "has_media": bool(self.all_media()), "has_form_errors": self.media_form_errors, "has_missing_refs": has_missing_refs, } def archive_logos(self): """ Archives any uploaded logos in the application. """ has_archived = False if LOGO_ARCHIVE_KEY not in self.archived_media: self.archived_media[LOGO_ARCHIVE_KEY] = {} for slug, logo_data in list(self.logo_refs.items()): self.archived_media[LOGO_ARCHIVE_KEY][slug] = logo_data has_archived = True del self.logo_refs[slug] return has_archived def restore_logos(self): """ Restores any uploaded logos in the application. """ has_restored = False if hasattr( self, 'archived_media') and LOGO_ARCHIVE_KEY in self.archived_media: for slug, logo_data in list( self.archived_media[LOGO_ARCHIVE_KEY].items()): self.logo_refs[slug] = logo_data has_restored = True del self.archived_media[LOGO_ARCHIVE_KEY][slug] return has_restored
class OpenmrsCaseConfig(DocumentSchema): # "id_matchers": [ # { # "case_property": "nid", # "identifier_type_id": "e2b966d0-1d5f-11e0-b929-000c29ad1d07", # "doc_type": "IdMatcher" # }, # { # "case_property": "openmrs_uuid", # "identifier_type_id": "uuid", # "doc_type": "IdMatcher" # } # ] id_matchers = SchemaListProperty(IdMatcher) # "person_properties": { # "gender": { # "doc_type": "CaseProperty", # "case_property": "gender" # }, # "birthdate": { # "doc_type": "CaseProperty", # "case_property": "dob" # } # } person_properties = SchemaDictProperty(ValueSource) # "person_preferred_name": { # "givenName": { # "doc_type": "CaseProperty", # "case_property": "given_name" # }, # "middleName": { # "doc_type": "CaseProperty", # "case_property": "middle_name" # }, # "familyName": { # "doc_type": "CaseProperty", # "case_property": "family_name" # } # } person_preferred_name = SchemaDictProperty(ValueSource) # "person_preferred_address": { # "address1": { # "doc_type": "CaseProperty", # "case_property": "address_1" # }, # "address2": { # "doc_type": "CaseProperty", # "case_property": "address_2" # }, # "cityVillage": { # "doc_type": "CaseProperty", # "case_property": "city" # } # } person_preferred_address = SchemaDictProperty(ValueSource) # "person_attributes": { # "e2b966d0-1d5f-11e0-b929-000c29ad1d07": { # "doc_type": "CaseProperty", # "case_property": "hiv_status" # } # } person_attributes = SchemaDictProperty(ValueSource)
class OpenmrsRepeater(CaseRepeater): class Meta(object): app_label = 'repeaters' include_app_id_param = False friendly_name = _("Forward to OpenMRS") payload_generator_classes = (FormRepeaterJsonPayloadGenerator, ) location_id = StringProperty(default='') openmrs_config = SchemaProperty(OpenmrsConfig) _has_config = True # self.white_listed_case_types must have exactly one case type set # for Atom feed integration to add cases for OpenMRS patients. # self.location_id must be set to determine their case owner. The # owner is set to the first CommCareUser instance found at that # location. atom_feed_enabled = BooleanProperty(default=False) atom_feed_status = SchemaDictProperty(AtomFeedStatus) def __init__(self, *args, **kwargs): super(OpenmrsRepeater, self).__init__(*args, **kwargs) def __eq__(self, other): return (isinstance(other, self.__class__) and self.get_id == other.get_id) @classmethod def wrap(cls, data): if 'atom_feed_last_polled_at' in data: data['atom_feed_status'] = { ATOM_FEED_NAME_PATIENT: { 'last_polled_at': data.pop('atom_feed_last_polled_at'), 'last_page': data.pop('atom_feed_last_page', None), } } return super(OpenmrsRepeater, cls).wrap(data) @cached_property def requests(self): return Requests(self.domain, self.url, self.username, self.plaintext_password, verify=self.verify) @cached_property def observation_mappings(self): obs_mappings = defaultdict(list) for form_config in self.openmrs_config.form_configs: for obs_mapping in form_config.openmrs_observations: if obs_mapping.value.check_direction( DIRECTION_IMPORT) and obs_mapping.case_property: obs_mappings[obs_mapping.concept].append(obs_mapping) return obs_mappings @memoized def payload_doc(self, repeat_record): return FormAccessors(repeat_record.domain).get_form( repeat_record.payload_id) @property def form_class_name(self): """ The class name used to determine which edit form to use """ return self.__class__.__name__ @classmethod def available_for_domain(cls, domain): return OPENMRS_INTEGRATION.enabled(domain) def allowed_to_forward(self, payload): """ Forward the payload if ... * it did not come from OpenMRS, and * CaseRepeater says it's OK for the case types and users of any of the payload's cases, and * this repeater forwards to the right OpenMRS server for any of the payload's cases. :param payload: An XFormInstance (not a case) """ if payload.xmlns == XMLNS_OPENMRS: # payload came from OpenMRS. Don't send it back. return False case_blocks = extract_case_blocks(payload) case_ids = [case_block['@case_id'] for case_block in case_blocks] cases = CaseAccessors(payload.domain).get_cases(case_ids, ordered=True) if not any( CaseRepeater.allowed_to_forward(self, case) for case in cases): # If none of the case updates in the payload are allowed to # be forwarded, drop it. return False if not self.location_id: # If this repeater does not have a location, all payloads # should go to it. return True repeaters = [ repeater for case in cases for repeater in get_case_location_ancestor_repeaters(case) ] # If this repeater points to the wrong OpenMRS server for this # payload then let the right repeater handle it. return self in repeaters def get_payload(self, repeat_record): payload = super(OpenmrsRepeater, self).get_payload(repeat_record) return json.loads(payload) def send_request(self, repeat_record, payload): value_sources = chain( self.openmrs_config.case_config.patient_identifiers.values(), self.openmrs_config.case_config.person_properties.values(), self.openmrs_config.case_config.person_preferred_name.values(), self.openmrs_config.case_config.person_preferred_address.values(), self.openmrs_config.case_config.person_attributes.values(), ) case_trigger_infos = get_relevant_case_updates_from_form_json( self.domain, payload, case_types=self.white_listed_case_types, extra_fields=[ vs.case_property for vs in value_sources if hasattr(vs, 'case_property') ]) form_question_values = get_form_question_values(payload) return send_openmrs_data(self.requests, self.domain, payload, self.openmrs_config, case_trigger_infos, form_question_values)