class OpenmrsFormConfig(DocumentSchema): xmlns = StringProperty() # Used to determine the start of a visit and an encounter. The end # of a visit is set to one day (specifically 23:59:59) later. If not # given, the value defaults to when the form was completed according # to the device, /meta/timeEnd. openmrs_start_datetime = DictProperty(required=False) openmrs_visit_type = StringProperty() openmrs_encounter_type = StringProperty() openmrs_form = StringProperty() openmrs_observations = ListProperty(ObservationMapping) bahmni_diagnoses = ListProperty(ObservationMapping)
class PropertyWeight(DocumentSchema): case_property = StringProperty() weight = DecimalProperty() match_type = StringProperty(required=False, choices=MATCH_TYPES, default=MATCH_TYPE_DEFAULT) match_params = ListProperty(required=False)
class Dhis2CaseConfig(DocumentSchema): """ A Dhis2CaseConfig maps a case type to a tracked entity type. """ case_type = StringProperty() # The ID of the Tracked Entity type. e.g. the ID of "Person" te_type_id = StringProperty() # CommCare case indices to export as DHIS2 relationships relationships_to_export = SchemaListProperty(RelationshipConfig) # The case property to store the ID of the corresponding Tracked # Entity instance. If this is not set, MOTECH will search for a # matching Tracked Entity on every payload. tei_id = DictProperty() # The corresponding Org Unit of the case's location org_unit_id = DictProperty() # Attribute Type ID to case property / constant attributes = DictProperty() # Events for this Tracked Entity: form_configs = ListProperty(Dhis2FormConfig) finder_config = SchemaProperty(FinderConfig)
class StandaloneTranslationDoc(TranslationMixin, CouchDocLockableMixIn): """ There is either 0 or 1 StandaloneTranslationDoc doc for each (domain, area). """ domain = StringProperty() # For example, "sms" area = StringProperty() langs = ListProperty() @property def default_lang(self): if len(self.langs) > 0: return self.langs[0] else: return None @classmethod def get_obj(cls, domain, area, *args, **kwargs): return StandaloneTranslationDoc.view("translations/standalone", key=[domain, area], include_docs=True).one() @classmethod def create_obj(cls, domain, area, *args, **kwargs): obj = StandaloneTranslationDoc( domain=domain, area=area, ) obj.save() return obj
class IndexedCaseMapping(DocumentSchema): identifier = StringProperty(required=True, default=DEFAULT_PARENT_IDENTIFIER) case_type = StringProperty(required=True) relationship = StringProperty(required=True, choices=INDEX_RELATIONSHIPS, default=INDEX_RELATIONSHIP_EXTENSION) # Sets case property values of a new extension case or child case. case_properties = ListProperty(required=True)
class Toggle(Document): """ A very simple implementation of a feature toggle. Just a list of items attached to a slug. """ slug = StringProperty() enabled_users = ListProperty() last_modified = DateTimeProperty() class Meta(object): app_label = 'toggle' def save(self, **params): if ('_id' not in self._doc): self._doc['_id'] = generate_toggle_id(self.slug) self.last_modified = datetime.utcnow() super(Toggle, self).save(**params) self.bust_cache() @classmethod @quickcache(['cls.__name__', 'docid'], timeout=60 * 60 * 24) def cached_get(cls, docid): try: return cls.get(docid) except ResourceNotFound: return None @classmethod def get(cls, docid): if not docid.startswith(TOGGLE_ID_PREFIX): docid = generate_toggle_id(docid) return super(Toggle, cls).get(docid, rev=None, db=None, dynamic_properties=True) def add(self, item): """ Adds an item to the toggle. Only saves if necessary. """ if item not in self.enabled_users: self.enabled_users.append(item) self.save() def remove(self, item): """ Removes an item from the toggle. Only saves if necessary. """ if item in self.enabled_users: self.enabled_users.remove(item) self.save() def delete(self): super(Toggle, self).delete() self.bust_cache() def bust_cache(self): self.cached_get.clear(self.__class__, self.slug)
class CommCareCaseGroup(UndoableDocument): """ This is a group of CommCareCases. Useful for managing cases in larger projects. """ name = StringProperty() domain = StringProperty() cases = ListProperty() timezone = StringProperty() def get_time_zone(self): # Necessary for the CommCareCaseGroup to interact with # CommConnect, as if using the CommCareMobileContactMixin. # However, the entire mixin is not necessary. return self.timezone def get_cases(self, limit=None, skip=None): case_ids = self.cases if skip is not None: case_ids = case_ids[skip:] if limit is not None: case_ids = case_ids[:limit] for case in CommCareCase.objects.iter_cases(case_ids, self.domain): if not case.is_deleted: yield case def get_total_cases(self, clean_list=False): if clean_list: self.clean_cases() return len(self.cases) def clean_cases(self): cleaned_list = [] changed = False for case in CommCareCase.objects.iter_cases(self.cases, self.domain): if not case.is_deleted: cleaned_list.append(case.case_id) else: changed = True if changed: self.cases = cleaned_list self.save() def create_delete_record(self, *args, **kwargs): return DeleteCaseGroupRecord(*args, **kwargs) def clear_caches(self): from corehq.apps.casegroups.dbaccessors import get_case_groups_in_domain get_case_groups_in_domain.clear(self.domain) def save(self, *args, **kwargs): self.clear_caches() super(CommCareCaseGroup, self).save(*args, **kwargs) def delete(self, *args, **kwargs): self.clear_caches() super(CommCareCaseGroup, self).delete(*args, **kwargs)
class OpenmrsImporter(Document): """ Import cases from an OpenMRS instance using a report """ domain = StringProperty() server_url = StringProperty() # e.g. "http://www.example.com/openmrs" username = StringProperty() password = StringProperty() # If a domain has multiple OpenmrsImporter instances, for which CommCare location is this one authoritative? location_id = StringProperty() # How often should cases be imported import_frequency = StringProperty(choices=IMPORT_FREQUENCY_CHOICES, default=IMPORT_FREQUENCY_MONTHLY) log_level = IntegerProperty() # OpenMRS UUID of the report of patients to be imported report_uuid = StringProperty() # Can include template params, e.g. {"endDate": "{{ today }}"} # Available template params: "today", "location" report_params = DictProperty() # The case type of imported cases case_type = StringProperty() # The ID of the owner of imported cases, if all imported cases are to have the same owner. To assign imported # cases to different owners, see `location_type` below. owner_id = StringProperty() # If report_params includes "{{ location }}" then location_type_name is used to determine which locations to # pull the report for. Those locations will need an "openmrs_uuid" param set. Imported cases will be owned by # the first mobile worker assigned to that location. If this OpenmrsImporter.location_id is set, only # sub-locations will be returned location_type_name = StringProperty() # external_id should always be the OpenMRS UUID of the patient (and not, for example, a national ID number) # because it is immutable. external_id_column is the column that contains the UUID external_id_column = StringProperty() # Space-separated column(s) to be concatenated to create the case name (e.g. "givenName familyName") name_columns = StringProperty() column_map = ListProperty(ColumnMapping) def __str__(self): return self.server_url
class OpenmrsConfig(DocumentSchema): """ Configuration for an OpenMRS repeater is stored in an ``OpenmrsConfig`` document. The ``case_config`` property maps CommCare case properties (mostly) to patient data, and uses the ``OpenmrsCaseConfig`` document schema. The ``form_configs`` property maps CommCare form questions (mostly) to event, encounter and observation data, and uses the ``OpenmrsFormConfig`` document schema. """ openmrs_provider = StringProperty(required=False) case_config = SchemaProperty(OpenmrsCaseConfig) form_configs = ListProperty(OpenmrsFormConfig)
class CustomDataSourceConfiguration(JsonObject): """ For custom data sources maintained in the repository """ _datasource_id_prefix = CUSTOM_PREFIX domains = ListProperty() config = DictProperty() @classmethod def get_doc_id(cls, domain, table_id): return '{}{}-{}'.format(cls._datasource_id_prefix, domain, table_id) @classmethod def all(cls): for path in settings.CUSTOM_DATA_SOURCES: with open(path) as f: wrapped = cls.wrap(json.load(f)) for domain in wrapped.domains: doc = copy(wrapped.config) doc['domain'] = domain doc['_id'] = cls.get_doc_id(domain, doc['table_id']) yield DataSourceConfiguration.wrap(doc) @classmethod def by_domain(cls, domain): """ Returns a list of DataSourceConfiguration objects, NOT CustomDataSourceConfigurations. """ return [ds for ds in cls.all() if ds.domain == domain] @classmethod def by_id(cls, config_id): """ Returns a DataSourceConfiguration object, NOT a CustomDataSourceConfiguration. """ for ds in cls.all(): if ds.get_id == config_id: return ds raise BadSpecError(_('The data source referenced by this report could ' 'not be found.'))
class CustomReportConfiguration(JsonObject): """ For statically defined reports based off of custom data sources """ domains = ListProperty() report_id = StringProperty() data_source_table = StringProperty() config = DictProperty() @classmethod def get_doc_id(cls, domain, report_id): return '{}{}-{}'.format(CUSTOM_PREFIX, domain, report_id) @classmethod def all(cls): for path in settings.CUSTOM_UCR_REPORTS: with open(path) as f: wrapped = cls.wrap(json.load(f)) for domain in wrapped.domains: doc = copy(wrapped.config) doc['domain'] = domain doc['_id'] = cls.get_doc_id(domain, wrapped.report_id) doc['config_id'] = CustomDataSourceConfiguration.get_doc_id(domain, wrapped.data_source_table) yield ReportConfiguration.wrap(doc) @classmethod def by_domain(cls, domain): """ Returns a list of ReportConfiguration objects, NOT CustomReportConfigurations. """ return [ds for ds in cls.all() if ds.domain == domain] @classmethod def by_id(cls, config_id): """ Returns a ReportConfiguration object, NOT CustomReportConfigurations. """ for ds in cls.all(): if ds.get_id == config_id: return ds raise BadSpecError(_('The report configuration referenced by this report could ' 'not be found.'))
class Toggle(Document): """ A very simple implementation of a feature toggle. Just a list of items attached to a slug. """ slug = StringProperty() enabled_users = ListProperty() last_modified = DateTimeProperty() def save(self, **params): if ('_id' not in self._doc): self._doc['_id'] = generate_toggle_id(self.slug) self.last_modified = datetime.utcnow() super(Toggle, self).save(**params) @classmethod def get(cls, docid, rev=None, db=None, dynamic_properties=True): if not docid.startswith(TOGGLE_ID_PREFIX): docid = generate_toggle_id(docid) return super(Toggle, cls).get(docid, rev=None, db=None, dynamic_properties=True) def add(self, item): """ Adds an item to the toggle. Only saves if necessary. """ if item not in self.enabled_users: self.enabled_users.append(item) self.save() def remove(self, item): """ Removes an item from the toggle. Only saves if necessary. """ if item in self.enabled_users: self.enabled_users.remove(item) self.save()
class WeightedPropertyPatientFinder(PatientFinder): """ Finds patients that match cases by assigning weights to matching property values, and adding those weights to calculate a confidence score. """ # Identifiers that are searchable in OpenMRS. e.g. # [ 'bahmni_id', 'household_id', 'last_name'] searchable_properties = ListProperty() # The weight assigned to a matching property. # [ # {"case_property": "bahmni_id", "weight": 0.9}, # {"case_property": "household_id", "weight": 0.9}, # {"case_property": "dob", "weight": 0.75}, # {"case_property": "first_name", "weight": 0.025}, # {"case_property": "last_name", "weight": 0.025}, # {"case_property": "municipality", "weight": 0.2}, # ] property_weights = ListProperty(PropertyWeight) # The threshold that the sum of weights must pass for a CommCare case to # be considered a match to an OpenMRS patient threshold = DecimalProperty(default=1.0) # If more than one patient passes `threshold`, the margin by which the # weight of the best match must exceed the weight of the second-best match # to be considered correct. confidence_margin = DecimalProperty(default=0.667) # Default: Matches two thirds better than second-best def __init__(self, *args, **kwargs): super(WeightedPropertyPatientFinder, self).__init__(*args, **kwargs) self._property_map = {} def set_property_map(self, case_config): """ Set self._property_map to map OpenMRS properties and attributes to case properties. """ # Example value of case_config:: # # { # "person_properties": { # "birthdate": { # "doc_type": "CaseProperty", # "case_property": "dob" # } # }, # "person_preferred_name": { # "givenName": { # "doc_type": "CaseProperty", # "case_property": "given_name" # }, # "familyName": { # "doc_type": "CaseProperty", # "case_property": "family_name" # } # }, # "person_preferred_address": { # "address1": { # "doc_type": "CaseProperty", # "case_property": "address_1" # }, # "address2": { # "doc_type": "CaseProperty", # "case_property": "address_2" # } # }, # "person_attributes": { # "c1f4239f-3f10-11e4-adec-0800271c1b75": { # "doc_type": "CaseProperty", # "case_property": "caste" # }, # "c1f455e7-3f10-11e4-adec-0800271c1b75": { # "doc_type": "CasePropertyMap", # "case_property": "class", # "value_map": { # "sc": "c1fcd1c6-3f10-11e4-adec-0800271c1b75", # "general": "c1fc20ab-3f10-11e4-adec-0800271c1b75", # "obc": "c1fb51cc-3f10-11e4-adec-0800271c1b75", # "other_caste": "c207073d-3f10-11e4-adec-0800271c1b75", # "st": "c20478b6-3f10-11e4-adec-0800271c1b75" # } # } # } # // ... # } # for person_prop, value_source in case_config['person_properties'].items(): jsonpath = parse('person.' + person_prop) self._property_map.update(get_caseproperty_jsonpathvaluemap(jsonpath, value_source)) for attr_uuid, value_source in case_config['person_attributes'].items(): # jsonpath_rw offers programmatic JSONPath expressions. For details on how to create JSONPath # expressions programmatically see the # `jsonpath_rw documentation <https://github.com/kennknowles/python-jsonpath-rw#programmatic-jsonpath>`__ # # The `Where` JSONPath expression "*jsonpath1* `where` *jsonpath2*" returns nodes matching *jsonpath1* # where a child matches *jsonpath2*. `Cmp` does a comparison in *jsonpath2*. It accepts a # comparison operator and a value. The JSONPath expression below is the equivalent of:: # # (person.attributes[*] where attributeType.uuid eq attr_uuid).value # # This `for` loop will let us extract the person attribute values where their attribute type UUIDs # match those configured in case_config['person_attributes'] jsonpath = Child( Where( Child(Child(Fields('person'), Fields('attributes')), Slice()), Cmp(Child(Fields('attributeType'), Fields('uuid')), eq, attr_uuid) ), Fields('value') ) self._property_map.update(get_caseproperty_jsonpathvaluemap(jsonpath, value_source)) for name_prop, value_source in case_config['person_preferred_name'].items(): jsonpath = parse('person.preferredName.' + name_prop) self._property_map.update(get_caseproperty_jsonpathvaluemap(jsonpath, value_source)) for addr_prop, value_source in case_config['person_preferred_address'].items(): jsonpath = parse('person.preferredAddress.' + addr_prop) self._property_map.update(get_caseproperty_jsonpathvaluemap(jsonpath, value_source)) for id_type_uuid, value_source in case_config['patient_identifiers'].items(): if id_type_uuid == 'uuid': jsonpath = parse('uuid') else: # The JSONPath expression below is the equivalent of:: # # (identifiers[*] where identifierType.uuid eq id_type_uuid).identifier # # Similar to `person_attributes` above, this will extract the person identifier values where # their identifier type UUIDs match those configured in case_config['patient_identifiers'] jsonpath = Child( Where( Child(Fields('identifiers'), Slice()), Cmp(Child(Fields('identifierType'), Fields('uuid')), eq, id_type_uuid) ), Fields('identifier') ) self._property_map.update(get_caseproperty_jsonpathvaluemap(jsonpath, value_source)) def get_score(self, patient, case): """ Return the sum of weighted properties to give an OpenMRS patient a score of how well they match a CommCare case. """ def weights(): for property_weight in self.property_weights: prop = property_weight['case_property'] weight = property_weight['weight'] matches = self._property_map[prop].jsonpath.find(patient) for match in matches: patient_value = match.value value_map = self._property_map[prop].value_map case_value = case.get_case_property(prop) is_equal = value_map.get(patient_value, patient_value) == case_value yield weight if is_equal else 0 return sum(weights()) def find_patients(self, requests, case, case_config): """ Matches cases to patients. Returns a list of patients, each with a confidence score >= self.threshold """ from corehq.motech.openmrs.logger import logger from corehq.motech.openmrs.repeater_helpers import search_patients self.set_property_map(case_config) candidates = {} # key on OpenMRS UUID to filter duplicates for prop in self.searchable_properties: value = case.get_case_property(prop) if value: response_json = search_patients(requests, value) for patient in response_json['results']: score = self.get_score(patient, case) if score >= self.threshold: candidates[patient['uuid']] = PatientScore(patient, score) if not candidates: logger.info( 'Unable to match case "%s" (%s): No candidate patients found.', case.name, case.get_id, ) return [] if len(candidates) == 1: patient = list(candidates.values())[0].patient logger.info( 'Matched case "%s" (%s) to ONLY patient candidate: \n%s', case.name, case.get_id, pformat(patient, indent=2), ) return [patient] patients_scores = sorted(six.itervalues(candidates), key=lambda candidate: candidate.score, reverse=True) if patients_scores[0].score / patients_scores[1].score > 1 + self.confidence_margin: # There is more than a `confidence_margin` difference # (defaults to 10%) in score between the best-ranked # patient and the second-best-ranked patient. Let's go with # Patient One. patient = patients_scores[0].patient logger.info( 'Matched case "%s" (%s) to BEST patient candidate: \n%s', case.name, case.get_id, pformat(patients_scores, indent=2), ) return [patient] # We can't be sure. Just send them all. logger.info( 'Unable to match case "%s" (%s) to patient candidates: \n%s', case.name, case.get_id, pformat(patients_scores, indent=2), ) return [ps.patient for ps in patients_scores]
class StaticReportConfiguration(JsonObject): """ For statically defined reports based off of custom data sources This class keeps the full list of static report configurations relevant to the current environment in memory and upon requests builds a new report configuration from the static report config. See 0002-keep-static-ucr-configurations-in-memory.md """ domains = ListProperty(required=True) report_id = StringProperty(validators=(_check_ids)) data_source_table = StringProperty() config = DictProperty() custom_configurable_report = StringProperty() server_environment = ListProperty(required=True) @classmethod def get_doc_id(cls, domain, report_id, custom_configurable_report): return '{}{}-{}'.format( STATIC_PREFIX if not custom_configurable_report else CUSTOM_REPORT_PREFIX, domain, report_id, ) @classmethod def _all(cls): def __get_all(): for path_or_glob in settings.STATIC_UCR_REPORTS: if os.path.isfile(path_or_glob): yield _get_wrapped_object_from_file(path_or_glob, cls) else: files = glob.glob(path_or_glob) for path in files: yield _get_wrapped_object_from_file(path, cls) filter_by_env = settings.UNIT_TESTING or settings.DEBUG return __get_all() if filter_by_env else _filter_by_server_env( __get_all()) @classmethod @memoized def by_id_mapping(cls): return { cls.get_doc_id(domain, wrapped.report_id, wrapped.custom_configurable_report): (domain, wrapped) for wrapped in cls._all() for domain in wrapped.domains } @classmethod def all(cls): """Only used in tests""" for wrapped in StaticReportConfiguration._all(): for domain in wrapped.domains: yield cls._get_report_config(wrapped, domain) @classmethod def by_domain(cls, domain): """ Returns a list of ReportConfiguration objects, NOT StaticReportConfigurations. """ return [ cls._get_report_config(wrapped, dom) for dom, wrapped in cls.by_id_mapping().values() if domain == dom ] @classmethod def by_id(cls, config_id, domain): """Returns a ReportConfiguration object, NOT StaticReportConfigurations. """ try: report_domain, wrapped = cls.by_id_mapping()[config_id] except KeyError: raise BadSpecError( _('The report configuration referenced by this report could ' 'not be found: %(report_id)s') % {'report_id': config_id}) if domain and report_domain != domain: raise DocumentNotFound( "Document {} of class {} not in domain {}!".format( config_id, ReportConfiguration.__class__.__name__, domain, )) return cls._get_report_config(wrapped, report_domain) @classmethod def by_ids(cls, config_ids): mapping = cls.by_id_mapping() config_by_ids = {} for config_id in set(config_ids): try: domain, wrapped = mapping[config_id] except KeyError: raise ReportConfigurationNotFoundError( _("The following report configuration could not be found: {}" .format(config_id))) config_by_ids[config_id] = cls._get_report_config(wrapped, domain) return config_by_ids @classmethod def report_class_by_domain_and_id(cls, domain, config_id): try: report_domain, wrapped = cls.by_id_mapping()[config_id] except KeyError: raise BadSpecError( _('The report configuration referenced by this report could not be found.' )) if report_domain != domain: raise DocumentNotFound( "Document {} of class {} not in domain {}!".format( config_id, ReportConfiguration.__class__.__name__, domain, )) return wrapped.custom_configurable_report @classmethod def _get_report_config(cls, static_config, domain): doc = copy(static_config.to_json()['config']) doc['domain'] = domain doc['_id'] = cls.get_doc_id(domain, static_config.report_id, static_config.custom_configurable_report) doc['config_id'] = StaticDataSourceConfiguration.get_doc_id( domain, static_config.data_source_table) return ReportConfiguration.wrap(doc)
class StaticDataSourceConfiguration(JsonObject): """ For custom data sources maintained in the repository. This class keeps the full list of static data source configurations relevant to the current environment in memory and upon requests builds a new data source configuration from the static config. See 0002-keep-static-ucr-configurations-in-memory.md """ _datasource_id_prefix = STATIC_PREFIX domains = ListProperty(required=True) server_environment = ListProperty(required=True) config = DictProperty() @classmethod def get_doc_id(cls, domain, table_id): return '{}{}-{}'.format(cls._datasource_id_prefix, domain, table_id) @classmethod @memoized def by_id_mapping(cls): """Memoized method that maps domains to static data source config""" return { cls.get_doc_id(domain, wrapped.config['table_id']): (domain, wrapped) for wrapped in cls._all() for domain in wrapped.domains } @classmethod def _all(cls): """ :return: Generator of all wrapped configs read from disk """ def __get_all(): for path_or_glob in settings.STATIC_DATA_SOURCES: if os.path.isfile(path_or_glob): yield _get_wrapped_object_from_file(path_or_glob, cls) else: files = glob.glob(path_or_glob) for path in files: yield _get_wrapped_object_from_file(path, cls) for provider_path in settings.STATIC_DATA_SOURCE_PROVIDERS: provider_fn = to_function(provider_path, failhard=True) for wrapped, path in provider_fn(): yield wrapped return __get_all() if settings.UNIT_TESTING else _filter_by_server_env( __get_all()) @classmethod def all(cls): """Unoptimized method that get's all configs by re-reading from disk""" for wrapped in cls._all(): for domain in wrapped.domains: yield cls._get_datasource_config(wrapped, domain) @classmethod def by_domain(cls, domain): return [ cls._get_datasource_config(wrapped, dom) for dom, wrapped in cls.by_id_mapping().values() if domain == dom ] @classmethod def by_id(cls, config_id): try: domain, wrapped = cls.by_id_mapping()[config_id] except KeyError: raise StaticDataSourceConfigurationNotFoundError( _('The data source referenced by this report could not be found.' )) return cls._get_datasource_config(wrapped, domain) @classmethod def _get_datasource_config(cls, static_config, domain): doc = deepcopy(static_config.to_json()['config']) doc['domain'] = domain doc['_id'] = cls.get_doc_id(domain, doc['table_id']) return DataSourceConfiguration.wrap(doc)
class ReportConfiguration(UnicodeMixIn, QuickCachedDocumentMixin, Document): """ A report configuration. These map 1:1 with reports that show up in the UI. """ domain = StringProperty(required=True) visible = BooleanProperty(default=True) # config_id of the datasource config_id = StringProperty(required=True) data_source_type = StringProperty( default=DATA_SOURCE_TYPE_STANDARD, choices=[DATA_SOURCE_TYPE_STANDARD, DATA_SOURCE_TYPE_AGGREGATE]) title = StringProperty() description = StringProperty() aggregation_columns = StringListProperty() filters = ListProperty() columns = ListProperty() configured_charts = ListProperty() sort_expression = ListProperty() soft_rollout = DecimalProperty(default=0) # no longer used report_meta = SchemaProperty(ReportMeta) custom_query_provider = StringProperty(required=False) def __unicode__(self): return '{} - {}'.format(self.domain, self.title) def save(self, *args, **kwargs): self.report_meta.last_modified = datetime.utcnow() super(ReportConfiguration, self).save(*args, **kwargs) @property @memoized def filters_without_prefilters(self): return [f for f in self.filters if f['type'] != 'pre'] @property @memoized def prefilters(self): return [f for f in self.filters if f['type'] == 'pre'] @property @memoized def config(self): return get_datasource_config(self.config_id, self.domain, self.data_source_type)[0] @property @memoized def report_columns(self): return [ ReportColumnFactory.from_spec(c, self.is_static) for c in self.columns ] @property @memoized def ui_filters(self): return [ReportFilterFactory.from_spec(f, self) for f in self.filters] @property @memoized def charts(self): return [ChartFactory.from_spec(g._obj) for g in self.configured_charts] @property @memoized def location_column_id(self): cols = [col for col in self.report_columns if col.type == 'location'] if cols: return cols[0].column_id @property def map_config(self): def map_col(column): if column['column_id'] != self.location_column_id: return { 'column_id': column['column_id'], 'label': column['display'] } if self.location_column_id: return { 'location_column_id': self.location_column_id, 'layer_name': { 'XFormInstance': _('Forms'), 'CommCareCase': _('Cases') }.get(self.config.referenced_doc_type, "Layer"), 'columns': [x for x in (map_col(col) for col in self.columns) if x] } @property @memoized def sort_order(self): return [ ReportOrderByFactory.from_spec(e) for e in self.sort_expression ] @property def table_id(self): return self.config.table_id def get_ui_filter(self, filter_slug): for filter in self.ui_filters: if filter.name == filter_slug: return filter return None def get_languages(self): """ Return the languages used in this report's column and filter display properties. Note that only explicitly identified languages are returned. So, if the display properties are all strings, "en" would not be returned. """ langs = set() for item in self.columns + self.filters: if isinstance(item['display'], dict): langs |= set(item['display'].keys()) return langs def validate(self, required=True): from corehq.apps.userreports.reports.data_source import ConfigurableReportDataSource def _check_for_duplicates(supposedly_unique_list, error_msg): # http://stackoverflow.com/questions/9835762/find-and-list-duplicates-in-python-list duplicate_items = set([ item for item in supposedly_unique_list if supposedly_unique_list.count(item) > 1 ]) if len(duplicate_items) > 0: raise BadSpecError( _(error_msg).format(', '.join(sorted(duplicate_items)))) super(ReportConfiguration, self).validate(required) # check duplicates before passing to factory since it chokes on them _check_for_duplicates( [FilterSpec.wrap(f).slug for f in self.filters], 'Filters cannot contain duplicate slugs: {}', ) _check_for_duplicates( [ column_id for c in self.report_columns for column_id in c.get_column_ids() ], 'Columns cannot contain duplicate column_ids: {}', ) # these calls all implicitly do validation ConfigurableReportDataSource.from_spec(self) self.ui_filters self.charts self.sort_order @classmethod @quickcache(['cls.__name__', 'domain']) def by_domain(cls, domain): return get_report_configs_for_domain(domain) @classmethod @quickcache(['cls.__name__', 'domain', 'data_source_id']) def count_by_data_source(cls, domain, data_source_id): return get_number_of_report_configs_by_data_source( domain, data_source_id) def clear_caches(self): super(ReportConfiguration, self).clear_caches() self.by_domain.clear(self.__class__, self.domain) self.count_by_data_source.clear(self.__class__, self.domain, self.config_id) @property def is_static(self): return report_config_id_is_static(self._id)
class DataSourceConfiguration(CachedCouchDocumentMixin, Document, AbstractUCRDataSource): """ A data source configuration. These map 1:1 with database tables that get created. Each data source can back an arbitrary number of reports. """ domain = StringProperty(required=True) engine_id = StringProperty(default=UCR_ENGINE_ID) backend_id = StringProperty(default=UCR_SQL_BACKEND) # no longer used referenced_doc_type = StringProperty(required=True) table_id = StringProperty(required=True) display_name = StringProperty() base_item_expression = DictProperty() configured_filter = DictProperty() configured_indicators = ListProperty() named_expressions = DictProperty() named_filters = DictProperty() meta = SchemaProperty(DataSourceMeta) is_deactivated = BooleanProperty(default=False) last_modified = DateTimeProperty() asynchronous = BooleanProperty(default=False) sql_column_indexes = SchemaListProperty(SQLColumnIndexes) disable_destructive_rebuild = BooleanProperty(default=False) sql_settings = SchemaProperty(SQLSettings) class Meta(object): # prevent JsonObject from auto-converting dates etc. string_conversions = () def __str__(self): return '{} - {}'.format(self.domain, self.display_name) def save(self, **params): self.last_modified = datetime.utcnow() super(DataSourceConfiguration, self).save(**params) @property def data_source_id(self): return self._id def filter(self, document): filter_fn = self._get_main_filter() return filter_fn(document, EvaluationContext(document, 0)) def deleted_filter(self, document): filter_fn = self._get_deleted_filter() return filter_fn and filter_fn(document, EvaluationContext( document, 0)) @memoized def _get_main_filter(self): return self._get_filter([self.referenced_doc_type]) @memoized def _get_deleted_filter(self): return self._get_filter(get_deleted_doc_types( self.referenced_doc_type), include_configured=False) def _get_filter(self, doc_types, include_configured=True): if not doc_types: return None extras = ([self.configured_filter] if include_configured and self.configured_filter else []) built_in_filters = [ self._get_domain_filter_spec(), { 'type': 'or', 'filters': [{ "type": "boolean_expression", "expression": { "type": "property_name", "property_name": "doc_type", }, "operator": "eq", "property_value": doc_type, } for doc_type in doc_types], }, ] return FilterFactory.from_spec( { 'type': 'and', 'filters': built_in_filters + extras, }, context=self.get_factory_context(), ) def _get_domain_filter_spec(self): return { "type": "boolean_expression", "expression": { "type": "property_name", "property_name": "domain", }, "operator": "eq", "property_value": self.domain, } @property @memoized def named_expression_objects(self): named_expression_specs = deepcopy(self.named_expressions) named_expressions = {} spec_error = None while named_expression_specs: number_generated = 0 for name, expression in list(named_expression_specs.items()): try: named_expressions[name] = ExpressionFactory.from_spec( expression, FactoryContext(named_expressions=named_expressions, named_filters={})) number_generated += 1 del named_expression_specs[name] except BadSpecError as bad_spec_error: # maybe a nested name resolution issue, try again on the next pass spec_error = bad_spec_error if number_generated == 0 and named_expression_specs: # we unsuccessfully generated anything on this pass and there are still unresolved # references. we have to fail. assert spec_error is not None raise spec_error return named_expressions @property @memoized def named_filter_objects(self): return { name: FilterFactory.from_spec( filter, FactoryContext(self.named_expression_objects, {})) for name, filter in self.named_filters.items() } def get_factory_context(self): return FactoryContext(self.named_expression_objects, self.named_filter_objects) @property @memoized def default_indicators(self): default_indicators = [ IndicatorFactory.from_spec( { "column_id": "doc_id", "type": "expression", "display_name": "document id", "datatype": "string", "is_nullable": False, "is_primary_key": True, "expression": { "type": "root_doc", "expression": { "type": "property_name", "property_name": "_id" } } }, self.get_factory_context()) ] default_indicators.append( IndicatorFactory.from_spec({ "type": "inserted_at", }, self.get_factory_context())) if self.base_item_expression: default_indicators.append( IndicatorFactory.from_spec({ "type": "repeat_iteration", }, self.get_factory_context())) return default_indicators @property @memoized def indicators(self): return CompoundIndicator( self.display_name, self.default_indicators + [ IndicatorFactory.from_spec(indicator, self.get_factory_context()) for indicator in self.configured_indicators ], None, ) @property @memoized def parsed_expression(self): if self.base_item_expression: return ExpressionFactory.from_spec( self.base_item_expression, context=self.get_factory_context()) return None def get_columns(self): return self.indicators.get_columns() @property @memoized def columns_by_id(self): return {c.id: c for c in self.get_columns()} def get_column_by_id(self, column_id): return self.columns_by_id.get(column_id) def get_items(self, document, eval_context=None): if self.filter(document): if not self.base_item_expression: return [document] else: result = self.parsed_expression(document, eval_context) if result is None: return [] elif isinstance(result, list): return result else: return [result] else: return [] def get_all_values(self, doc, eval_context=None): if not eval_context: eval_context = EvaluationContext(doc) rows = [] for item in self.get_items(doc, eval_context): indicators = self.indicators.get_values(item, eval_context) rows.append(indicators) eval_context.increment_iteration() return rows def get_report_count(self): """ Return the number of ReportConfigurations that reference this data source. """ return ReportConfiguration.count_by_data_source(self.domain, self._id) def validate(self, required=True): super(DataSourceConfiguration, self).validate(required) # these two properties implicitly call other validation self._get_main_filter() self._get_deleted_filter() # validate indicators and column uniqueness columns = [c.id for c in self.indicators.get_columns()] unique_columns = set(columns) if len(columns) != len(unique_columns): for column in set(columns): columns.remove(column) raise DuplicateColumnIdError(columns=columns) if self.referenced_doc_type not in VALID_REFERENCED_DOC_TYPES: raise BadSpecError( _('Report contains invalid referenced_doc_type: {}').format( self.referenced_doc_type)) self.parsed_expression @classmethod def by_domain(cls, domain): return get_datasources_for_domain(domain) @classmethod def all_ids(cls): return [ res['id'] for res in cls.get_db().view( 'userreports/data_sources_by_build_info', reduce=False, include_docs=False) ] @classmethod def all(cls): for result in iter_docs(cls.get_db(), cls.all_ids()): yield cls.wrap(result) @property def is_static(self): return id_is_static(self._id) def deactivate(self): if not self.is_static: self.is_deactivated = True self.save() get_indicator_adapter(self).drop_table() def get_case_type_or_xmlns_filter(self): """Returns a list of case types or xmlns from the filter of this data source. If this can't figure out the case types or xmlns's that filter, then returns [None] Currently always returns a list because it is called by a loop in _iteratively_build_table Could be reworked to return [] to be more pythonic """ if self.referenced_doc_type not in FILTER_INTERPOLATION_DOC_TYPES: return [None] property_name = FILTER_INTERPOLATION_DOC_TYPES[ self.referenced_doc_type] prop_value = self._filter_interploation_helper(self.configured_filter, property_name) return prop_value or [None] def _filter_interploation_helper(self, config_filter, property_name): filter_type = config_filter.get('type') if filter_type == 'and': sub_config_filters = [ self._filter_interploation_helper(f, property_name) for f in config_filter.get('filters') ] for filter_ in sub_config_filters: if filter_[0]: return filter_ if filter_type != 'boolean_expression': return [None] if config_filter['operator'] not in ('eq', 'in'): return [None] expression = config_filter['expression'] if expression['type'] == 'property_name' and expression[ 'property_name'] == property_name: prop_value = config_filter['property_value'] if not isinstance(prop_value, list): prop_value = [prop_value] return prop_value return [None]
class RepeatRecord(Document): """ An record of a particular instance of something that needs to be forwarded with a link to the proper repeater object """ domain = StringProperty() repeater_id = StringProperty() repeater_type = StringProperty() payload_id = StringProperty() overall_tries = IntegerProperty(default=0) max_possible_tries = IntegerProperty(default=6) attempts = ListProperty(RepeatRecordAttempt) cancelled = BooleanProperty(default=False) registered_on = DateTimeProperty() last_checked = DateTimeProperty() failure_reason = StringProperty() next_check = DateTimeProperty() succeeded = BooleanProperty(default=False) @property def record_id(self): return self._id @classmethod def wrap(cls, data): should_bootstrap_attempts = ('attempts' not in data) self = super(RepeatRecord, cls).wrap(data) if should_bootstrap_attempts and self.last_checked: assert not self.attempts self.attempts = [ RepeatRecordAttempt( cancelled=self.cancelled, datetime=self.last_checked, failure_reason=self.failure_reason, success_response=None, next_check=self.next_check, succeeded=self.succeeded, ) ] return self @property @memoized def repeater(self): try: return Repeater.get(self.repeater_id) except ResourceNotFound: return None @property def url(self): warnings.warn( "RepeatRecord.url is deprecated. Use Repeater.get_url instead", DeprecationWarning) if self.repeater: return self.repeater.get_url(self) @property def state(self): state = RECORD_PENDING_STATE if self.succeeded: state = RECORD_SUCCESS_STATE elif self.cancelled: state = RECORD_CANCELLED_STATE elif self.failure_reason: state = RECORD_FAILURE_STATE return state @classmethod def all(cls, domain=None, due_before=None, limit=None): json_now = json_format_datetime(due_before or datetime.utcnow()) repeat_records = RepeatRecord.view( "repeaters/repeat_records_by_next_check", startkey=[domain], endkey=[domain, json_now, {}], include_docs=True, reduce=False, limit=limit, ) return repeat_records @classmethod def count(cls, domain=None): results = RepeatRecord.view( "repeaters/repeat_records_by_next_check", startkey=[domain], endkey=[domain, {}], reduce=True, ).one() return results['value'] if results else 0 def add_attempt(self, attempt): self.attempts.append(attempt) self.last_checked = attempt.datetime self.next_check = attempt.next_check self.succeeded = attempt.succeeded self.cancelled = attempt.cancelled self.failure_reason = attempt.failure_reason def get_numbered_attempts(self): for i, attempt in enumerate(self.attempts): yield i + 1, attempt def postpone_by(self, duration): self.last_checked = datetime.utcnow() self.next_check = self.last_checked + duration self.save() def make_set_next_try_attempt(self, failure_reason): assert self.succeeded is False assert self.next_check is not None now = datetime.utcnow() return RepeatRecordAttempt( cancelled=False, datetime=now, failure_reason=failure_reason, success_response=None, next_check=now + _get_retry_interval(self.last_checked, now), succeeded=False, ) def try_now(self): # try when we haven't succeeded and either we've # never checked, or it's time to check again return not self.succeeded def get_payload(self): return self.repeater.get_payload(self) def get_attempt_info(self): return self.repeater.get_attempt_info(self) def handle_payload_exception(self, exception): now = datetime.utcnow() return RepeatRecordAttempt( cancelled=True, datetime=now, failure_reason=str(exception), success_response=None, next_check=None, succeeded=False, ) def fire(self, force_send=False): if self.try_now() or force_send: self.overall_tries += 1 try: attempt = self.repeater.fire_for_record(self) except Exception as e: log_repeater_error_in_datadog(self.domain, status_code=None, repeater_type=self.repeater_type) attempt = self.handle_payload_exception(e) raise finally: # pycharm warns attempt might not be defined. # that'll only happen if fire_for_record raise a non-Exception exception (e.g. SIGINT) # or handle_payload_exception raises an exception. I'm okay with that. -DMR self.add_attempt(attempt) self.save() @staticmethod def _format_response(response): if not _is_response(response): return None response_body = getattr(response, "text", "") return '{}: {}.\n{}'.format(response.status_code, response.reason, response_body) def handle_success(self, response): """ Log success in Datadog and return a success RepeatRecordAttempt. ``response`` can be a Requests response instance, or True if the payload did not result in an API call. """ now = datetime.utcnow() if _is_response(response): # ^^^ Don't bother logging success in Datadog if the payload # did not need to be sent. (This can happen with DHIS2 if # the form that triggered the forwarder doesn't contain data # for a DHIS2 Event.) log_repeater_success_in_datadog(self.domain, response.status_code, self.repeater_type) return RepeatRecordAttempt( cancelled=False, datetime=now, failure_reason=None, success_response=self._format_response(response), next_check=None, succeeded=True, info=self.get_attempt_info(), ) def handle_failure(self, response): """Do something with the response if the repeater fails """ return self._make_failure_attempt(self._format_response(response), response) def handle_exception(self, exception): """handle internal exceptions """ return self._make_failure_attempt(str(exception), None) def _make_failure_attempt(self, reason, response): log_repeater_error_in_datadog( self.domain, response.status_code if response else None, self.repeater_type) if self.repeater.allow_retries( response) and self.overall_tries < self.max_possible_tries: return self.make_set_next_try_attempt(reason) else: now = datetime.utcnow() return RepeatRecordAttempt( cancelled=True, datetime=now, failure_reason=reason, success_response=None, next_check=None, succeeded=False, info=self.get_attempt_info(), ) def cancel(self): self.next_check = None self.cancelled = True def attempt_forward_now(self): from corehq.motech.repeaters.tasks import process_repeat_record def is_ready(): return self.next_check < datetime.utcnow() def already_processed(): return self.succeeded or self.cancelled or self.next_check is None if already_processed() or not is_ready(): return # Set the next check to happen an arbitrarily long time from now so # if something goes horribly wrong with the delayed task it will not # be lost forever. A check at this time is expected to occur rarely, # if ever, because `process_repeat_record` will usually succeed or # reset the next check to sometime sooner. self.next_check = datetime.utcnow() + timedelta(hours=48) try: self.save() except ResourceConflict: # Another process beat us to the punch. This takes advantage # of Couch DB's optimistic locking, which prevents a process # with stale data from overwriting the work of another. return process_repeat_record.delay(self) def requeue(self): self.cancelled = False self.succeeded = False self.failure_reason = '' self.overall_tries = 0 self.next_check = datetime.utcnow()
class Group(QuickCachedDocumentMixin, UndoableDocument): """ The main use case for these 'groups' of users is currently so that we can break down reports by arbitrary regions. (Things like who sees what reports are determined by permissions.) """ domain = StringProperty() name = StringProperty() # a list of user ids for users users = ListProperty() # a list of user ids that have been removed from the Group. # This is recorded so that we can update the user at a later point removed_users = SetProperty() path = ListProperty() case_sharing = BooleanProperty() reporting = BooleanProperty(default=True) last_modified = DateTimeProperty() # custom data can live here metadata = DictProperty() @classmethod def wrap(cls, data): last_modified = data.get('last_modified') # if it's missing a Z because of the Aug. 2014 migration # that added this in iso_format() without Z, then add a Z if last_modified and dt_no_Z_re.match(last_modified): data['last_modified'] += 'Z' return super(Group, cls).wrap(data) def save(self, *args, **kwargs): self.last_modified = datetime.utcnow() super(Group, self).save(*args, **kwargs) refresh_group_views() @classmethod def save_docs(cls, docs, use_uuids=True): utcnow = datetime.utcnow() for doc in docs: doc['last_modified'] = utcnow super(Group, cls).save_docs(docs, use_uuids) refresh_group_views() bulk_save = save_docs def delete(self): super(Group, self).delete() refresh_group_views() @classmethod def delete_docs(cls, docs, **params): super(Group, cls).delete_docs(docs, **params) refresh_group_views() bulk_delete = delete_docs def clear_caches(self): super(Group, self).clear_caches() self.by_domain.clear(self.__class__, self.domain) self.ids_by_domain.clear(self.__class__, self.domain) def add_user(self, couch_user_id, save=True): if not isinstance(couch_user_id, str): couch_user_id = couch_user_id.user_id if couch_user_id not in self.users: self.users.append(couch_user_id) if couch_user_id in self.removed_users: self.removed_users.remove(couch_user_id) if save: self.save() def remove_user(self, couch_user_id): ''' Returns True if it removed a user, False otherwise ''' if not isinstance(couch_user_id, str): couch_user_id = couch_user_id.user_id if couch_user_id in self.users: for i in range(0, len(self.users)): if self.users[i] == couch_user_id: del self.users[i] self.removed_users.add(couch_user_id) return True return False def add_group(self, group): group.add_to_group(self) def add_to_group(self, group): """ food = Food(path=[food_id]) fruit = Fruit(path=[fruit_id]) If fruit.add_to_group(food._id): then update fruit.path to be [food_id, fruit_id] """ group_id = group._id if group_id in self.path: raise Exception("Group %s is already a member of %s" % ( self.get_id, group_id, )) new_path = [group_id] new_path.extend(self.path) self.path = new_path self.save() def remove_group(self, group): group.remove_from_group(self) def remove_from_group(self, group): """ food = Food(path=[food_id]) fruit = Fruit(path=[food_id, fruit_id]) If fruit.remove_from_group(food._id): then update fruit.path to be [fruit_id] """ group_id = group._id if group_id not in self.path: raise Exception("Group %s is not a member of %s" % ( self.get_id, group_id )) index = 0 for i in range(0, len(self.path)): if self.path[i] == group_id: index = i break self.path = self.path[index:] self.save() def get_user_ids(self, is_active=True): return [user.user_id for user in self.get_users(is_active=is_active)] @memoized def get_users(self, is_active=True, only_commcare=False): def is_relevant_user(user): if user.is_deleted(): return False if only_commcare and user.__class__ != CommCareUser().__class__: return False if is_active and not user.is_active: return False return True users = map(CouchUser.wrap_correctly, iter_docs(self.get_db(), self.users)) return list(filter(is_relevant_user, users)) @memoized def get_static_user_ids(self, is_active=True): return [user.user_id for user in self.get_static_users(is_active)] @classmethod def get_static_user_ids_for_groups(cls, group_ids): static_user_ids = [] for group_id in group_ids: group = cls.get(group_id) static_user_ids.append(group.get_static_user_ids()) return static_user_ids @memoized def get_static_users(self, is_active=True): return self.get_users(is_active) @classmethod @quickcache(['cls.__name__', 'domain']) def by_domain(cls, domain): return group_by_domain(domain) @classmethod def choices_by_domain(cls, domain): group_ids = cls.ids_by_domain(domain) group_choices = [] for group_doc in iter_docs(cls.get_db(), group_ids): group_choices.append((group_doc['_id'], group_doc['name'])) return group_choices @classmethod @quickcache(['cls.__name__', 'domain']) def ids_by_domain(cls, domain): return get_group_ids_by_domain(domain) @classmethod def by_name(cls, domain, name, one=True): result = stale_group_by_name(domain, name) if one and result: return result[0] else: return result @classmethod def by_user_id(cls, user_id, wrap=True): results = cls.view('groups/by_user', key=user_id, include_docs=wrap) if wrap: return results else: return [r['id'] for r in results] @classmethod def get_case_sharing_accessible_locations(cls, domain, user): return [ location.case_sharing_group_object() for location in SQLLocation.objects.accessible_to_user(domain, user).filter(location_type__shares_cases=True) ] @classmethod def get_case_sharing_groups(cls, domain, wrap=True): all_groups = cls.by_domain(domain) if wrap: groups = [group for group in all_groups if group.case_sharing] groups.extend([ location.case_sharing_group_object() for location in SQLLocation.objects.filter(domain=domain, location_type__shares_cases=True) ]) return groups else: return [group._id for group in all_groups if group.case_sharing] @classmethod def get_reporting_groups(cls, domain): key = ['^Reporting', domain] return cls.view( 'groups/by_name', startkey=key, endkey=key + [{}], include_docs=True, stale=settings.COUCH_STALE_QUERY, ).all() def create_delete_record(self, *args, **kwargs): return DeleteGroupRecord(*args, **kwargs) @property def display_name(self): if self.name: return self.name else: return "[No Name]" @classmethod def user_in_group(cls, user_id, group_id): if not user_id or not group_id: return False c = cls.get_db().view( 'groups/by_user', key=user_id, startkey_docid=group_id, endkey_docid=group_id ).count() if c == 0: return False elif c == 1: return True else: raise Exception( "This should just logically not be possible unless the group " "has the user in there twice" ) def is_member_of(self, domain): return self.domain == domain @property def is_deleted(self): return self.doc_type.endswith(DELETED_SUFFIX) def __repr__(self): return ("Group(domain={self.domain!r}, name={self.name!r}, " "case_sharing={self.case_sharing!r})").format(self=self)
class OpenmrsConfig(DocumentSchema): case_config = SchemaProperty(OpenmrsCaseConfig) form_configs = ListProperty(OpenmrsFormConfig)
class OpenmrsFormConfig(DocumentSchema): xmlns = StringProperty() openmrs_visit_type = StringProperty() openmrs_encounter_type = StringProperty() openmrs_form = StringProperty() openmrs_observations = ListProperty(ObservationMapping)
class DataSourceConfiguration(UnicodeMixIn, CachedCouchDocumentMixin, Document): """ A data source configuration. These map 1:1 with database tables that get created. Each data source can back an arbitrary number of reports. """ domain = StringProperty(required=True) engine_id = StringProperty(default=UCR_ENGINE_ID) referenced_doc_type = StringProperty(required=True) table_id = StringProperty(required=True) display_name = StringProperty() base_item_expression = DictProperty() configured_filter = DictProperty() configured_indicators = ListProperty() named_expressions = DictProperty() named_filters = DictProperty() meta = SchemaProperty(DataSourceMeta) is_deactivated = BooleanProperty(default=False) last_modified = DateTimeProperty() class Meta(object): # prevent JsonObject from auto-converting dates etc. string_conversions = () def __unicode__(self): return u'{} - {}'.format(self.domain, self.display_name) def save(self, **params): self.last_modified = datetime.utcnow() super(DataSourceConfiguration, self).save(**params) def filter(self, document): filter_fn = self._get_main_filter() return filter_fn(document, EvaluationContext(document, 0)) def deleted_filter(self, document): filter_fn = self._get_deleted_filter() return filter_fn and filter_fn(document, EvaluationContext( document, 0)) @memoized def _get_main_filter(self): return self._get_filter([self.referenced_doc_type]) @memoized def _get_deleted_filter(self): return self._get_filter(get_deleted_doc_types( self.referenced_doc_type), include_configured=False) def _get_filter(self, doc_types, include_configured=True): if not doc_types: return None extras = ([self.configured_filter] if include_configured and self.configured_filter else []) built_in_filters = [ self._get_domain_filter_spec(), { 'type': 'or', 'filters': [{ 'type': 'property_match', 'property_name': 'doc_type', 'property_value': doc_type, } for doc_type in doc_types], }, ] return FilterFactory.from_spec( { 'type': 'and', 'filters': built_in_filters + extras, }, context=self._get_factory_context(), ) def _get_domain_filter_spec(self): return { 'type': 'property_match', 'property_name': 'domain', 'property_value': self.domain, } @property @memoized def named_expression_objects(self): return { name: ExpressionFactory.from_spec(expression, FactoryContext.empty()) for name, expression in self.named_expressions.items() } @property @memoized def named_filter_objects(self): return { name: FilterFactory.from_spec( filter, FactoryContext(self.named_expression_objects, {})) for name, filter in self.named_filters.items() } def _get_factory_context(self): return FactoryContext(self.named_expression_objects, self.named_filter_objects) @property @memoized def default_indicators(self): default_indicators = [ IndicatorFactory.from_spec( { "column_id": "doc_id", "type": "expression", "display_name": "document id", "datatype": "string", "is_nullable": False, "is_primary_key": True, "expression": { "type": "root_doc", "expression": { "type": "property_name", "property_name": "_id" } } }, self._get_factory_context()) ] default_indicators.append( IndicatorFactory.from_spec({ "type": "inserted_at", }, self._get_factory_context())) if self.base_item_expression: default_indicators.append( IndicatorFactory.from_spec({ "type": "repeat_iteration", }, self._get_factory_context())) return default_indicators @property @memoized def indicators(self): return CompoundIndicator( self.display_name, self.default_indicators + [ IndicatorFactory.from_spec(indicator, self._get_factory_context()) for indicator in self.configured_indicators ]) @property @memoized def parsed_expression(self): if self.base_item_expression: return ExpressionFactory.from_spec( self.base_item_expression, context=self._get_factory_context()) return None def get_columns(self): return self.indicators.get_columns() def get_items(self, document): if self.filter(document): if not self.base_item_expression: return [document] else: result = self.parsed_expression(document) if result is None: return [] elif isinstance(result, list): return result else: return [result] else: return [] def get_all_values(self, doc): return [ self.indicators.get_values(item, EvaluationContext(doc, i)) for i, item in enumerate(self.get_items(doc)) ] def get_report_count(self): """ Return the number of ReportConfigurations that reference this data source. """ return ReportConfiguration.count_by_data_source(self.domain, self._id) def validate(self, required=True): super(DataSourceConfiguration, self).validate(required) # these two properties implicitly call other validation self._get_main_filter() self._get_deleted_filter() # validate indicators and column uniqueness columns = [c.id for c in self.indicators.get_columns()] unique_columns = set(columns) if len(columns) != len(unique_columns): for column in set(columns): columns.remove(column) raise BadSpecError( _('Report contains duplicate column ids: {}').format(', '.join( set(columns)))) self.parsed_expression @classmethod def by_domain(cls, domain): return get_datasources_for_domain(domain) @classmethod def all_ids(cls): return [ res['id'] for res in cls.get_db().view( 'userreports/data_sources_by_build_info', reduce=False, include_docs=False) ] @classmethod def all(cls): for result in iter_docs(cls.get_db(), cls.all_ids()): yield cls.wrap(result) @property def is_static(self): return id_is_static(self._id) def deactivate(self): if not self.is_static: self.is_deactivated = True self.save() IndicatorSqlAdapter(self).drop_table()
class OpenmrsCaseConfig(DocumentSchema): # "patient_identifiers": { # "e2b966d0-1d5f-11e0-b929-000c29ad1d07": { # "case_property": "nid" # }, # "uuid": { # "case_property": "openmrs_uuid", # } # } patient_identifiers = DictProperty() # The patient_identifiers that are considered reliable # "match_on_ids": ["uuid", "e2b966d0-1d5f-11e0-b929-000c29ad1d07", match_on_ids = ListProperty() # "person_properties": { # "gender": { # "case_property": "gender" # }, # "birthdate": { # "case_property": "dob" # } # } person_properties = DictProperty() # "patient_finder": { # "doc_type": "WeightedPropertyPatientFinder", # "searchable_properties": ["nid", "family_name"], # "property_weights": [ # {"case_property": "nid", "weight": 0.9}, # // if "match_type" is not given it defaults to "exact" # {"case_property": "family_name", "weight": 0.4}, # { # "case_property": "given_name", # "weight": 0.3, # "match_type": "levenshtein", # // levenshtein function takes edit_distance / len # "match_params": [0.2] # // i.e. 0.2 (20%) is one edit for every 5 characters # // e.g. "Riyaz" matches "Riaz" but not "Riazz" # }, # {"case_property": "city", "weight": 0.2}, # { # "case_property": "dob", # "weight": 0.3, # "match_type": "days_diff", # // days_diff matches based on days difference from given date # "match_params": [364] # } # ] # } patient_finder = PatientFinder(required=False) # "person_preferred_name": { # "givenName": { # "case_property": "given_name" # }, # "middleName": { # "case_property": "middle_name" # }, # "familyName": { # "case_property": "family_name" # } # } person_preferred_name = DictProperty() # "person_preferred_address": { # "address1": { # "case_property": "address_1" # }, # "address2": { # "case_property": "address_2" # }, # "cityVillage": { # "case_property": "city" # } # } person_preferred_address = DictProperty() # "person_attributes": { # "c1f4239f-3f10-11e4-adec-0800271c1b75": { # "case_property": "caste" # }, # "c1f455e7-3f10-11e4-adec-0800271c1b75": { # "case_property": "class", # "value_map": { # "sc": "c1fcd1c6-3f10-11e4-adec-0800271c1b75", # "general": "c1fc20ab-3f10-11e4-adec-0800271c1b75", # "obc": "c1fb51cc-3f10-11e4-adec-0800271c1b75", # "other_caste": "c207073d-3f10-11e4-adec-0800271c1b75", # "st": "c20478b6-3f10-11e4-adec-0800271c1b75" # } # } # } person_attributes = DictProperty() # Create cases when importing via the Atom feed import_creates_cases = BooleanProperty(default=True) # If we ever need to disable updating cases, ``import_updates_cases`` # could be added here. Similarly, we could replace # ``patient_finder.create_missing`` with ``export_creates_patients`` # and ``export_updates_patients`` @classmethod def wrap(cls, data): if 'id_matchers' in data: # Convert legacy id_matchers to patient_identifiers. e.g. # [{'doc_type': 'IdMatcher' # 'identifier_type_id': 'e2b966d0-1d5f-11e0-b929-000c29ad1d07', # 'case_property': 'nid'}] # to # {'e2b966d0-1d5f-11e0-b929-000c29ad1d07': {'doc_type': 'CaseProperty', 'case_property': 'nid'}}, patient_identifiers = { m['identifier_type_id']: { 'doc_type': 'CaseProperty', 'case_property': m['case_property'] } for m in data['id_matchers'] } data['patient_identifiers'] = patient_identifiers data['match_on_ids'] = list(patient_identifiers) data.pop('id_matchers') # Set default data types for known properties for property_, value_source in chain( data.get('person_properties', {}).items(), data.get('person_preferred_name', {}).items(), data.get('person_preferred_address', {}).items(), ): data_type = OPENMRS_PROPERTIES[property_] value_source.setdefault('external_data_type', data_type) return super(OpenmrsCaseConfig, cls).wrap(data)
class OpenmrsImporter(Document): """ Import cases from an OpenMRS instance using a report """ domain = StringProperty() # TODO: (2020-03-06) Migrate to ConnectionSettings server_url = StringProperty() # e.g. "http://www.example.com/openmrs" username = StringProperty() password = StringProperty() notify_addresses_str = StringProperty( default="") # See also notify_addresses() # If a domain has multiple OpenmrsImporter instances, for which CommCare location is this one authoritative? location_id = StringProperty() # How often should cases be imported import_frequency = StringProperty(choices=IMPORT_FREQUENCY_CHOICES, default=IMPORT_FREQUENCY_MONTHLY) log_level = IntegerProperty() # Timezone name. If not specified, the domain's timezone will be used. timezone = StringProperty() # OpenMRS UUID of the report of patients to be imported report_uuid = StringProperty() # Can include template params, e.g. {"endDate": "{{ today }}"} # Available template params: "today", "location" report_params = DictProperty() # The case type of imported cases case_type = StringProperty() # The ID of the owner of imported cases, if all imported cases are to have the same owner. To assign imported # cases to different owners, see `location_type` below. owner_id = StringProperty() # If report_params includes "{{ location }}" then location_type_name is used to determine which locations to # pull the report for. Those locations will need an "openmrs_uuid" param set. Imported cases will be owned by # the first mobile worker assigned to that location. If this OpenmrsImporter.location_id is set, only # sub-locations will be returned location_type_name = StringProperty() # external_id should always be the OpenMRS UUID of the patient (and not, for example, a national ID number) # because it is immutable. external_id_column is the column that contains the UUID external_id_column = StringProperty() # Space-separated column(s) to be concatenated to create the case name (e.g. "givenName familyName") name_columns = StringProperty() column_map = ListProperty(ColumnMapping) def __str__(self): url = "@".join((self.username, self.server_url)) if self.username else self.server_url return f"<{self.__class__.__name__} {self._id} {url}>" @property def notify_addresses(self): return [ addr for addr in re.split('[, ]+', self.notify_addresses_str) if addr ] @memoized def get_timezone(self): if self.timezone: return coerce_timezone_value(self.timezone) else: return get_timezone_for_domain(self.domain) def should_import_today(self): today = datetime.today() return (self.import_frequency == IMPORT_FREQUENCY_DAILY or (self.import_frequency == IMPORT_FREQUENCY_WEEKLY and today.weekday() == 1 # Tuesday ) or (self.import_frequency == IMPORT_FREQUENCY_MONTHLY and today.day == 1))
class SQLSettings(DocumentSchema): partition_config = SchemaListProperty(SQLPartition) # no longer used citus_config = SchemaProperty(CitusConfig) primary_key = ListProperty()
class QuestionMeta(DocumentSchema): options = ListProperty() repeat_context = StringProperty() class Meta(object): app_label = 'export'
class StaticDataSourceConfiguration(JsonObject): """ For custom data sources maintained in the repository """ _datasource_id_prefix = STATIC_PREFIX domains = ListProperty() config = DictProperty() @classmethod def get_doc_id(cls, domain, table_id): return '{}{}-{}'.format(cls._datasource_id_prefix, domain, table_id) @classmethod @skippable_quickcache([], skip_arg='rebuild') def by_id_mapping(cls, rebuild=False): mapping = {} for wrapped, path in cls._all(): for domain in wrapped.domains: ds_id = cls.get_doc_id(domain, wrapped.config['table_id']) mapping[ds_id] = StaticDataSourceMetadata(ds_id, path, domain) return mapping @classmethod def _all(cls): for path in settings.STATIC_DATA_SOURCES: with open(path) as f: yield cls.wrap(json.load(f)), path for provider_path in settings.STATIC_DATA_SOURCE_PROVIDERS: provider_fn = to_function(provider_path, failhard=True) for wrapped, path in provider_fn(): yield wrapped, path @classmethod def all(cls): for wrapped, path in cls._all(): for domain in wrapped.domains: yield cls._get_datasource_config(wrapped, domain) @classmethod def by_domain(cls, domain): """ Returns a list of DataSourceConfiguration objects, NOT StaticDataSourceConfigurations. """ return [ds for ds in cls.all() if ds.domain == domain] @classmethod def by_id(cls, config_id): """ Returns a DataSourceConfiguration object, NOT a StaticDataSourceConfiguration. """ mapping = cls.by_id_mapping() if config_id not in mapping: mapping = cls.by_id_mapping(rebuild=True) metadata = mapping.get(config_id, None) if not metadata: raise StaticDataSourceConfigurationNotFoundError( _('The data source referenced by this report could not be found.' )) return cls._get_from_metadata(metadata) @classmethod def _get_from_metadata(cls, metadata): with open(metadata.path) as f: wrapped = cls.wrap(json.load(f)) return cls._get_datasource_config(wrapped, metadata.domain) @classmethod def _get_datasource_config(cls, static_config, domain): doc = deepcopy(static_config.to_json()['config']) doc['domain'] = domain doc['_id'] = cls.get_doc_id(domain, doc['table_id']) return DataSourceConfiguration.wrap(doc)
class OpenmrsConfig(DocumentSchema): openmrs_provider = StringProperty(required=False) case_config = SchemaProperty(OpenmrsCaseConfig) form_configs = ListProperty(OpenmrsFormConfig)
class RepeatRecord(Document): """ An record of a particular instance of something that needs to be forwarded with a link to the proper repeater object """ domain = StringProperty() repeater_id = StringProperty() repeater_type = StringProperty() payload_id = StringProperty() overall_tries = IntegerProperty(default=0) max_possible_tries = IntegerProperty(default=3) attempts = ListProperty(RepeatRecordAttempt) cancelled = BooleanProperty(default=False) registered_on = DateTimeProperty() last_checked = DateTimeProperty() failure_reason = StringProperty() next_check = DateTimeProperty() succeeded = BooleanProperty(default=False) @property def record_id(self): return self._id @classmethod def wrap(cls, data): should_bootstrap_attempts = ('attempts' not in data) self = super(RepeatRecord, cls).wrap(data) if should_bootstrap_attempts and self.last_checked: assert not self.attempts self.attempts = [RepeatRecordAttempt( cancelled=self.cancelled, datetime=self.last_checked, failure_reason=self.failure_reason, success_response=None, next_check=self.next_check, succeeded=self.succeeded, )] return self @property @memoized def repeater(self): try: return Repeater.get(self.repeater_id) except ResourceNotFound: return None @property def url(self): warnings.warn("RepeatRecord.url is deprecated. Use Repeater.get_url instead", DeprecationWarning) if self.repeater: return self.repeater.get_url(self) @property def state(self): state = RECORD_PENDING_STATE if self.succeeded: state = RECORD_SUCCESS_STATE elif self.cancelled: state = RECORD_CANCELLED_STATE elif self.failure_reason: state = RECORD_FAILURE_STATE return state @classmethod def all(cls, domain=None, due_before=None, limit=None): json_now = json_format_datetime(due_before or datetime.utcnow()) repeat_records = RepeatRecord.view("repeaters/repeat_records_by_next_check", startkey=[domain], endkey=[domain, json_now, {}], include_docs=True, reduce=False, limit=limit, ) return repeat_records @classmethod def count(cls, domain=None): results = RepeatRecord.view("repeaters/repeat_records_by_next_check", startkey=[domain], endkey=[domain, {}], reduce=True, ).one() return results['value'] if results else 0 def add_attempt(self, attempt): self.attempts.append(attempt) self.last_checked = attempt.datetime self.next_check = attempt.next_check self.succeeded = attempt.succeeded self.cancelled = attempt.cancelled self.failure_reason = attempt.failure_reason def get_numbered_attempts(self): for i, attempt in enumerate(self.attempts): yield i + 1, attempt def postpone_by(self, duration): self.last_checked = datetime.utcnow() self.next_check = self.last_checked + duration self.save() def make_set_next_try_attempt(self, failure_reason): # we use an exponential back-off to avoid submitting to bad urls # too frequently. assert self.succeeded is False assert self.next_check is not None window = timedelta(minutes=0) if self.last_checked: window = self.next_check - self.last_checked window += (window // 2) # window *= 1.5 if window < MIN_RETRY_WAIT: window = MIN_RETRY_WAIT elif window > MAX_RETRY_WAIT: window = MAX_RETRY_WAIT now = datetime.utcnow() return RepeatRecordAttempt( cancelled=False, datetime=now, failure_reason=failure_reason, success_response=None, next_check=now + window, succeeded=False, ) def try_now(self): # try when we haven't succeeded and either we've # never checked, or it's time to check again return not self.succeeded def get_payload(self): return self.repeater.get_payload(self) def get_attempt_info(self): return self.repeater.get_attempt_info(self) def handle_payload_exception(self, exception): now = datetime.utcnow() return RepeatRecordAttempt( cancelled=True, datetime=now, failure_reason=six.text_type(exception), success_response=None, next_check=None, succeeded=False, ) def fire(self, force_send=False): if self.try_now() or force_send: self.overall_tries += 1 try: attempt = self.repeater.fire_for_record(self) except Exception as e: log_repeater_error_in_datadog(self.domain, status_code=None, repeater_type=self.repeater_type) attempt = self.handle_payload_exception(e) raise finally: # pycharm warns attempt might not be defined. # that'll only happen if fire_for_record raise a non-Exception exception (e.g. SIGINT) # or handle_payload_exception raises an exception. I'm okay with that. -DMR self.add_attempt(attempt) self.save() @staticmethod def _format_response(response): return '{}: {}.\n{}'.format( response.status_code, response.reason, getattr(response, 'content', None)) def handle_success(self, response): """Do something with the response if the repeater succeeds """ now = datetime.utcnow() log_repeater_success_in_datadog( self.domain, response.status_code if response else None, self.repeater_type ) return RepeatRecordAttempt( cancelled=False, datetime=now, failure_reason=None, success_response=self._format_response(response) if response else None, next_check=None, succeeded=True, info=self.get_attempt_info(), ) def handle_failure(self, response): """Do something with the response if the repeater fails """ return self._make_failure_attempt(self._format_response(response), response) def handle_exception(self, exception): """handle internal exceptions """ return self._make_failure_attempt(six.text_type(exception), None) def _make_failure_attempt(self, reason, response): log_repeater_error_in_datadog(self.domain, response.status_code if response else None, self.repeater_type) if self.repeater.allow_retries(response) and self.overall_tries < self.max_possible_tries: return self.make_set_next_try_attempt(reason) else: now = datetime.utcnow() return RepeatRecordAttempt( cancelled=True, datetime=now, failure_reason=reason, success_response=None, next_check=None, succeeded=False, info=self.get_attempt_info(), ) def cancel(self): self.next_check = None self.cancelled = True def requeue(self): self.cancelled = False self.succeeded = False self.failure_reason = '' self.overall_tries = 0 self.next_check = datetime.utcnow()
class StaticReportConfiguration(JsonObject): """ For statically defined reports based off of custom data sources """ domains = ListProperty() report_id = StringProperty() data_source_table = StringProperty() config = DictProperty() custom_configurable_report = StringProperty() @classmethod def get_doc_id(cls, domain, report_id, custom_configurable_report): return '{}{}-{}'.format( STATIC_PREFIX if not custom_configurable_report else CUSTOM_REPORT_PREFIX, domain, report_id, ) @classmethod def _all(cls): for path in settings.STATIC_UCR_REPORTS: with open(path) as f: yield cls.wrap(json.load(f)), path @classmethod @skippable_quickcache([], skip_arg='rebuild') def by_id_mapping(cls, rebuild=False): mapping = {} for wrapped, path in StaticReportConfiguration._all(): for domain in wrapped.domains: config_id = cls.get_doc_id(domain, wrapped.report_id, wrapped.custom_configurable_report) mapping[config_id] = StaticReportMetadata( config_id, path, domain) return mapping @classmethod def all(cls): for wrapped, path in StaticReportConfiguration._all(): for domain in wrapped.domains: yield cls._get_report_config(wrapped, domain) @classmethod def by_domain(cls, domain): """ Returns a list of ReportConfiguration objects, NOT StaticReportConfigurations. """ return [ds for ds in cls.all() if ds.domain == domain] @classmethod def by_id(cls, config_id): """ Returns a ReportConfiguration object, NOT StaticReportConfigurations. """ mapping = cls.by_id_mapping() if config_id not in mapping: mapping = cls.by_id_mapping(rebuild=True) metadata = mapping.get(config_id, None) if not metadata: raise BadSpecError( _('The report configuration referenced by this report could ' 'not be found.')) return cls._get_from_metadata(metadata) @classmethod def by_ids(cls, config_ids): config_ids = set(config_ids) mapping = cls.by_id_mapping() if not config_ids <= set(mapping.keys()): mapping = cls.by_id_mapping(rebuild=True) return_configs = [] for config_id in config_ids: metadata = mapping.get(config_id, None) if not metadata: raise ReportConfigurationNotFoundError( _("The following report configuration could not be found: {}" .format(config_id))) return_configs.append(cls._get_from_metadata(metadata)) return return_configs @classmethod def report_class_by_domain_and_id(cls, domain, config_id): for wrapped, path in cls._all(): if cls.get_doc_id(domain, wrapped.report_id, wrapped.custom_configurable_report) == config_id: return wrapped.custom_configurable_report raise BadSpecError( _('The report configuration referenced by this report could ' 'not be found.')) @classmethod def _get_from_metadata(cls, metadata): with open(metadata.path) as f: wrapped = cls.wrap(json.load(f)) domain = metadata.domain return cls._get_report_config(wrapped, domain) @classmethod def _get_report_config(cls, static_config, domain): doc = copy(static_config.to_json()['config']) doc['domain'] = domain doc['_id'] = cls.get_doc_id(domain, static_config.report_id, static_config.custom_configurable_report) doc['config_id'] = StaticDataSourceConfiguration.get_doc_id( domain, static_config.data_source_table) return ReportConfiguration.wrap(doc)