class CObservationAddendum(OldDocument): observed_date = DateProperty() art_observations = SchemaListProperty(CObservation) nonart_observations = SchemaListProperty(CObservation) created_by = StringProperty() created_date = OldDateTimeProperty() notes = StringProperty() # placeholder if need be class Meta: app_label = 'pact'
class Dhis2FormConfig(DocumentSchema): xmlns = StringProperty(required=True) program_id = StringProperty(required=True) enrollment_date = DictProperty(required=False) incident_date = DictProperty(required=False) program_stage_id = DictProperty(required=False) program_status = DictProperty( required=False, default={"value": DHIS2_PROGRAM_STATUS_ACTIVE}) org_unit_id = DictProperty( required=False, default={"form_user_ancestor_location_field": LOCATION_DHIS_ID}) event_date = DictProperty(required=True, default={ "form_question": "/metadata/received_on", "external_data_type": DHIS2_DATA_TYPE_DATE, }) event_status = DictProperty( required=False, default={"value": DHIS2_EVENT_STATUS_COMPLETED}) completed_date = DictProperty(required=False) datavalue_maps = SchemaListProperty(FormDataValueMap) event_location = DictProperty(required=False, default={}) @classmethod def wrap(cls, data): if isinstance(data.get('org_unit_id'), str): # Convert org_unit_id from a string to a ConstantValue data['org_unit_id'] = {'value': data['org_unit_id']} if isinstance(data.get('event_status'), str): data['event_status'] = {'value': data['event_status']} if isinstance(data.get('program_status'), str): data['program_status'] = {'value': data['program_status']} return super(Dhis2FormConfig, cls).wrap(data)
class ApplicationAccess(QuickCachedDocumentMixin, Document): """ This is used to control which users/groups can access which applications on cloudcare. """ domain = StringProperty() app_groups = SchemaListProperty(AppGroup, default=[]) restrict = BooleanProperty(default=False)
class DataSetMap(Document): # domain and UCR uniquely identify a DataSetMap domain = StringProperty() connection_settings_id = IntegerProperty(required=False, default=None) ucr_id = StringProperty() # UCR ReportConfig id description = StringProperty() frequency = StringProperty(choices=SEND_FREQUENCIES, default=SEND_FREQUENCY_MONTHLY) # Day of the month for monthly/quarterly frequency. Day of the week # for weekly frequency. Uses ISO-8601, where Monday = 1, Sunday = 7. day_to_send = IntegerProperty() data_set_id = StringProperty() # If UCR adds values to an existing DataSet org_unit_id = StringProperty() # If all values are for the same OrganisationUnit. org_unit_column = StringProperty() # if not org_unit_id: use org_unit_column period = StringProperty() # If all values are for the same period. Monthly is YYYYMM, quarterly is YYYYQ# period_column = StringProperty() # if not period: use period_column attribute_option_combo_id = StringProperty() # Optional. DHIS2 defaults this to categoryOptionCombo complete_date = StringProperty() # Optional complete_date_option = StringProperty( default=COMPLETE_DATE_EMPTY, choices=COMPLETE_DATE_CHOICES, ) complete_date_column = StringProperty() # Optional datavalue_maps = SchemaListProperty(DataValueMap) @property def connection_settings(self): if self.connection_settings_id: return ConnectionSettings.objects.get(pk=self.connection_settings_id) @property def pk(self): return self._id
class CustomDataFieldsDefinition(SyncCouchToSQLMixin, QuickCachedDocumentMixin, Document): """ Per-project user-defined fields such as custom user data. """ field_type = StringProperty() base_doc = "CustomDataFieldsDefinition" domain = StringProperty() fields = SchemaListProperty(CustomDataField) @classmethod def _migration_get_fields(cls): return ["domain", "field_type"] def _migration_sync_to_sql(self, sql_object): for field_name in self._migration_get_fields(): value = getattr(self, field_name) setattr(sql_object, field_name, value) if not sql_object.id: sql_object.save(sync_to_couch=False) sql_object.set_fields([ SQLField( slug=field.slug, is_required=field.is_required, label=field.label, choices=field.choices, regex=field.regex, regex_msg=field.regex_msg, ) for field in self.fields ]) sql_object.save(sync_to_couch=False) @classmethod def _migration_get_sql_model_class(cls): return SQLCustomDataFieldsDefinition
class Dhis2FormConfig(DocumentSchema): xmlns = StringProperty(required=True) program_id = StringProperty(required=True) org_unit_id = SchemaProperty( ValueSource, required=False, default=FormUserAncestorLocationField(location_field=LOCATION_DHIS_ID)) event_date = SchemaProperty(ValueSource, required=True, default=FormQuestion( form_question="/metadata/received_on", external_data_type=DHIS2_DATA_TYPE_DATE, )) event_status = StringProperty( choices=DHIS2_EVENT_STATUSES, default=DHIS2_EVENT_STATUS_COMPLETED, ) completed_date = SchemaProperty(ValueSource, required=False) datavalue_maps = SchemaListProperty(FormDataValueMap) @classmethod def wrap(cls, data): if isinstance(data.get('org_unit_id'), str): # Convert org_unit_id from a string to a ConstantString data['org_unit_id'] = { 'doc_type': 'ConstantString', 'value': data['org_unit_id'] } return super(Dhis2FormConfig, cls).wrap(data)
class Dhis2CaseConfig(DocumentSchema): """ A Dhis2CaseConfig maps a case type to a tracked entity type. """ case_type = StringProperty() # The ID of the Tracked Entity type. e.g. the ID of "Person" te_type_id = StringProperty() # CommCare case indices to export as DHIS2 relationships relationships_to_export = SchemaListProperty(RelationshipConfig) # The case property to store the ID of the corresponding Tracked # Entity instance. If this is not set, MOTECH will search for a # matching Tracked Entity on every payload. tei_id = DictProperty() # The corresponding Org Unit of the case's location org_unit_id = DictProperty() # Attribute Type ID to case property / constant attributes = DictProperty() # Events for this Tracked Entity: form_configs = ListProperty(Dhis2FormConfig) finder_config = SchemaProperty(FinderConfig)
class GroupExportConfiguration(Document): """ An export configuration allows you to setup a collection of exports that all run together. Used by the management command or a scheduled job to run a bunch of exports on a schedule. """ full_exports = SchemaListProperty(ExportConfiguration) custom_export_ids = StringListProperty() def get_custom_exports(self): for custom in list(self.custom_export_ids): custom_export = self._get_custom(custom) if custom_export: yield custom_export def _get_custom(self, custom_id): """ Get a custom export, or delete it's reference if not found """ try: return SavedExportSchema.get(custom_id) except ResourceNotFound: try: self.custom_export_ids.remove(custom_id) self.save() except ValueError: pass @property @memoized def all_configs(self): """ Return an iterator of config-like objects that include the main configs + the custom export configs. """ return [full for full in self.full_exports] + \ [custom.to_export_config() for custom in self.get_custom_exports()] @property def all_export_schemas(self): """ Return an iterator of ExportSchema-like objects that include the main configs + the custom export configs. """ for full in self.full_exports: yield DefaultExportSchema(index=full.index, type=full.type) for custom in self.get_custom_exports(): yield custom @property @memoized def all_exports(self): """ Returns an iterator of tuples consisting of the export config and an ExportSchema-like document that can be used to get at the data. """ return list(zip(self.all_configs, self.all_export_schemas))
class IndexedCaseMapping(DocumentSchema): identifier = StringProperty(required=True, default=DEFAULT_PARENT_IDENTIFIER) case_type = StringProperty(required=True) relationship = StringProperty(required=True, choices=INDEX_RELATIONSHIPS, default=INDEX_RELATIONSHIP_EXTENSION) # Sets case property values of a new extension case or child case. case_properties = SchemaListProperty(ValueSource, required=True)
class FieldList(DocumentSchema): """ List of fields for different combinations of properties """ field_list = SchemaListProperty(FixtureItemField) def to_api_json(self): value = self.to_json() del value['doc_type'] for field in value['field_list']: del field['doc_type'] return value
class ApplicationAccess(QuickCachedDocumentMixin, Document): """ This is used to control which users/groups can access which applications on cloudcare. """ domain = StringProperty() app_groups = SchemaListProperty(AppGroup, default=[]) restrict = BooleanProperty(default=False) @classmethod def get_by_domain(cls, domain): from corehq.apps.cloudcare.dbaccessors import get_application_access_for_domain self = get_application_access_for_domain(domain) return self or cls(domain=domain) def clear_caches(self): from corehq.apps.cloudcare.dbaccessors import get_application_access_for_domain get_application_access_for_domain.clear(self.domain) super(ApplicationAccess, self).clear_caches() def user_can_access_app(self, user, app): user_id = user['_id'] app_id = app['_id'] if not self.restrict or user['doc_type'] == 'WebUser': return True app_group = None for app_group in self.app_groups: if app_group.app_id in (app_id, app['copy_of'] or ()): break if app_group: return Group.user_in_group(user_id, app_group.group_id) else: return False @classmethod def get_template_json(cls, domain, apps): app_ids = dict([(app['_id'], app) for app in apps]) self = ApplicationAccess.get_by_domain(domain) j = self.to_json() merged_access_list = [] for a in j['app_groups']: app_id = a['app_id'] if app_id in app_ids: merged_access_list.append(a) del app_ids[app_id] for app in app_ids.values(): merged_access_list.append({'app_id': app['_id'], 'group_id': None}) j['app_groups'] = merged_access_list return j
class Dhis2FormConfig(DocumentSchema): xmlns = StringProperty() program_id = StringProperty(required=True) org_unit_id = SchemaProperty(ValueSource, required=False) event_date = SchemaProperty(ValueSource, required=True) event_status = StringProperty( choices=DHIS2_EVENT_STATUSES, default=DHIS2_EVENT_STATUS_COMPLETED, ) datavalue_maps = SchemaListProperty(FormDataValueMap) @classmethod def wrap(cls, data): if isinstance(data.get('org_unit_id'), str): # Convert org_unit_id from a string to a ConstantString data['org_unit_id'] = { 'doc_type': 'ConstantString', 'value': data['org_unit_id'] } return super(Dhis2FormConfig, cls).wrap(data)
class CaseState(LooselyEqualDocumentSchema, IndexHoldingMixIn): """ Represents the state of a case on a phone. """ case_id = StringProperty() type = StringProperty() indices = SchemaListProperty(CommCareCaseIndex) @classmethod def from_case(cls, case): if isinstance(case, dict): return cls.wrap({ 'case_id': case['_id'], 'type': case['type'], 'indices': case['indices'], }) return cls( case_id=case.case_id, type=case.type, indices=case.indices, )
class StaticDataSourceConfiguration(JsonObject): """ For custom data sources maintained in the repository. This class keeps the full list of static data source configurations relevant to the current environment in memory and upon requests builds a new data source configuration from the static config. See 0002-keep-static-ucr-configurations-in-memory.md """ _datasource_id_prefix = STATIC_PREFIX domains = ListProperty(required=True) server_environment = ListProperty(required=True) config = DictProperty() mirrored_engine_ids = SchemaListProperty(MirroredEngineIds) @classmethod def get_doc_id(cls, domain, table_id): return '{}{}-{}'.format(cls._datasource_id_prefix, domain, table_id) @classmethod @memoized def by_id_mapping(cls): """Memoized method that maps domains to static data source config""" return { cls.get_doc_id(domain, wrapped.config['table_id']): (domain, wrapped) for wrapped in cls._all() for domain in wrapped.domains } @classmethod def _all(cls): """ :return: Generator of all wrapped configs read from disk """ def __get_all(): paths = list(settings.STATIC_DATA_SOURCES) paths.extend(static_ucr_data_source_paths()) for path_or_glob in paths: if os.path.isfile(path_or_glob): yield _get_wrapped_object_from_file(path_or_glob, cls) else: files = glob.glob(path_or_glob) for path in files: yield _get_wrapped_object_from_file(path, cls) for provider_path in settings.STATIC_DATA_SOURCE_PROVIDERS: provider_fn = to_function(provider_path, failhard=True) for wrapped, path in provider_fn(): yield wrapped return __get_all() if settings.UNIT_TESTING else _filter_by_server_env(__get_all()) @classmethod def all(cls): """Unoptimized method that get's all configs by re-reading from disk""" for wrapped in cls._all(): for domain in wrapped.domains: yield cls._get_datasource_config(wrapped, domain) @classmethod def by_domain(cls, domain): return [ cls._get_datasource_config(wrapped, dom) for dom, wrapped in cls.by_id_mapping().values() if domain == dom ] @classmethod def by_id(cls, config_id): try: domain, wrapped = cls.by_id_mapping()[config_id] except KeyError: raise StaticDataSourceConfigurationNotFoundError(_( 'The data source %(config_id)s referenced by this report could not be found.' ) % {'config_id': config_id}) return cls._get_datasource_config(wrapped, domain) @classmethod def _get_datasource_config(cls, static_config, domain): doc = deepcopy(static_config.to_json()['config']) doc['domain'] = domain doc['_id'] = cls.get_doc_id(domain, doc['table_id']) def _get_mirrored_engine_ids(): for env in static_config.mirrored_engine_ids: if env.server_environment == settings.SERVER_ENVIRONMENT: return env.engine_ids return [] doc['mirrored_engine_ids'] = _get_mirrored_engine_ids() return DataSourceConfiguration.wrap(doc)
class SQLSettings(DocumentSchema): partition_config = SchemaListProperty(SQLPartition) # no longer used citus_config = SchemaProperty(CitusConfig) primary_key = ListProperty()
class DataSetMap(Document): # domain and UCR uniquely identify a DataSetMap domain = StringProperty() ucr_id = StringProperty() # UCR ReportConfig id description = StringProperty() frequency = StringProperty(choices=SEND_FREQUENCIES, default=SEND_FREQUENCY_MONTHLY) day_to_send = IntegerProperty() data_set_id = StringProperty() # If UCR adds values to an existing DataSet org_unit_id = StringProperty() # If all values are for the same OrganisationUnit. org_unit_column = StringProperty() # if not org_unit_id: use org_unit_column period = StringProperty() # If all values are for the same period. Monthly is YYYYMM, quarterly is YYYYQ# period_column = StringProperty() # if not period: use period_column attribute_option_combo_id = StringProperty() # Optional. DHIS2 defaults this to categoryOptionCombo complete_date = StringProperty() # Optional datavalue_maps = SchemaListProperty(DataValueMap) @quickcache(['self.domain', 'self.ucr_id']) def get_datavalue_map_dict(self): dict_ = {dvm.column: dict(dvm, is_org_unit=False, is_period=False) for dvm in self.datavalue_maps} if self.org_unit_column: dict_[self.org_unit_column] = {'is_org_unit': True, 'is_period': False} if self.period_column: dict_[self.period_column] = {'is_org_unit': False, 'is_period': True} return dict_ def get_datavalues(self, ucr_row): """ Returns rows of "dataElement", "categoryOptionCombo", "value", and optionally "period", "orgUnit" and "comment" for this DataSet where ucr_row looks like:: { "org_unit_id": "ABC", "data_element_cat_option_combo_1": 123, "data_element_cat_option_combo_2": 456, "data_element_cat_option_combo_3": 789, } """ dv_map = self.get_datavalue_map_dict() datavalues = [] org_unit = None period = None # First pass is to collate data element IDs and values for key, value in ucr_row.items(): if key in dv_map: if dv_map[key]['is_org_unit']: org_unit = value elif dv_map[key]['is_period']: period = value else: datavalue = { 'dataElement': dv_map[key]['data_element_id'], 'categoryOptionCombo': dv_map[key]['category_option_combo_id'], 'value': value, } if dv_map[key].get('comment'): datavalue['comment'] = dv_map[key]['comment'] datavalues.append(datavalue) # Second pass is to set period and org unit if period or org_unit: for datavalue in datavalues: if period: datavalue['period'] = period if org_unit: datavalue['orgUnit'] = org_unit return datavalues def get_dataset(self, send_date): report_config = get_report_config(self.domain, self.ucr_id) date_filter = get_date_filter(report_config) if self.frequency == SEND_FREQUENCY_MONTHLY: date_range = get_previous_month(send_date) period = date_range.startdate.strftime('%Y%m') elif self.frequency == SEND_FREQUENCY_QUARTERLY: date_range = get_previous_quarter(send_date) period = date_range.startdate.strftime('%Y') + 'Q' + str((date_range.startdate.month // 3) + 1) ucr_data = get_ucr_data(report_config, date_filter, date_range) datavalues = (self.get_datavalues(row) for row in ucr_data) # one UCR row may have many DataValues dataset = { 'dataValues': list(chain.from_iterable(datavalues)) # get a single list of DataValues } if self.data_set_id: dataset['dataSet'] = self.data_set_id if self.org_unit_id: dataset['orgUnit'] = self.org_unit_id if self.period: dataset['period'] = self.period elif not self.period_column: dataset['period'] = period if self.attribute_option_combo_id: dataset['attributeOptionCombo'] = self.attribute_option_combo_id if self.complete_date: dataset['completeDate'] = self.complete_date return dataset def should_send_on_date(self, send_date): return self.day_to_send == send_date.day and ( self.frequency == SEND_FREQUENCY_MONTHLY or self.frequency == SEND_FREQUENCY_QUARTERLY and send_date.month in [1, 4, 7, 10])
class Domain(QuickCachedDocumentMixin, Document, SnapshotMixin): """Domain is the highest level collection of people/stuff in the system. Pretty much everything happens at the domain-level, including user membership, permission to see data, reports, charts, etc.""" name = StringProperty() is_active = BooleanProperty() date_created = DateTimeProperty() default_timezone = StringProperty( default=getattr(settings, "TIME_ZONE", "UTC")) case_sharing = BooleanProperty(default=False) secure_submissions = BooleanProperty(default=False) cloudcare_releases = StringProperty( choices=['stars', 'nostars', 'default'], default='default') organization = StringProperty() hr_name = StringProperty() # the human-readable name for this project creating_user = StringProperty( ) # username of the user who created this domain # domain metadata project_type = StringProperty() # e.g. MCH, HIV customer_type = StringProperty() # plus, full, etc. is_test = StringProperty(choices=["true", "false", "none"], default="none") description = StringProperty() short_description = StringProperty() is_shared = BooleanProperty(default=False) commtrack_enabled = BooleanProperty(default=False) call_center_config = SchemaProperty(CallCenterProperties) has_careplan = BooleanProperty(default=False) restrict_superusers = BooleanProperty(default=False) allow_domain_requests = BooleanProperty(default=False) location_restriction_for_users = BooleanProperty(default=False) usercase_enabled = BooleanProperty(default=False) hipaa_compliant = BooleanProperty(default=False) use_sql_backend = BooleanProperty(default=False) case_display = SchemaProperty(CaseDisplaySettings) # CommConnect settings commconnect_enabled = BooleanProperty(default=False) survey_management_enabled = BooleanProperty(default=False) # Whether or not a case can register via sms sms_case_registration_enabled = BooleanProperty(default=False) # Case type to apply to cases registered via sms sms_case_registration_type = StringProperty() # Owner to apply to cases registered via sms sms_case_registration_owner_id = StringProperty() # Submitting user to apply to cases registered via sms sms_case_registration_user_id = StringProperty() # Whether or not a mobile worker can register via sms sms_mobile_worker_registration_enabled = BooleanProperty(default=False) use_default_sms_response = BooleanProperty(default=False) default_sms_response = StringProperty() chat_message_count_threshold = IntegerProperty() custom_chat_template = StringProperty( ) # See settings.CUSTOM_CHAT_TEMPLATES custom_case_username = StringProperty( ) # Case property to use when showing the case's name in a chat window # If empty, sms can be sent at any time. Otherwise, only send during # these windows of time. SMS_QUEUE_ENABLED must be True in localsettings # for this be considered. restricted_sms_times = SchemaListProperty(DayTimeWindow) # If empty, this is ignored. Otherwise, the framework will make sure # that during these days/times, no automated outbound sms will be sent # to someone if they have sent in an sms within sms_conversation_length # minutes. Outbound sms sent from a user in a chat window, however, will # still be sent. This is meant to prevent chat conversations from being # interrupted by automated sms reminders. # SMS_QUEUE_ENABLED must be True in localsettings for this to be # considered. sms_conversation_times = SchemaListProperty(DayTimeWindow) # In minutes, see above. sms_conversation_length = IntegerProperty(default=10) # Set to True to prevent survey questions and answers form being seen in # SMS chat windows. filter_surveys_from_chat = BooleanProperty(default=False) # The below option only matters if filter_surveys_from_chat = True. # If set to True, invalid survey responses will still be shown in the chat # window, while questions and valid responses will be filtered out. show_invalid_survey_responses_in_chat = BooleanProperty(default=False) # If set to True, if a message is read by anyone it counts as being read by # everyone. Set to False so that a message is only counted as being read # for a user if only that user has read it. count_messages_as_read_by_anyone = BooleanProperty(default=False) # Set to True to allow sending sms and all-label surveys to cases whose # phone number is duplicated with another contact send_to_duplicated_case_numbers = BooleanProperty(default=True) enable_registration_welcome_sms_for_case = BooleanProperty(default=False) enable_registration_welcome_sms_for_mobile_worker = BooleanProperty( default=False) sms_survey_date_format = StringProperty() # exchange/domain copying stuff is_snapshot = BooleanProperty(default=False) is_approved = BooleanProperty(default=False) snapshot_time = DateTimeProperty() published = BooleanProperty(default=False) license = StringProperty(choices=LICENSES, default='cc') title = StringProperty() cda = SchemaProperty(LicenseAgreement) multimedia_included = BooleanProperty(default=True) downloads = IntegerProperty( default=0) # number of downloads for this specific snapshot full_downloads = IntegerProperty( default=0) # number of downloads for all snapshots from this domain author = StringProperty() phone_model = StringProperty() attribution_notes = StringProperty() publisher = StringProperty(choices=["organization", "user"], default="user") yt_id = StringProperty() snapshot_head = BooleanProperty(default=False) deployment = SchemaProperty(Deployment) image_path = StringProperty() image_type = StringProperty() cached_properties = DictProperty() internal = SchemaProperty(InternalProperties) dynamic_reports = SchemaListProperty(DynamicReportSet) # extra user specified properties tags = StringListProperty() area = StringProperty(choices=AREA_CHOICES) sub_area = StringProperty(choices=SUB_AREA_CHOICES) launch_date = DateTimeProperty # to be eliminated from projects and related documents when they are copied for the exchange _dirty_fields = ('admin_password', 'admin_password_charset', 'city', 'countries', 'region', 'customer_type') default_mobile_worker_redirect = StringProperty(default=None) last_modified = DateTimeProperty(default=datetime(2015, 1, 1)) # when turned on, use SECURE_TIMEOUT for sessions of users who are members of this domain secure_sessions = BooleanProperty(default=False) two_factor_auth = BooleanProperty(default=False) strong_mobile_passwords = BooleanProperty(default=False) # There is no longer a way to request a report builder trial, so this property should be removed in the near # future. (Keeping it for now in case a user has requested a trial and but has not yet been granted it) requested_report_builder_trial = StringListProperty() requested_report_builder_subscription = StringListProperty() @classmethod def wrap(cls, data): # for domains that still use original_doc should_save = False if 'original_doc' in data: original_doc = data['original_doc'] del data['original_doc'] should_save = True if original_doc: original_doc = Domain.get_by_name(original_doc) data['copy_history'] = [original_doc._id] # for domains that have a public domain license if 'license' in data: if data.get("license", None) == "public": data["license"] = "cc" should_save = True if 'slug' in data and data["slug"]: data["hr_name"] = data["slug"] del data["slug"] if 'is_test' in data and isinstance(data["is_test"], bool): data["is_test"] = "true" if data["is_test"] else "false" should_save = True if 'cloudcare_releases' not in data: data['cloudcare_releases'] = 'nostars' # legacy default setting # Don't actually remove location_types yet. We can migrate fully and # remove this after everything's hunky-dory in production. 2015-03-06 if 'location_types' in data: data['obsolete_location_types'] = data.pop('location_types') self = super(Domain, cls).wrap(data) if self.deployment is None: self.deployment = Deployment() if should_save: self.save() return self def get_default_timezone(self): """return a timezone object from self.default_timezone""" import pytz return pytz.timezone(self.default_timezone) @staticmethod @quickcache(['name'], timeout=24 * 60 * 60) def is_secure_session_required(name): domain = Domain.get_by_name(name) return domain and domain.secure_sessions @staticmethod @skippable_quickcache(['couch_user._id', 'is_active'], skip_arg='strict', timeout=5 * 60, memoize_timeout=10) def active_for_couch_user(couch_user, is_active=True, strict=False): domain_names = couch_user.get_domains() return Domain.view( "domain/by_status", keys=[[is_active, d] for d in domain_names], reduce=False, include_docs=True, stale=settings.COUCH_STALE_QUERY if not strict else None, ).all() @staticmethod def active_for_user(user, is_active=True, strict=False): if isinstance(user, AnonymousUser): return [] from corehq.apps.users.models import CouchUser if isinstance(user, CouchUser): couch_user = user else: couch_user = CouchUser.from_django_user(user) if couch_user: return Domain.active_for_couch_user(couch_user, is_active=is_active, strict=strict) else: return [] @classmethod def field_by_prefix(cls, field, prefix=''): # unichr(0xfff8) is something close to the highest character available res = cls.view( "domain/fields_by_prefix", group=True, startkey=[field, True, prefix], endkey=[field, True, "%s%c" % (prefix, unichr(0xfff8)), {}]) vals = [(d['value'], d['key'][2]) for d in res] vals.sort(reverse=True) return [(v[1], v[0]) for v in vals] def add(self, model_instance, is_active=True): """ Add something to this domain, through the generic relation. Returns the created membership object """ # Add membership info to Couch couch_user = model_instance.get_profile().get_couch_user() couch_user.add_domain_membership(self.name) couch_user.save() def applications(self): return get_brief_apps_in_domain(self.name) def full_applications(self, include_builds=True): from corehq.apps.app_manager.models import Application, RemoteApp WRAPPERS = {'Application': Application, 'RemoteApp': RemoteApp} def wrap_application(a): return WRAPPERS[a['doc']['doc_type']].wrap(a['doc']) if include_builds: startkey = [self.name] endkey = [self.name, {}] else: startkey = [self.name, None] endkey = [self.name, None, {}] return Application.get_db().view('app_manager/applications', startkey=startkey, endkey=endkey, include_docs=True, wrapper=wrap_application).all() @cached_property def versions(self): apps = self.applications() return list(set(a.application_version for a in apps)) @cached_property def has_case_management(self): for app in self.full_applications(): if app.doc_type == 'Application': if app.has_case_management(): return True return False @cached_property def has_media(self): for app in self.full_applications(): if app.doc_type == 'Application' and app.has_media(): return True return False @property def use_cloudcare_releases(self): return self.cloudcare_releases != 'nostars' def all_users(self): from corehq.apps.users.models import CouchUser return CouchUser.by_domain(self.name) def recent_submissions(self): return domain_has_submission_in_last_30_days(self.name) @cached_property def languages(self): apps = self.applications() return set(chain.from_iterable([a.langs for a in apps])) def readable_languages(self): return ', '.join(lang_lookup[lang] or lang for lang in self.languages()) def __unicode__(self): return self.name @classmethod @skippable_quickcache(['name'], skip_arg='strict', timeout=30 * 60) def get_by_name(cls, name, strict=False): if not name: # get_by_name should never be called with name as None (or '', etc) # I fixed the code in such a way that if I raise a ValueError # all tests pass and basic pages load, # but in order not to break anything in the wild, # I'm opting to notify by email if/when this happens # but fall back to the previous behavior of returning None if settings.DEBUG: raise ValueError('%r is not a valid domain name' % name) else: _assert = soft_assert(notify_admins=True, exponential_backoff=False) _assert(False, '%r is not a valid domain name' % name) return None def _get_by_name(stale=False): extra_args = {'stale': settings.COUCH_STALE_QUERY} if stale else {} result = cls.view("domain/domains", key=name, reduce=False, include_docs=True, **extra_args).first() if not isinstance(result, Domain): # A stale view may return a result with no doc if the doc has just been deleted. # In this case couchdbkit just returns the raw view result as a dict return None else: return result domain = _get_by_name(stale=(not strict)) if domain is None and not strict: # on the off chance this is a brand new domain, try with strict domain = _get_by_name(stale=False) return domain @classmethod def get_or_create_with_name(cls, name, is_active=False, secure_submissions=True): result = cls.view("domain/domains", key=name, reduce=False, include_docs=True).first() if result: return result else: new_domain = Domain( name=name, is_active=is_active, date_created=datetime.utcnow(), secure_submissions=secure_submissions, ) new_domain.save(**get_safe_write_kwargs()) return new_domain @classmethod def generate_name(cls, hr_name, max_length=25): ''' Generate a URL-friendly name based on a given human-readable name. Normalizes given name, then looks for conflicting domains, addressing conflicts by adding "-1", "-2", etc. May return None if it fails to generate a new, unique name. Throws exception if it can't figure out a name, which shouldn't happen unless max_length is absurdly short. ''' name = name_to_url(hr_name, "project") if Domain.get_by_name(name): prefix = name while len(prefix): name = next_available_name( prefix, Domain.get_names_by_prefix(prefix + '-')) if Domain.get_by_name(name): # should never happen raise NameUnavailableException if len(name) <= max_length: return name prefix = prefix[:-1] raise NameUnavailableException return name @classmethod def get_all(cls, include_docs=True): domains = Domain.view("domain/not_snapshots", include_docs=False).all() if not include_docs: return domains else: return imap(cls.wrap, iter_docs(cls.get_db(), [d['id'] for d in domains])) @classmethod def get_all_names(cls): return [d['key'] for d in cls.get_all(include_docs=False)] @classmethod def get_all_ids(cls): return [d['id'] for d in cls.get_all(include_docs=False)] @classmethod def get_names_by_prefix(cls, prefix): return [ d['key'] for d in Domain.view("domain/domains", startkey=prefix, endkey=prefix + u"zzz", reduce=False, include_docs=False).all() ] def case_sharing_included(self): return self.case_sharing or reduce(lambda x, y: x or y, [ getattr(app, 'case_sharing', False) for app in self.applications() ], False) def save(self, **params): self.last_modified = datetime.utcnow() if not self._rev: # mark any new domain as timezone migration complete set_migration_complete(self.name) super(Domain, self).save(**params) from corehq.apps.domain.signals import commcare_domain_post_save results = commcare_domain_post_save.send_robust(sender='domain', domain=self) for result in results: # Second argument is None if there was no error if result[1]: notify_exception( None, message="Error occured during domain post_save %s: %s" % (self.name, str(result[1]))) def save_copy(self, new_domain_name=None, new_hr_name=None, user=None, copy_by_id=None, share_reminders=True, share_user_roles=True): from corehq.apps.app_manager.dbaccessors import get_app from corehq.apps.reminders.models import CaseReminderHandler from corehq.apps.fixtures.models import FixtureDataItem from corehq.apps.app_manager.dbaccessors import get_brief_apps_in_domain from corehq.apps.domain.dbaccessors import get_doc_ids_in_domain_by_class from corehq.apps.fixtures.models import FixtureDataType from corehq.apps.users.models import UserRole db = Domain.get_db() new_id = db.copy_doc(self.get_id)['id'] if new_domain_name is None: new_domain_name = new_id with CriticalSection( ['request_domain_name_{}'.format(new_domain_name)]): new_domain_name = Domain.generate_name(new_domain_name) new_domain = Domain.get(new_id) new_domain.name = new_domain_name new_domain.hr_name = new_hr_name new_domain.copy_history = self.get_updated_history() new_domain.is_snapshot = False new_domain.snapshot_time = None new_domain.organization = None # TODO: use current user's organization (?) # reset stuff new_domain.cda.signed = False new_domain.cda.date = None new_domain.cda.type = None new_domain.cda.user_id = None new_domain.cda.user_ip = None new_domain.is_test = "none" new_domain.internal = InternalProperties() new_domain.creating_user = user.username if user else None for field in self._dirty_fields: if hasattr(new_domain, field): delattr(new_domain, field) # Saving the domain should happen before we import any apps since # importing apps can update the domain object (for example, if user # as a case needs to be enabled) new_domain.save() new_app_components = {} # a mapping of component's id to its copy def copy_data_items(old_type_id, new_type_id): for item in FixtureDataItem.by_data_type( self.name, old_type_id): comp = self.copy_component(item.doc_type, item._id, new_domain_name, user=user) comp.data_type_id = new_type_id comp.save() def get_latest_app_id(doc_id): app = get_app(self.name, doc_id).get_latest_saved() if app: return app._id, app.doc_type for app in get_brief_apps_in_domain(self.name): doc_id, doc_type = app.get_id, app.doc_type original_doc_id = doc_id if copy_by_id and doc_id not in copy_by_id: continue if not self.is_snapshot: doc_id, doc_type = get_latest_app_id(doc_id) or (doc_id, doc_type) component = self.copy_component(doc_type, doc_id, new_domain_name, user=user) if component: new_app_components[original_doc_id] = component for doc_id in get_doc_ids_in_domain_by_class( self.name, FixtureDataType): if copy_by_id and doc_id not in copy_by_id: continue component = self.copy_component('FixtureDataType', doc_id, new_domain_name, user=user) copy_data_items(doc_id, component._id) if share_reminders: for doc_id in get_doc_ids_in_domain_by_class( self.name, CaseReminderHandler): self.copy_component('CaseReminderHandler', doc_id, new_domain_name, user=user) if share_user_roles: for doc_id in get_doc_ids_in_domain_by_class( self.name, UserRole): self.copy_component('UserRole', doc_id, new_domain_name, user=user) if user: def add_dom_to_user(user): user.add_domain_membership(new_domain_name, is_admin=True) apply_update(user, add_dom_to_user) def update_events(handler): """ Change the form_unique_id to the proper form for each event in a newly copied CaseReminderHandler """ from corehq.apps.app_manager.models import FormBase for event in handler.events: if not event.form_unique_id: continue form = FormBase.get_form(event.form_unique_id) form_app = form.get_app() m_index, f_index = form_app.get_form_location(form.unique_id) form_copy = new_app_components[form_app._id].get_module( m_index).get_form(f_index) event.form_unique_id = form_copy.unique_id def update_for_copy(handler): handler.active = False update_events(handler) if share_reminders: for handler in CaseReminderHandler.get_handlers(new_domain_name): apply_update(handler, update_for_copy) return new_domain def reminder_should_be_copied(self, handler): from corehq.apps.reminders.models import ON_DATETIME return (handler.start_condition_type != ON_DATETIME and handler.user_group_id is None) def copy_component(self, doc_type, id, new_domain_name, user=None): from corehq.apps.app_manager.models import import_app from corehq.apps.users.models import UserRole from corehq.apps.reminders.models import CaseReminderHandler from corehq.apps.fixtures.models import FixtureDataType, FixtureDataItem str_to_cls = { 'UserRole': UserRole, 'CaseReminderHandler': CaseReminderHandler, 'FixtureDataType': FixtureDataType, 'FixtureDataItem': FixtureDataItem, } if doc_type in ('Application', 'RemoteApp'): new_doc = import_app(id, new_domain_name) new_doc.copy_history.append(id) new_doc.case_sharing = False # when copying from app-docs that don't have # unique_id attribute on Modules new_doc.ensure_module_unique_ids(should_save=False) else: cls = str_to_cls[doc_type] db = cls.get_db() if doc_type == 'CaseReminderHandler': cur_doc = cls.get(id) if not self.reminder_should_be_copied(cur_doc): return None new_id = db.copy_doc(id)['id'] new_doc = cls.get(new_id) for field in self._dirty_fields: if hasattr(new_doc, field): delattr(new_doc, field) if hasattr(cls, '_meta_fields'): for field in cls._meta_fields: if not field.startswith('_') and hasattr(new_doc, field): delattr(new_doc, field) new_doc.domain = new_domain_name if doc_type == 'FixtureDataType': new_doc.copy_from = id new_doc.is_global = True if self.is_snapshot and doc_type == 'Application': new_doc.prepare_multimedia_for_exchange() new_doc.save() return new_doc def save_snapshot(self, share_reminders, copy_by_id=None): if self.is_snapshot: return self else: try: copy = self.save_copy(copy_by_id=copy_by_id, share_reminders=share_reminders, share_user_roles=False) except NameUnavailableException: return None copy.is_snapshot = True head = self.snapshots(limit=1).first() if head and head.snapshot_head: head.snapshot_head = False head.save() copy.snapshot_head = True copy.snapshot_time = datetime.utcnow() del copy.deployment copy.save() return copy def snapshots(self, **view_kwargs): return Domain.view('domain/snapshots', startkey=[self._id, {}], endkey=[self._id], include_docs=True, reduce=False, descending=True, **view_kwargs) @memoized def published_snapshot(self): snapshots = self.snapshots().all() for snapshot in snapshots: if snapshot.published: return snapshot return None def update_deployment(self, **kwargs): self.deployment.update(kwargs) self.save() def update_internal(self, **kwargs): self.internal.update(kwargs) self.save() def display_name(self): if self.is_snapshot: return "Snapshot of %s" % self.copied_from.display_name() return self.hr_name or self.name def long_display_name(self): if self.is_snapshot: return format_html("Snapshot of {}", self.copied_from.display_name()) return self.hr_name or self.name __str__ = long_display_name def get_license_display(self): return LICENSES.get(self.license) def get_license_url(self): return LICENSE_LINKS.get(self.license) def copies(self): return Domain.view('domain/copied_from_snapshot', key=self._id, include_docs=True) def copies_of_parent(self): return Domain.view('domain/copied_from_snapshot', keys=[s._id for s in self.copied_from.snapshots()], include_docs=True) def delete(self): self._pre_delete() super(Domain, self).delete() def _pre_delete(self): from corehq.apps.domain.signals import commcare_domain_pre_delete from corehq.apps.domain.deletion import apply_deletion_operations dynamic_deletion_operations = [] results = commcare_domain_pre_delete.send_robust(sender='domain', domain=self) for result in results: response = result[1] if isinstance(response, Exception): raise DomainDeleteException( u"Error occurred during domain pre_delete {}: {}".format( self.name, str(response))) elif response: assert isinstance(response, list) dynamic_deletion_operations.extend(response) # delete all associated objects for db, related_doc_ids in get_all_doc_ids_for_domain_grouped_by_db( self.name): iter_bulk_delete(db, related_doc_ids, chunksize=500) apply_deletion_operations(self.name, dynamic_deletion_operations) def all_media(self, from_apps=None): # todo add documentation or refactor from corehq.apps.hqmedia.models import CommCareMultimedia dom_with_media = self if not self.is_snapshot else self.copied_from if self.is_snapshot: app_ids = [ app.copied_from.get_id for app in self.full_applications() ] if from_apps: from_apps = set( [a_id for a_id in app_ids if a_id in from_apps]) else: from_apps = app_ids if from_apps: media = [] media_ids = set() apps = [ app for app in dom_with_media.full_applications() if app.get_id in from_apps ] for app in apps: if app.doc_type != 'Application': continue for _, m in app.get_media_objects(): if m.get_id not in media_ids: media.append(m) media_ids.add(m.get_id) return media return CommCareMultimedia.view('hqmedia/by_domain', key=dom_with_media.name, include_docs=True).all() def most_restrictive_licenses(self, apps_to_check=None): from corehq.apps.hqmedia.utils import most_restrictive licenses = [ m.license['type'] for m in self.all_media(from_apps=apps_to_check) if m.license ] return most_restrictive(licenses) @classmethod def get_module_by_name(cls, domain_name): """ import and return the python module corresponding to domain_name, or None if it doesn't exist. """ from corehq.apps.domain.utils import get_domain_module_map module_name = get_domain_module_map().get(domain_name, domain_name) try: return import_module(module_name) if module_name else None except ImportError: return None @property @memoized def commtrack_settings(self): # this import causes some dependency issues so lives in here from corehq.apps.commtrack.models import CommtrackConfig if self.commtrack_enabled: return CommtrackConfig.for_domain(self.name) else: return None @property def has_custom_logo(self): return (self['_attachments'] and LOGO_ATTACHMENT in self['_attachments']) def get_custom_logo(self): if not self.has_custom_logo: return None return (self.fetch_attachment(LOGO_ATTACHMENT), self['_attachments'][LOGO_ATTACHMENT]['content_type']) def get_case_display(self, case): """Get the properties display definition for a given case""" return self.case_display.case_details.get(case.type) def get_form_display(self, form): """Get the properties display definition for a given XFormInstance""" return self.case_display.form_details.get(form.xmlns) @property def total_downloads(self): """ Returns the total number of downloads from every snapshot created from this domain """ from corehq.apps.domain.dbaccessors import count_downloads_for_all_snapshots return count_downloads_for_all_snapshots(self.get_id) @property @memoized def download_count(self): """ Updates and returns the total number of downloads from every sister snapshot. """ if self.is_snapshot: self.full_downloads = self.copied_from.total_downloads return self.full_downloads @property @memoized def published_by(self): from corehq.apps.users.models import CouchUser pb_id = self.cda.user_id return CouchUser.get_by_user_id(pb_id) if pb_id else None @property def name_of_publisher(self): return self.published_by.human_friendly_name if self.published_by else "" @property def location_types(self): from corehq.apps.locations.models import LocationType return LocationType.objects.filter(domain=self.name).all() @memoized def has_privilege(self, privilege): from corehq.apps.accounting.utils import domain_has_privilege return domain_has_privilege(self, privilege) @property @memoized def uses_locations(self): from corehq import privileges from corehq.apps.locations.models import LocationType return (self.has_privilege(privileges.LOCATIONS) and (self.commtrack_enabled or LocationType.objects.filter(domain=self.name).exists())) @property def supports_multiple_locations_per_user(self): """ This method is a wrapper around the toggle that enables multiple location functionality. Callers of this method should know that this is special functionality left around for special applications, and not a feature flag that should be set normally. """ return toggles.MULTIPLE_LOCATIONS_PER_USER.enabled(self.name) def convert_to_commtrack(self): """ One-stop-shop to make a domain CommTrack """ from corehq.apps.commtrack.util import make_domain_commtrack make_domain_commtrack(self) def clear_caches(self): from .utils import domain_restricts_superusers super(Domain, self).clear_caches() self.get_by_name.clear(self.__class__, self.name) self.is_secure_session_required.clear(self.name) domain_restricts_superusers.clear(self.name)
class FixtureDataType(Document): domain = StringProperty() is_global = BooleanProperty(default=False) tag = StringProperty() fields = SchemaListProperty(FixtureTypeField) item_attributes = StringListProperty() description = StringProperty() copy_from = StringProperty() @classmethod def wrap(cls, obj): if not obj["doc_type"] == "FixtureDataType": raise ResourceNotFound # Migrate fixtures without attributes on item-fields to fields with attributes if obj["fields"] and isinstance(obj['fields'][0], basestring): obj['fields'] = [{ 'field_name': f, 'properties': [] } for f in obj['fields']] # Migrate fixtures without attributes on items to items with attributes if 'item_attributes' not in obj: obj['item_attributes'] = [] return super(FixtureDataType, cls).wrap(obj) # support for old fields @property def fields_without_attributes(self): fields_without_attributes = [] for fixt_field in self.fields: fields_without_attributes.append(fixt_field.field_name) return fields_without_attributes @classmethod def total_by_domain(cls, domain): from corehq.apps.fixtures.dbaccessors import \ get_number_of_fixture_data_types_in_domain return get_number_of_fixture_data_types_in_domain(domain) @classmethod def by_domain(cls, domain): from corehq.apps.fixtures.dbaccessors import \ get_fixture_data_types_in_domain return get_fixture_data_types_in_domain(domain) @classmethod def by_domain_tag(cls, domain, tag): return cls.view('fixtures/data_types_by_domain_tag', key=[domain, tag], reduce=False, include_docs=True, descending=True) @classmethod def fixture_tag_exists(cls, domain, tag): fdts = FixtureDataType.by_domain(domain) for fdt in fdts: if tag == fdt.tag: return fdt return False def recursive_delete(self, transaction): item_ids = [] for item in FixtureDataItem.by_data_type(self.domain, self.get_id): transaction.delete(item) item_ids.append(item.get_id) transaction.delete_all( FixtureOwnership.for_all_item_ids(item_ids, self.domain)) transaction.delete(self) @classmethod def delete_fixtures_by_domain(cls, domain, transaction): for type in FixtureDataType.by_domain(domain): type.recursive_delete(transaction)
class DataSourceConfiguration(CachedCouchDocumentMixin, Document, AbstractUCRDataSource): """ A data source configuration. These map 1:1 with database tables that get created. Each data source can back an arbitrary number of reports. """ domain = StringProperty(required=True) engine_id = StringProperty(default=UCR_ENGINE_ID) backend_id = StringProperty(default=UCR_SQL_BACKEND) # no longer used referenced_doc_type = StringProperty(required=True) table_id = StringProperty(required=True) display_name = StringProperty() base_item_expression = DictProperty() configured_filter = DictProperty() configured_indicators = ListProperty() named_expressions = DictProperty() named_filters = DictProperty() meta = SchemaProperty(DataSourceMeta) is_deactivated = BooleanProperty(default=False) last_modified = DateTimeProperty() asynchronous = BooleanProperty(default=False) sql_column_indexes = SchemaListProperty(SQLColumnIndexes) disable_destructive_rebuild = BooleanProperty(default=False) sql_settings = SchemaProperty(SQLSettings) class Meta(object): # prevent JsonObject from auto-converting dates etc. string_conversions = () def __str__(self): return '{} - {}'.format(self.domain, self.display_name) def save(self, **params): self.last_modified = datetime.utcnow() super(DataSourceConfiguration, self).save(**params) @property def data_source_id(self): return self._id def filter(self, document): filter_fn = self._get_main_filter() return filter_fn(document, EvaluationContext(document, 0)) def deleted_filter(self, document): filter_fn = self._get_deleted_filter() return filter_fn and filter_fn(document, EvaluationContext( document, 0)) @memoized def _get_main_filter(self): return self._get_filter([self.referenced_doc_type]) @memoized def _get_deleted_filter(self): return self._get_filter(get_deleted_doc_types( self.referenced_doc_type), include_configured=False) def _get_filter(self, doc_types, include_configured=True): if not doc_types: return None extras = ([self.configured_filter] if include_configured and self.configured_filter else []) built_in_filters = [ self._get_domain_filter_spec(), { 'type': 'or', 'filters': [{ "type": "boolean_expression", "expression": { "type": "property_name", "property_name": "doc_type", }, "operator": "eq", "property_value": doc_type, } for doc_type in doc_types], }, ] return FilterFactory.from_spec( { 'type': 'and', 'filters': built_in_filters + extras, }, context=self.get_factory_context(), ) def _get_domain_filter_spec(self): return { "type": "boolean_expression", "expression": { "type": "property_name", "property_name": "domain", }, "operator": "eq", "property_value": self.domain, } @property @memoized def named_expression_objects(self): named_expression_specs = deepcopy(self.named_expressions) named_expressions = {} spec_error = None while named_expression_specs: number_generated = 0 for name, expression in list(named_expression_specs.items()): try: named_expressions[name] = ExpressionFactory.from_spec( expression, FactoryContext(named_expressions=named_expressions, named_filters={})) number_generated += 1 del named_expression_specs[name] except BadSpecError as bad_spec_error: # maybe a nested name resolution issue, try again on the next pass spec_error = bad_spec_error if number_generated == 0 and named_expression_specs: # we unsuccessfully generated anything on this pass and there are still unresolved # references. we have to fail. assert spec_error is not None raise spec_error return named_expressions @property @memoized def named_filter_objects(self): return { name: FilterFactory.from_spec( filter, FactoryContext(self.named_expression_objects, {})) for name, filter in self.named_filters.items() } def get_factory_context(self): return FactoryContext(self.named_expression_objects, self.named_filter_objects) @property @memoized def default_indicators(self): default_indicators = [ IndicatorFactory.from_spec( { "column_id": "doc_id", "type": "expression", "display_name": "document id", "datatype": "string", "is_nullable": False, "is_primary_key": True, "expression": { "type": "root_doc", "expression": { "type": "property_name", "property_name": "_id" } } }, self.get_factory_context()) ] default_indicators.append( IndicatorFactory.from_spec({ "type": "inserted_at", }, self.get_factory_context())) if self.base_item_expression: default_indicators.append( IndicatorFactory.from_spec({ "type": "repeat_iteration", }, self.get_factory_context())) return default_indicators @property @memoized def indicators(self): return CompoundIndicator( self.display_name, self.default_indicators + [ IndicatorFactory.from_spec(indicator, self.get_factory_context()) for indicator in self.configured_indicators ], None, ) @property @memoized def parsed_expression(self): if self.base_item_expression: return ExpressionFactory.from_spec( self.base_item_expression, context=self.get_factory_context()) return None def get_columns(self): return self.indicators.get_columns() @property @memoized def columns_by_id(self): return {c.id: c for c in self.get_columns()} def get_column_by_id(self, column_id): return self.columns_by_id.get(column_id) def get_items(self, document, eval_context=None): if self.filter(document): if not self.base_item_expression: return [document] else: result = self.parsed_expression(document, eval_context) if result is None: return [] elif isinstance(result, list): return result else: return [result] else: return [] def get_all_values(self, doc, eval_context=None): if not eval_context: eval_context = EvaluationContext(doc) rows = [] for item in self.get_items(doc, eval_context): indicators = self.indicators.get_values(item, eval_context) rows.append(indicators) eval_context.increment_iteration() return rows def get_report_count(self): """ Return the number of ReportConfigurations that reference this data source. """ return ReportConfiguration.count_by_data_source(self.domain, self._id) def validate(self, required=True): super(DataSourceConfiguration, self).validate(required) # these two properties implicitly call other validation self._get_main_filter() self._get_deleted_filter() # validate indicators and column uniqueness columns = [c.id for c in self.indicators.get_columns()] unique_columns = set(columns) if len(columns) != len(unique_columns): for column in set(columns): columns.remove(column) raise DuplicateColumnIdError(columns=columns) if self.referenced_doc_type not in VALID_REFERENCED_DOC_TYPES: raise BadSpecError( _('Report contains invalid referenced_doc_type: {}').format( self.referenced_doc_type)) self.parsed_expression @classmethod def by_domain(cls, domain): return get_datasources_for_domain(domain) @classmethod def all_ids(cls): return [ res['id'] for res in cls.get_db().view( 'userreports/data_sources_by_build_info', reduce=False, include_docs=False) ] @classmethod def all(cls): for result in iter_docs(cls.get_db(), cls.all_ids()): yield cls.wrap(result) @property def is_static(self): return id_is_static(self._id) def deactivate(self): if not self.is_static: self.is_deactivated = True self.save() get_indicator_adapter(self).drop_table() def get_case_type_or_xmlns_filter(self): """Returns a list of case types or xmlns from the filter of this data source. If this can't figure out the case types or xmlns's that filter, then returns [None] Currently always returns a list because it is called by a loop in _iteratively_build_table Could be reworked to return [] to be more pythonic """ if self.referenced_doc_type not in FILTER_INTERPOLATION_DOC_TYPES: return [None] property_name = FILTER_INTERPOLATION_DOC_TYPES[ self.referenced_doc_type] prop_value = self._filter_interploation_helper(self.configured_filter, property_name) return prop_value or [None] def _filter_interploation_helper(self, config_filter, property_name): filter_type = config_filter.get('type') if filter_type == 'and': sub_config_filters = [ self._filter_interploation_helper(f, property_name) for f in config_filter.get('filters') ] for filter_ in sub_config_filters: if filter_[0]: return filter_ if filter_type != 'boolean_expression': return [None] if config_filter['operator'] not in ('eq', 'in'): return [None] expression = config_filter['expression'] if expression['type'] == 'property_name' and expression[ 'property_name'] == property_name: prop_value = config_filter['property_value'] if not isinstance(prop_value, list): prop_value = [prop_value] return prop_value return [None]
class Domain(QuickCachedDocumentMixin, BlobMixin, Document, SnapshotMixin): """ Domain is the highest level collection of people/stuff in the system. Pretty much everything happens at the domain-level, including user membership, permission to see data, reports, charts, etc. Exceptions: accounting has some models that combine multiple domains, which make "enterprise" multi-domain features like the enterprise dashboard possible. Naming conventions: Most often, variables representing domain names are named `domain`, and variables representing domain objects are named `domain_obj`. New code should follow this convention, unless it's in an area that consistently uses `domain` for the object and `domain_name` for the string. There's a `project` attribute attached to requests that's a domain object. In spite of this, don't use `project` in new code. """ _blobdb_type_code = BLOB_CODES.domain name = StringProperty() is_active = BooleanProperty() date_created = DateTimeProperty() default_timezone = StringProperty( default=getattr(settings, "TIME_ZONE", "UTC")) case_sharing = BooleanProperty(default=False) secure_submissions = BooleanProperty(default=False) cloudcare_releases = StringProperty( choices=['stars', 'nostars', 'default'], default='default') organization = StringProperty() hr_name = StringProperty() # the human-readable name for this project project_description = StringProperty() # Brief description of the project creating_user = StringProperty( ) # username of the user who created this domain # domain metadata project_type = StringProperty() # e.g. MCH, HIV customer_type = StringProperty() # plus, full, etc. is_test = StringProperty(choices=["true", "false", "none"], default="none") description = StringProperty() short_description = StringProperty() is_shared = BooleanProperty(default=False) commtrack_enabled = BooleanProperty(default=False) call_center_config = SchemaProperty(CallCenterProperties) restrict_superusers = BooleanProperty(default=False) allow_domain_requests = BooleanProperty(default=False) location_restriction_for_users = BooleanProperty(default=False) usercase_enabled = BooleanProperty(default=False) hipaa_compliant = BooleanProperty(default=False) use_sql_backend = BooleanProperty(default=False) first_domain_for_user = BooleanProperty(default=False) case_display = SchemaProperty(CaseDisplaySettings) # CommConnect settings survey_management_enabled = BooleanProperty(default=False) # Whether or not a case can register via sms sms_case_registration_enabled = BooleanProperty(default=False) # Case type to apply to cases registered via sms sms_case_registration_type = StringProperty() # Owner to apply to cases registered via sms sms_case_registration_owner_id = StringProperty() # Submitting user to apply to cases registered via sms sms_case_registration_user_id = StringProperty() # Whether or not a mobile worker can register via sms sms_mobile_worker_registration_enabled = BooleanProperty(default=False) use_default_sms_response = BooleanProperty(default=False) default_sms_response = StringProperty() chat_message_count_threshold = IntegerProperty() sms_language_fallback = StringProperty() custom_chat_template = StringProperty( ) # See settings.CUSTOM_CHAT_TEMPLATES custom_case_username = StringProperty( ) # Case property to use when showing the case's name in a chat window # If empty, sms can be sent at any time. Otherwise, only send during # these windows of time. SMS_QUEUE_ENABLED must be True in localsettings # for this be considered. restricted_sms_times = SchemaListProperty(DayTimeWindow) # If empty, this is ignored. Otherwise, the framework will make sure # that during these days/times, no automated outbound sms will be sent # to someone if they have sent in an sms within sms_conversation_length # minutes. Outbound sms sent from a user in a chat window, however, will # still be sent. This is meant to prevent chat conversations from being # interrupted by automated sms reminders. # SMS_QUEUE_ENABLED must be True in localsettings for this to be # considered. sms_conversation_times = SchemaListProperty(DayTimeWindow) # In minutes, see above. sms_conversation_length = IntegerProperty(default=10) # Set to True to prevent survey questions and answers form being seen in # SMS chat windows. filter_surveys_from_chat = BooleanProperty(default=False) # The below option only matters if filter_surveys_from_chat = True. # If set to True, invalid survey responses will still be shown in the chat # window, while questions and valid responses will be filtered out. show_invalid_survey_responses_in_chat = BooleanProperty(default=False) # If set to True, if a message is read by anyone it counts as being read by # everyone. Set to False so that a message is only counted as being read # for a user if only that user has read it. count_messages_as_read_by_anyone = BooleanProperty(default=False) enable_registration_welcome_sms_for_case = BooleanProperty(default=False) enable_registration_welcome_sms_for_mobile_worker = BooleanProperty( default=False) sms_survey_date_format = StringProperty() granted_messaging_access = BooleanProperty(default=False) # Allowed outbound SMS per day # If this is None, then the default is applied. See get_daily_outbound_sms_limit() custom_daily_outbound_sms_limit = IntegerProperty() # Allowed number of case updates or closes from automatic update rules in the daily rule run. # If this value is None, the value in settings.MAX_RULE_UPDATES_IN_ONE_RUN is used. auto_case_update_limit = IntegerProperty() # Allowed number of max OData feeds that this domain can create. # If this value is None, the value in settings.DEFAULT_ODATA_FEED_LIMIT is used odata_feed_limit = IntegerProperty() # exchange/domain copying stuff is_snapshot = BooleanProperty(default=False) is_approved = BooleanProperty(default=False) snapshot_time = DateTimeProperty() published = BooleanProperty(default=False) license = StringProperty(choices=LICENSES, default='cc') title = StringProperty() cda = SchemaProperty(LicenseAgreement) multimedia_included = BooleanProperty(default=True) downloads = IntegerProperty( default=0) # number of downloads for this specific snapshot full_downloads = IntegerProperty( default=0) # number of downloads for all snapshots from this domain author = StringProperty() phone_model = StringProperty() attribution_notes = StringProperty() publisher = StringProperty(choices=["organization", "user"], default="user") yt_id = StringProperty() snapshot_head = BooleanProperty(default=False) deployment = SchemaProperty(Deployment) cached_properties = DictProperty() internal = SchemaProperty(InternalProperties) dynamic_reports = SchemaListProperty(DynamicReportSet) # extra user specified properties tags = StringListProperty() area = StringProperty(choices=AREA_CHOICES) sub_area = StringProperty(choices=SUB_AREA_CHOICES) launch_date = DateTimeProperty last_modified = DateTimeProperty(default=datetime(2015, 1, 1)) # when turned on, use SECURE_TIMEOUT for sessions of users who are members of this domain secure_sessions = BooleanProperty(default=False) two_factor_auth = BooleanProperty(default=False) strong_mobile_passwords = BooleanProperty(default=False) requested_report_builder_subscription = StringListProperty() report_whitelist = StringListProperty() # seconds between sending mobile UCRs to users. Can be overridden per user default_mobile_ucr_sync_interval = IntegerProperty() @classmethod def wrap(cls, data): # for domains that still use original_doc should_save = False if 'original_doc' in data: original_doc = data['original_doc'] del data['original_doc'] should_save = True if original_doc: original_doc = Domain.get_by_name(original_doc) data['copy_history'] = [original_doc._id] # for domains that have a public domain license if 'license' in data: if data.get("license", None) == "public": data["license"] = "cc" should_save = True if 'slug' in data and data["slug"]: data["hr_name"] = data["slug"] del data["slug"] if 'is_test' in data and isinstance(data["is_test"], bool): data["is_test"] = "true" if data["is_test"] else "false" should_save = True if 'cloudcare_releases' not in data: data['cloudcare_releases'] = 'nostars' # legacy default setting # Don't actually remove location_types yet. We can migrate fully and # remove this after everything's hunky-dory in production. 2015-03-06 if 'location_types' in data: data['obsolete_location_types'] = data.pop('location_types') if 'granted_messaging_access' not in data: # enable messaging for domains created before this flag was added data['granted_messaging_access'] = True self = super(Domain, cls).wrap(data) if self.deployment is None: self.deployment = Deployment() if should_save: self.save() return self def get_default_timezone(self): """return a timezone object from self.default_timezone""" import pytz return pytz.timezone(self.default_timezone) @staticmethod @quickcache(['name'], timeout=24 * 60 * 60) def is_secure_session_required(name): domain_obj = Domain.get_by_name(name) return domain_obj and domain_obj.secure_sessions @staticmethod @quickcache(['couch_user._id', 'is_active'], timeout=5 * 60, memoize_timeout=10) def active_for_couch_user(couch_user, is_active=True): domain_names = couch_user.get_domains() return Domain.view( "domain/by_status", keys=[[is_active, d] for d in domain_names], reduce=False, include_docs=True, ).all() @staticmethod def active_for_user(user, is_active=True): if isinstance(user, AnonymousUser): return [] from corehq.apps.users.models import CouchUser if isinstance(user, CouchUser): couch_user = user else: couch_user = CouchUser.from_django_user(user) if couch_user: return Domain.active_for_couch_user(couch_user, is_active=is_active) else: return [] def add(self, model_instance, is_active=True): """ Add something to this domain, through the generic relation. Returns the created membership object """ # Add membership info to Couch couch_user = model_instance.get_profile().get_couch_user() couch_user.add_domain_membership(self.name) couch_user.save() def applications(self): return get_brief_apps_in_domain(self.name) def full_applications(self, include_builds=True): from corehq.apps.app_manager.util import get_correct_app_class from corehq.apps.app_manager.models import Application def wrap_application(a): return get_correct_app_class(a['doc']).wrap(a['doc']) if include_builds: startkey = [self.name] endkey = [self.name, {}] else: startkey = [self.name, None] endkey = [self.name, None, {}] return Application.get_db().view('app_manager/applications', startkey=startkey, endkey=endkey, include_docs=True, wrapper=wrap_application).all() @cached_property def versions(self): apps = self.applications() return list(set(a.application_version for a in apps)) @cached_property def has_media(self): from corehq.apps.app_manager.util import is_remote_app for app in self.full_applications(): if not is_remote_app(app) and app.has_media(): return True return False @property def use_cloudcare_releases(self): return self.cloudcare_releases != 'nostars' def all_users(self): from corehq.apps.users.models import CouchUser return CouchUser.by_domain(self.name) def recent_submissions(self): return domain_has_submission_in_last_30_days(self.name) @classmethod @quickcache(['name'], skip_arg='strict', timeout=30 * 60, session_function=icds_conditional_session_key()) def get_by_name(cls, name, strict=False): if not name: # get_by_name should never be called with name as None (or '', etc) # I fixed the code in such a way that if I raise a ValueError # all tests pass and basic pages load, # but in order not to break anything in the wild, # I'm opting to notify by email if/when this happens # but fall back to the previous behavior of returning None if settings.DEBUG: raise ValueError('%r is not a valid domain name' % name) else: _assert = soft_assert(notify_admins=True, exponential_backoff=False) _assert(False, '%r is not a valid domain name' % name) return None def _get_by_name(stale=False): extra_args = {'stale': settings.COUCH_STALE_QUERY} if stale else {} result = cls.view("domain/domains", key=name, reduce=False, include_docs=True, **extra_args).first() if not isinstance(result, Domain): # A stale view may return a result with no doc if the doc has just been deleted. # In this case couchdbkit just returns the raw view result as a dict return None else: return result domain = _get_by_name(stale=(not strict)) if domain is None and not strict: # on the off chance this is a brand new domain, try with strict domain = _get_by_name(stale=False) return domain @classmethod def get_or_create_with_name(cls, name, is_active=False, secure_submissions=True, use_sql_backend=False): result = cls.view("domain/domains", key=name, reduce=False, include_docs=True).first() if result: return result else: new_domain = Domain( name=name, is_active=is_active, date_created=datetime.utcnow(), secure_submissions=secure_submissions, use_sql_backend=use_sql_backend, ) new_domain.save(**get_safe_write_kwargs()) return new_domain @classmethod def generate_name(cls, hr_name, max_length=25): ''' Generate a URL-friendly name based on a given human-readable name. Normalizes given name, then looks for conflicting domains, addressing conflicts by adding "-1", "-2", etc. May return None if it fails to generate a new, unique name. Throws exception if it can't figure out a name, which shouldn't happen unless max_length is absurdly short. ''' from corehq.apps.domain.utils import get_domain_url_slug from corehq.apps.domain.dbaccessors import domain_or_deleted_domain_exists name = get_domain_url_slug(hr_name, max_length=max_length) if not name: raise NameUnavailableException if domain_or_deleted_domain_exists(name): prefix = name while len(prefix): name = next_available_name( prefix, Domain.get_names_by_prefix(prefix + '-')) if domain_or_deleted_domain_exists(name): # should never happen raise NameUnavailableException if len(name) <= max_length: return name prefix = prefix[:-1] raise NameUnavailableException return name @classmethod def get_all(cls, include_docs=True): domains = Domain.view("domain/not_snapshots", include_docs=False).all() if not include_docs: return domains else: return map(cls.wrap, iter_docs(cls.get_db(), [d['id'] for d in domains])) @classmethod def get_all_names(cls): return sorted({d['key'] for d in cls.get_all(include_docs=False)}) @classmethod def get_all_ids(cls): return [d['id'] for d in cls.get_all(include_docs=False)] @classmethod def get_names_by_prefix(cls, prefix): return [ d['key'] for d in Domain.view("domain/domains", startkey=prefix, endkey=prefix + "zzz", reduce=False, include_docs=False).all() ] + [ d['key'] for d in Domain.view("domain/deleted_domains", startkey=prefix, endkey=prefix + "zzz", reduce=False, include_docs=False).all() ] def case_sharing_included(self): return self.case_sharing or reduce(lambda x, y: x or y, [ getattr(app, 'case_sharing', False) for app in self.applications() ], False) def save(self, **params): from corehq.apps.domain.dbaccessors import domain_or_deleted_domain_exists self.last_modified = datetime.utcnow() if not self._rev: if domain_or_deleted_domain_exists(self.name): raise NameUnavailableException(self.name) # mark any new domain as timezone migration complete set_tz_migration_complete(self.name) super(Domain, self).save(**params) from corehq.apps.domain.signals import commcare_domain_post_save results = commcare_domain_post_save.send_robust(sender='domain', domain=self) log_signal_errors(results, "Error occurred during domain post_save (%s)", {'domain': self.name}) def snapshots(self, **view_kwargs): return Domain.view('domain/snapshots', startkey=[self._id, {}], endkey=[self._id], include_docs=True, reduce=False, descending=True, **view_kwargs) def update_deployment(self, **kwargs): self.deployment.update(kwargs) self.save() def update_internal(self, **kwargs): self.internal.update(kwargs) self.save() def display_name(self): if self.is_snapshot: return "Snapshot of %s" % self.copied_from.display_name() return self.hr_name or self.name def long_display_name(self): if self.is_snapshot: return format_html("Snapshot of {}", self.copied_from.display_name()) return self.hr_name or self.name __str__ = long_display_name def get_license_display(self): return LICENSES.get(self.license) def get_license_url(self): return LICENSE_LINKS.get(self.license) def copies(self): return Domain.view('domain/copied_from_snapshot', key=self._id, include_docs=True) def copies_of_parent(self): return Domain.view('domain/copied_from_snapshot', keys=[s._id for s in self.copied_from.snapshots()], include_docs=True) def delete(self, leave_tombstone=False): if not leave_tombstone and not settings.UNIT_TESTING: raise ValueError( 'Cannot delete domain without leaving a tombstone except during testing' ) self._pre_delete() if leave_tombstone: domain = self.get(self._id) if not domain.doc_type.endswith('-Deleted'): domain.doc_type = '{}-Deleted'.format(domain.doc_type) domain.save() else: super().delete() # The save signals can undo effect of clearing the cache within the save # because they query the stale view (but attaches the up to date doc). # This is only a problem on delete/soft-delete, # because these change the presence in the index, not just the doc content. # Since this is rare, I'm opting to just re-clear the cache here # rather than making the signals use a strict lookup or something like that. self.clear_caches() def _pre_delete(self): from corehq.apps.domain.deletion import apply_deletion_operations # delete SQL models first because UCR tables are indexed by configs in couch apply_deletion_operations(self.name) # delete couch docs for db, related_doc_ids in get_all_doc_ids_for_domain_grouped_by_db( self.name): iter_bulk_delete(db, related_doc_ids, chunksize=500) @classmethod def get_module_by_name(cls, domain_name): """ import and return the python module corresponding to domain_name, or None if it doesn't exist. """ module_name = settings.DOMAIN_MODULE_MAP.get(domain_name, domain_name) try: return import_module(module_name) if module_name else None except ImportError: return None @property @memoized def commtrack_settings(self): # this import causes some dependency issues so lives in here from corehq.apps.commtrack.models import CommtrackConfig if self.commtrack_enabled: return CommtrackConfig.for_domain(self.name) else: return None @property def has_custom_logo(self): return self.has_attachment(LOGO_ATTACHMENT) def get_custom_logo(self): if not self.has_custom_logo: return None return (self.fetch_attachment(LOGO_ATTACHMENT), self.blobs[LOGO_ATTACHMENT].content_type) def put_attachment(self, *args, **kw): return super(Domain, self).put_attachment(domain=self.name, *args, **kw) def get_case_display(self, case): """Get the properties display definition for a given case""" return self.case_display.case_details.get(case.type) def get_form_display(self, form): """Get the properties display definition for a given XFormInstance""" return self.case_display.form_details.get(form.xmlns) @property def location_types(self): from corehq.apps.locations.models import LocationType return LocationType.objects.filter(domain=self.name).all() @memoized def has_privilege(self, privilege): from corehq.apps.accounting.utils import domain_has_privilege return domain_has_privilege(self, privilege) @property @memoized def uses_locations(self): from corehq import privileges from corehq.apps.locations.models import LocationType return (self.has_privilege(privileges.LOCATIONS) and (self.commtrack_enabled or LocationType.objects.filter(domain=self.name).exists())) def convert_to_commtrack(self): """ One-stop-shop to make a domain CommTrack """ from corehq.apps.commtrack.util import make_domain_commtrack make_domain_commtrack(self) def clear_caches(self): from .utils import domain_restricts_superusers super(Domain, self).clear_caches() self.get_by_name.clear(self.__class__, self.name) self.is_secure_session_required.clear(self.name) domain_restricts_superusers.clear(self.name) def get_daily_outbound_sms_limit(self): if self.custom_daily_outbound_sms_limit: return self.custom_daily_outbound_sms_limit # https://manage.dimagi.com/default.asp?274299 return 50000
class XFormInstance(DeferredBlobMixin, SafeSaveDocument, ComputedDocumentMixin, CouchDocLockableMixIn, AbstractXFormInstance): """An XForms instance.""" domain = StringProperty() app_id = StringProperty() xmlns = StringProperty() form = DictProperty() received_on = DateTimeProperty() server_modified_on = DateTimeProperty() # Used to tag forms that were forcefully submitted # without a touchforms session completing normally partial_submission = BooleanProperty(default=False) history = SchemaListProperty(XFormOperation) auth_context = DictProperty() submit_ip = StringProperty() path = StringProperty() openrosa_headers = DictProperty() last_sync_token = StringProperty() # almost always a datetime, but if it's not parseable it'll be a string date_header = DefaultProperty() build_id = StringProperty() export_tag = DefaultProperty(name='#export_tag') _blobdb_type_code = CODES.form_xml class Meta(object): app_label = 'couchforms' @classmethod def get(cls, docid, rev=None, db=None, dynamic_properties=True): # copied and tweaked from the superclass's method if not db: db = cls.get_db() cls._allow_dynamic_properties = dynamic_properties # on cloudant don't get the doc back until all nodes agree # on the copy, to avoid race conditions extras = get_safe_read_kwargs() try: if cls == XFormInstance: doc = db.get(docid, rev=rev, **extras) if doc['doc_type'] in doc_types(): return doc_types()[doc['doc_type']].wrap(doc) try: return XFormInstance.wrap(doc) except WrappingAttributeError: raise ResourceNotFound( "The doc with _id {} and doc_type {} can't be wrapped " "as an XFormInstance".format(docid, doc['doc_type'])) return db.get(docid, rev=rev, wrapper=cls.wrap, **extras) except ResourceNotFound: raise XFormNotFound(docid) @property def form_id(self): return self._id @form_id.setter def form_id(self, value): self._id = value @property def form_data(self): return DictProperty().unwrap(self.form)[1] @property def user_id(self): return getattr(self.metadata, 'userID', None) @property def is_error(self): return self.doc_type != 'XFormInstance' @property def is_duplicate(self): return self.doc_type == 'XFormDuplicate' @property def is_archived(self): return self.doc_type == 'XFormArchived' @property def is_deprecated(self): return self.doc_type == 'XFormDeprecated' @property def is_submission_error_log(self): return self.doc_type == 'SubmissionErrorLog' @property def is_deleted(self): return self.doc_type.endswith(DELETED_SUFFIX) @property def is_normal(self): return self.doc_type == 'XFormInstance' @property def deletion_id(self): return getattr(self, '-deletion_id', None) @property def deletion_date(self): return getattr(self, '-deletion_date', None) @property def metadata(self): if const.TAG_META in self.form: return Metadata.wrap( clean_metadata(self.to_json()[const.TAG_FORM][const.TAG_META])) return None @property def time_start(self): # Will be addressed in https://github.com/dimagi/commcare-hq/pull/19391/ return None @property def time_end(self): return None @property def commcare_version(self): return str(self.metadata.commcare_version) @property def app_version(self): return None def __str__(self): return "%s (%s)" % (self.type, self.xmlns) def save(self, **kwargs): # HACK: cloudant has a race condition when saving newly created forms # which throws errors here. use a try/retry loop here to get around # it until we find something more stable. RETRIES = 10 SLEEP = 0.5 # seconds tries = 0 self.server_modified_on = datetime.datetime.utcnow() while True: try: return super(XFormInstance, self).save(**kwargs) except PreconditionFailed: if tries == 0: logging.error('doc %s got a precondition failed', self._id) if tries < RETRIES: tries += 1 time.sleep(SLEEP) else: raise def get_data(self, path): """ Evaluates an xpath expression like: path/to/node and returns the value of that element, or None if there is no value. """ return safe_index(self, path.split("/")) def soft_delete(self): NotAllowed.check(self.domain) self.doc_type += DELETED_SUFFIX self.save() def get_xml(self): try: return self.fetch_attachment(ATTACHMENT_NAME) except ResourceNotFound: logging.warn("no xml found for %s, trying old attachment scheme." % self.get_id) try: return self[const.TAG_XML] except AttributeError: raise MissingFormXml(self.form_id) def get_attachment(self, attachment_name): return self.fetch_attachment(attachment_name) def get_xml_element(self): xml_string = self.get_xml() if not xml_string: return None return self._xml_string_to_element(xml_string) def _xml_string_to_element(self, xml_string): def _to_xml_element(payload): if isinstance(payload, str): payload = payload.encode('utf-8', errors='replace') return etree.fromstring(payload) try: return _to_xml_element(xml_string) except XMLSyntaxError: # there is a bug at least in pact code that double # saves a submission in a way that the attachments get saved in a base64-encoded format decoded_payload = base64.b64decode(xml_string) element = _to_xml_element(decoded_payload) # in this scenario resave the attachment properly in case future calls circumvent this method self.save() self.put_attachment(decoded_payload, ATTACHMENT_NAME) return element def put_attachment(self, content, name, **kw): if kw.get("type_code") is None: kw["type_code"] = (CODES.form_xml if name == ATTACHMENT_NAME else CODES.form_attachment) return super(XFormInstance, self).put_attachment(content, name, **kw) @property def attachments(self): """ Get the extra attachments for this form. This will not include the form itself """ def _meta_to_json(meta): is_image = False if meta.content_type is not None: is_image = True if meta.content_type.startswith( 'image/') else False meta_json = meta.to_json() meta_json['is_image'] = is_image return meta_json return { name: _meta_to_json(meta) for name, meta in self.blobs.items() if name != ATTACHMENT_NAME } def xml_md5(self): return hashlib.md5(self.get_xml()).hexdigest() def archive(self, user_id=None, trigger_signals=True): if not self.is_archived: FormAccessors.do_archive(self, True, user_id, trigger_signals) def unarchive(self, user_id=None, trigger_signals=True): if self.is_archived: FormAccessors.do_archive(self, False, user_id, trigger_signals)
class ExportTable(DocumentSchema): """ A table configuration, for export """ index = StringProperty() display = StringProperty() columns = SchemaListProperty(ExportColumn) order = ListProperty() @classmethod def wrap(cls, data): # hack: manually remove any references to _attachments at runtime data['columns'] = [ c for c in data['columns'] if not c['index'].startswith("_attachments.") ] return super(ExportTable, cls).wrap(data) @classmethod def default(cls, index): return cls(index=index, display="", columns=[]) @property @memoized def displays_by_index(self): return dict((c.index, c.get_display()) for c in self.columns) def get_column_configuration(self, all_cols): selected_cols = set() for c in self.columns: if c.doc_type in display_column_types: selected_cols.add(c.index) yield c.to_config_format() for c in all_cols: if c not in selected_cols: column = ExportColumn(index=c) column.display = self.displays_by_index[ c] if self.displays_by_index.has_key(c) else '' yield column.to_config_format(selected=False) def get_headers_row(self): from couchexport.export import FormattedRow headers = [] for col in self.columns: if issubclass(type(col), ComplexExportColumn): for header in col.get_headers(): headers.append(header) else: display = col.get_display() if col.index == 'id': id_len = len( filter(lambda part: part == '#', self.index.split('.'))) headers.append(display) if id_len > 1: for i in range(id_len): headers.append('{id}__{i}'.format(id=display, i=i)) else: headers.append(display) return FormattedRow(headers) @property @memoized def row_positions_by_index(self): return dict((h, i) for i, h in enumerate(self._headers) if self.displays_by_index.has_key(h)) @property @memoized def id_index(self): for i, column in enumerate(self.columns): if column.index == 'id': return i def get_items_in_order(self, row): row_data = list(row.get_data()) for column in self.columns: try: i = self.row_positions_by_index[column.index] val = row_data[i] except KeyError: val = '' if issubclass(type(column), ComplexExportColumn): for value in column.get_data(val): yield column, value else: yield column, val def trim(self, data, doc, apply_transforms, global_transform): from couchexport.export import FormattedRow, Constant, transform_error_constant if not hasattr(self, '_headers'): self._headers = tuple(data[0].get_data()) # skip first element without copying data = islice(data, 1, None) for row in data: id = None cells = [] for column, val in self.get_items_in_order(row): # TRANSFORM BABY! if apply_transforms: if column.transform and not isinstance(val, Constant): try: val = column.transform(val, doc) except Exception: val = transform_error_constant elif global_transform: val = global_transform(val, doc) if column.index == 'id': id = val else: cells.append(val) id_index = self.id_index if id else 0 row_id = row.id if id else None yield FormattedRow(cells, row_id, id_index=id_index)
class DynamicReportSet(DocumentSchema): """a set of dynamic reports grouped under a section header in the sidebar""" section_title = StringProperty() reports = SchemaListProperty(DynamicReportConfig)
class CustomDataFieldsDefinition(Document): """ Per-project user-defined fields such as custom user data. """ field_type = StringProperty() base_doc = "CustomDataFieldsDefinition" domain = StringProperty() fields = SchemaListProperty(CustomDataField) def get_fields(self, required_only=False, include_system=True): def _is_match(field): return not ((required_only and not field.is_required) or (not include_system and is_system_key(field.slug))) return filter(_is_match, self.fields) @classmethod def get_or_create(cls, domain, field_type): existing = get_by_domain_and_type(domain, field_type) if existing: return existing else: new = cls(domain=domain, field_type=field_type) new.save() return new def get_validator(self, data_field_class): """ Returns a validator to be used in bulk import """ def validate_choices(field, value): if field.choices and value and six.text_type( value) not in field.choices: return _( "'{value}' is not a valid choice for {slug}, the available " "options are: {options}.").format( value=value, slug=field.slug, options=', '.join(field.choices), ) def validate_regex(field, value): if field.regex and value and not re.search(field.regex, value): return _("'{value}' is not a valid match for {slug}").format( value=value, slug=field.slug) def validate_required(field, value): if field.is_required and not value: return _("Cannot create or update a {entity} without " "the required field: {field}.").format( entity=data_field_class.entity_string, field=field.slug) def validate_custom_fields(custom_fields): errors = [] for field in self.fields: value = custom_fields.get(field.slug, None) errors.append(validate_required(field, value)) errors.append(validate_choices(field, value)) errors.append(validate_regex(field, value)) return ' '.join(filter(None, errors)) return validate_custom_fields def get_model_and_uncategorized(self, data_dict): """ Splits data_dict into two dictionaries: one for data which matches the model and one for data that doesn't Does not include reserved fields. """ if not data_dict: return {}, {} model_data = {} uncategorized_data = {} slugs = [field.slug for field in self.fields] for k, v in data_dict.items(): if k in slugs: model_data[k] = v elif is_system_key(k): pass else: uncategorized_data[k] = v return model_data, uncategorized_data
class OpenmrsCaseConfig(DocumentSchema): id_matchers = SchemaListProperty(IdMatcher) person_properties = SchemaDictProperty(ValueSource) person_attributes = SchemaDictProperty(ValueSource)
class ExportTable(DocumentSchema): """ A table configuration, for export """ index = StringProperty() display = StringProperty() columns = SchemaListProperty(ExportColumn) @classmethod def wrap(cls, data): # hack: manually remove any references to _attachments at runtime data['columns'] = [ c for c in data['columns'] if not c['index'].startswith("_attachments.") ] return super(ExportTable, cls).wrap(data) @classmethod def default(cls, index): return cls(index=index, display="", columns=[]) @property @memoized def displays_by_index(self): return dict((c.index, c.get_display()) for c in self.columns) def get_column_configuration(self, all_cols): selected_cols = set() for c in self.columns: if c.doc_type in display_column_types: selected_cols.add(c.index) yield c.to_config_format() for c in all_cols: if c not in selected_cols: column = ExportColumn(index=c) column.display = self.displays_by_index[ c] if c in self.displays_by_index else '' yield column.to_config_format(selected=False) def get_headers_row(self): from couchexport.export import FormattedRow headers = [] for col in self.columns: if issubclass(type(col), ComplexExportColumn): for header in col.get_headers(): headers.append(header) else: display = col.get_display() if col.index == 'id': id_len = len([ part for part in self.index.split('.') if part == '#' ]) headers.append(display) if id_len > 1: for i in range(id_len): headers.append('{id}__{i}'.format(id=display, i=i)) else: headers.append(display) return FormattedRow(headers) @property @memoized def row_positions_by_index(self): return dict((h, i) for i, h in enumerate(self._headers) if h in self.displays_by_index) @property @memoized def id_index(self): for i, column in enumerate(self.columns): if column.index == 'id': return i def get_items_in_order(self, row): from couchexport.export import scalar_never_was row_data = list(row.get_data()) for column in self.columns: # If, for example, column.index references a question in a form # export and there are no forms that have a value for that question, # then that question does not show up in the schema for the export # and so column.index won't be found in self.row_positions_by_index. # In those cases we want to give a value of '---' to be consistent # with other "not applicable" export values. try: i = self.row_positions_by_index[column.index] val = row_data[i] except KeyError: val = scalar_never_was if issubclass(type(column), ComplexExportColumn): for value in column.get_data(val): yield column, value else: yield column, val def trim(self, data, doc, apply_transforms, global_transform): from couchexport.export import FormattedRow, Constant, transform_error_constant if not hasattr(self, '_headers'): self._headers = tuple(data[0].get_data()) # skip first element without copying data = islice(data, 1, None) rows = [] for row in data: id = None cells = [] for column, val in self.get_items_in_order(row): # TRANSFORM BABY! if apply_transforms: if column.transform and not isinstance(val, Constant): try: val = column.transform(val, doc) except Exception: val = transform_error_constant elif global_transform: val = global_transform(val, doc) if column.index == 'id': id = val else: cells.append(val) id_index = self.id_index if id else 0 row_id = row.id if id else None rows.append(FormattedRow(cells, row_id, id_index=id_index)) return rows
class SQLSettings(DocumentSchema): partition_config = SchemaListProperty(SQLPartition)
class SavedExportSchema(BaseSavedExportSchema, UnicodeMixIn): """ Lets you save an export format with a schema and list of columns and display names. """ name = StringProperty() default_format = StringProperty() is_safe = BooleanProperty(default=False) # Is the export de-identified? # self.index should always match self.schema.index # needs to be here so we can use in couch views index = JsonProperty() # id of an ExportSchema for checkpointed schemas schema_id = StringProperty() # user-defined table configuration tables = SchemaListProperty(ExportTable) # For us right now, 'form' or 'case' type = StringProperty() # ID of the new style export that it was converted to converted_saved_export_id = StringProperty() def __unicode__(self): return "%s (%s)" % (self.name, self.index) def transform(self, doc): return doc @property def global_transform_function(self): # will be called on every value in the doc during export return identity @property @memoized def schema(self): return ExportSchema.get(self.schema_id) @property def table_name(self): return self.sheet_name if self.sheet_name else "%s" % self._id @classmethod def default(cls, schema, name="", type='form'): return cls(name=name, index=schema.index, schema_id=schema.get_id, tables=[ExportTable.default(schema.tables[0][0])], type=type) @property @memoized def tables_by_index(self): return dict([t.index, t] for t in self.tables) def get_table_configuration(self, index): def column_configuration(): columns = self.schema.get_columns(index) if index in self.tables_by_index: return list( self.tables_by_index[index].get_column_configuration( columns)) else: return [ ExportColumn(index=c, display='').to_config_format(selected=False) for c in columns ] def display(): if index in self.tables_by_index: return self.tables_by_index[index].display else: return '' return { "index": index, "display": display(), "column_configuration": column_configuration(), "selected": index in self.tables_by_index } def get_table_headers(self, override_name=False): return ((self.table_name if override_name and i == 0 else t.index, [t.get_headers_row()]) for i, t in enumerate(self.tables)) @property def table_configuration(self): return [ self.get_table_configuration(index) for index, cols in self.schema.tables ] def update_schema(self): """ Update the schema for this object to include the latest columns from any relevant docs. Does NOT save the doc, just updates the in-memory object. """ from couchexport.schema import build_latest_schema schema = build_latest_schema(self.index) if schema: self.set_schema(schema) def set_schema(self, schema): """ Set the schema for this object. Does NOT save the doc, just updates the in-memory object. """ self.schema_id = schema.get_id def trim(self, document_table, doc, apply_transforms=True): tables = [] for table_index, data in document_table: if table_index in self.tables_by_index: # todo: currently (index, rows) instead of (display, rows); where best to convert to display? tables.append( (table_index, self.tables_by_index[table_index].trim( data, doc, apply_transforms, self.global_transform_function))) return tables def get_export_components(self, previous_export_id=None, filter=None): from couchexport.export import ExportConfiguration database = get_db() config = ExportConfiguration(database, self.index, previous_export_id, self.filter & filter) # get and checkpoint the latest schema updated_schema = config.get_latest_schema() export_schema_checkpoint = config.create_new_checkpoint() return config, updated_schema, export_schema_checkpoint def get_export_files(self, format=None, previous_export=None, filter=None, process=None, max_column_size=None, apply_transforms=True, limit=0, **kwargs): from couchexport.export import get_writer, get_formatted_rows if not format: format = self.default_format or Format.XLS_2007 config, updated_schema, export_schema_checkpoint = self.get_export_components( previous_export, filter) # transform docs onto output and save writer = get_writer(format) # open the doc and the headers formatted_headers = list(self.get_table_headers()) fd, path = tempfile.mkstemp() with os.fdopen(fd, 'wb') as tmp: writer.open(formatted_headers, tmp, max_column_size=max_column_size, table_titles=dict([(table.index, table.display) for table in self.tables if table.display])) total_docs = len(config.potentially_relevant_ids) if process: DownloadBase.set_progress(process, 0, total_docs) for i, doc in config.enum_docs(): if limit and i > limit: break if self.transform and apply_transforms: doc = self.transform(doc) formatted_tables = self.trim(get_formatted_rows(doc, updated_schema, separator="."), doc, apply_transforms=apply_transforms) writer.write(formatted_tables) if process: DownloadBase.set_progress(process, i + 1, total_docs) writer.close() if format == Format.PYTHON_DICT: return writer.get_preview() return ExportFiles(path, export_schema_checkpoint, format) def get_preview_data(self, export_filter, limit=50): return self.get_export_files(Format.PYTHON_DICT, None, export_filter, limit=limit) def download_data(self, format="", previous_export=None, filter=None, limit=0): """ If there is data, return an HTTPResponse with the appropriate data. If there is not data returns None. """ from couchexport.shortcuts import export_response files = self.get_export_files(format, previous_export, filter, limit=limit) return export_response(files.file, files.format, self.name) def to_export_config(self): """ Return an ExportConfiguration object that represents this. """ # confusingly, the index isn't the actual index property, # but is the index appended with the id to this document. # this is to avoid conflicts among multiple exports index = "%s-%s" % (self.index, self._id) if isinstance(self.index, six.string_types) else \ self.index + [self._id] # self.index required to be a string or list return ExportConfiguration(index=index, name=self.name, format=self.default_format) def custom_validate(self): if self.default_format == Format.XLS: for table in self.tables: if len(table.columns) > 255: raise CustomExportValidationError( "XLS files can only have 255 columns") # replaces `sheet_name = StringProperty()` def __get_sheet_name(self): return self.tables[0].display def __set_sheet_name(self, value): self.tables[0].display = value sheet_name = property(__get_sheet_name, __set_sheet_name) @classmethod def wrap(cls, data): # since this is a property now, trying to wrap it will fail hard if 'sheet_name' in data: del data['sheet_name'] return super(SavedExportSchema, cls).wrap(data)
class AbstractSyncLog(SafeSaveDocument): date = DateTimeProperty() domain = StringProperty() user_id = StringProperty() build_id = StringProperty() # only works with app-aware sync app_id = StringProperty() # only works with app-aware sync previous_log_id = StringProperty() # previous sync log, forming a chain duration = IntegerProperty() # in seconds log_format = StringProperty() # owner_ids_on_phone stores the ids the phone thinks it's the owner of. # This typically includes the user id, # as well as all groups that that user is a member of. owner_ids_on_phone = StringListProperty() # for debugging / logging previous_log_rev = StringProperty( ) # rev of the previous log at the time of creation last_submitted = DateTimeProperty( ) # last time a submission caused this to be modified rev_before_last_submitted = StringProperty( ) # rev when the last submission was saved last_cached = DateTimeProperty( ) # last time this generated a cached response hash_at_last_cached = StringProperty( ) # the state hash of this when it was last cached # save state errors and hashes here had_state_error = BooleanProperty(default=False) error_date = DateTimeProperty() error_hash = StringProperty() cache_payload_paths = DictProperty() last_ucr_sync_times = SchemaListProperty(UCRSyncLog) strict = True # for asserts @classmethod def wrap(cls, data): ret = super(AbstractSyncLog, cls).wrap(data) if hasattr(ret, 'has_assert_errors'): ret.strict = False return ret def save(self): self._synclog_sql = save_synclog_to_sql(self) def delete(self): if getattr(self, '_synclog_sql', None): self._synclog_sql.delete() def case_count(self): """ How many cases are associated with this. Used in reports. """ raise NotImplementedError() def phone_is_holding_case(self, case_id): raise NotImplementedError() def get_footprint_of_cases_on_phone(self): """ Gets the phone's flat list of all case ids on the phone, owned or not owned but relevant. """ raise NotImplementedError() def get_state_hash(self): return CaseStateHash( Checksum(self.get_footprint_of_cases_on_phone()).hexdigest()) def update_phone_lists(self, xform, case_list): """ Given a form an list of touched cases, update this sync log to reflect the updated state on the phone. """ raise NotImplementedError() @classmethod def from_other_format(cls, other_sync_log): """ Convert to an instance of a subclass from another subclass. Subclasses can override this to provide conversion functions. """ raise IncompatibleSyncLogType('Unable to convert from {} to {}'.format( type(other_sync_log), cls, )) # anything prefixed with 'tests_only' is only used in tests def tests_only_get_cases_on_phone(self): raise NotImplementedError() def test_only_clear_cases_on_phone(self): raise NotImplementedError() def test_only_get_dependent_cases_on_phone(self): raise NotImplementedError()
class GroupExportConfiguration(Document): """ An export configuration allows you to setup a collection of exports that all run together. Used by the management command or a scheduled job to run a bunch of exports on a schedule. """ full_exports = SchemaListProperty(ExportConfiguration) custom_export_ids = StringListProperty() def get_custom_exports(self): for custom in list(self.custom_export_ids): custom_export = self._get_custom(custom) if custom_export: yield custom_export def _get_custom(self, custom_id): """ Get a custom export, or delete it's reference if not found """ try: return SavedExportSchema.get(custom_id) except ResourceNotFound: try: self.custom_export_ids.remove(custom_id) self.save() except ValueError: pass @property @memoized def saved_exports(self): return self._saved_exports_from_configs(self.all_configs) def _saved_exports_from_configs(self, configs): exports = SavedBasicExport.view( "couchexport/saved_exports", keys=[json.dumps(config.index) for config in configs], include_docs=True, reduce=False, ).all() export_map = dict((json.dumps(export.configuration.index), export) for export in exports) return [ GroupExportComponent( config, export_map.get(json.dumps(config.index), None), self._id, list(self.all_configs).index(config)) for config in configs ] @property @memoized def all_configs(self): """ Return an iterator of config-like objects that include the main configs + the custom export configs. """ return [full for full in self.full_exports] + \ [custom.to_export_config() for custom in self.get_custom_exports()] @property def all_export_schemas(self): """ Return an iterator of ExportSchema-like objects that include the main configs + the custom export configs. """ for full in self.full_exports: yield DefaultExportSchema(index=full.index, type=full.type) for custom in self.get_custom_exports(): yield custom @property @memoized def all_exports(self): """ Returns an iterator of tuples consisting of the export config and an ExportSchema-like document that can be used to get at the data. """ return list(zip(self.all_configs, self.all_export_schemas))