class InternalProperties(DocumentSchema, UpdatableSchema): """ Project properties that should only be visible/editable by superusers """ sf_contract_id = StringProperty() sf_account_id = StringProperty() commcare_edition = StringProperty(choices=[ '', "plus", "community", "standard", "pro", "advanced", "enterprise" ], default="community") services = StringProperty(choices=["", "basic", "plus", "full", "custom"], default="") initiative = StringListProperty() workshop_region = StringProperty() project_state = StringProperty( choices=["", "POC", "transition", "at-scale"], default="") self_started = BooleanProperty() area = StringProperty() sub_area = StringProperty() using_adm = BooleanProperty() using_call_center = BooleanProperty() custom_eula = BooleanProperty() can_use_data = BooleanProperty() notes = StringProperty() organization_name = StringProperty() platform = StringListProperty() project_manager = StringProperty() phone_model = StringProperty() goal_time_period = IntegerProperty() goal_followup_rate = DecimalProperty()
class FixtureDataType(Document): domain = StringProperty() tag = StringProperty() name = StringProperty() fields = StringListProperty() @classmethod def by_domain(cls, domain): return cls.view('fixtures/data_types_by_domain', key=domain, reduce=False, include_docs=True) @classmethod def by_domain_tag(cls, domain, tag): return cls.view('fixtures/data_types_by_domain_tag', key=[domain, tag], reduce=False, include_docs=True) def recursive_delete(self, transaction): item_ids = [] for item in FixtureDataItem.by_data_type(self.domain, self.get_id): transaction.delete(item) item_ids.append(item.get_id) transaction.delete_all( FixtureOwnership.for_all_item_ids(item_ids, self.domain)) transaction.delete(self)
class Deployment(DocumentSchema, UpdatableSchema): date = DateTimeProperty() city = StringProperty() countries = StringListProperty() region = StringProperty( ) # e.g. US, LAC, SA, Sub-saharn Africa, East Africa, West Africa, Southeast Asia) description = StringProperty() public = BooleanProperty(default=False)
class InternalProperties(DocumentSchema, UpdatableSchema): """ Project properties that should only be visible/editable by superusers """ sf_contract_id = StringProperty() sf_account_id = StringProperty() commcare_edition = StringProperty(choices=["", "standard", "plus", "advanced"], default="") services = StringProperty(choices=["", "basic", "plus", "full", "custom"], default="") initiative = StringListProperty() project_state = StringProperty(choices=["", "POC", "transition", "at-scale"], default="") self_started = BooleanProperty() area = StringProperty() sub_area = StringProperty() using_adm = BooleanProperty() using_call_center = BooleanProperty() custom_eula = BooleanProperty() can_use_data = BooleanProperty() notes = StringProperty() organization_name = StringProperty() platform = StringListProperty()
class HQAnnouncement(Document, AdminCRUDDocumentMixin): """ For global, site-wide HQ Announcements. """ title = StringProperty() summary = StringProperty() highlighted_selectors = StringListProperty() date_created = DateTimeProperty() valid_until = DateTimeProperty() show_to_new_users = BooleanProperty(default=False) base_doc = "HQAnnouncement" _admin_crud_class = HQAnnouncementCRUDManager @property def as_html(self): return render_to_string("announcements/partials/base_announcement.html", { 'title': self.title, 'content': self.summary, 'announcement_id': self._id, })
class ReportNotification(CachedCouchDocumentMixin, Document): domain = StringProperty() owner_id = StringProperty() recipient_emails = StringListProperty() config_ids = StringListProperty() send_to_owner = BooleanProperty() attach_excel = BooleanProperty() # language is only used if some of the config_ids refer to UCRs or custom reports language = StringProperty() email_subject = StringProperty(default=DEFAULT_REPORT_NOTIF_SUBJECT) hour = IntegerProperty(default=8) minute = IntegerProperty(default=0) day = IntegerProperty(default=1) interval = StringProperty(choices=["daily", "weekly", "monthly"]) uuid = StringProperty() start_date = DateProperty(default=None) @property def is_editable(self): try: self.report_slug return False except AttributeError: return True @classmethod def by_domain_and_owner(cls, domain, owner_id, stale=True, **kwargs): if stale: kwargs['stale'] = settings.COUCH_STALE_QUERY key = [domain, owner_id] db = cls.get_db() result = cache_core.cached_view(db, "reportconfig/user_notifications", reduce=False, include_docs=True, startkey=key, endkey=key + [{}], wrapper=cls.wrap, **kwargs) return result @property @memoized def all_recipient_emails(self): # handle old documents if not self.owner_id: return frozenset([self.owner.get_email()]) emails = frozenset(self.recipient_emails) if self.send_to_owner and self.owner_email: emails |= {self.owner_email} return emails @property @memoized def owner_email(self): if self.owner.is_web_user(): return self.owner.username email = self.owner.get_email() try: validate_email(email) return email except Exception: pass @property @memoized def owner(self): id = self.owner_id return CouchUser.get_by_user_id(id) @property @memoized def configs(self): """ Access the notification's associated configs as a list, transparently returning an appropriate dummy for old notifications which have `report_slug` instead of `config_ids`. """ if self.config_ids: configs = [] for config_doc in iter_docs(ReportConfig.get_db(), self.config_ids): config = ReportConfig.wrap(config_doc) if not hasattr(config, 'deleted'): configs.append(config) def _sort_key(config_id): if config_id in self.config_ids: return self.config_ids.index(config_id) else: return len(self.config_ids) configs = sorted(configs, key=_sort_key) elif self.report_slug == 'admin_domains': raise UnsupportedScheduledReportError( "admin_domains is no longer " "supported as a schedulable report for the time being") else: # create a new ReportConfig object, useful for its methods and # calculated properties, but don't save it class ReadonlyReportConfig(ReportConfig): def save(self, *args, **kwargs): pass config = ReadonlyReportConfig() object.__setattr__(config, '_id', 'dummy') config.report_type = ProjectReportDispatcher.prefix config.report_slug = self.report_slug config.domain = self.domain config.owner_id = self.owner_id configs = [config] return tuple(configs) @property def day_name(self): if self.interval == 'weekly': return calendar.day_name[self.day] return { "daily": _("Every day"), "monthly": _("Day %s of every month" % self.day), }[self.interval] @classmethod def day_choices(cls): """Tuples for day of week number and human-readable day of week""" return tuple([(val, calendar.day_name[val]) for val in range(7)]) @classmethod def hour_choices(cls): """Tuples for hour number and human-readable hour""" return tuple([(val, "%s:00" % val) for val in range(24)]) @property @memoized def recipients_by_language(self): user_languages = { user['username']: user['language'] for user in get_user_docs_by_username(self.all_recipient_emails) if 'username' in user and 'language' in user } if self.language: fallback_language = self.language else: fallback_language = user_languages.get(self.owner_email, 'en') recipients = defaultdict(list) for email in self.all_recipient_emails: language = user_languages.get(email, fallback_language) recipients[language].append(email) return immutabledict(recipients) def get_secret(self, email): uuid = self._get_or_create_uuid() return hashlib.sha1((uuid + email).encode('utf-8')).hexdigest()[:20] def send(self): # Scenario: user has been removed from the domain that they # have scheduled reports for. Delete this scheduled report if not self.owner.is_member_of(self.domain): self.delete() return if self.owner.is_deleted(): self.delete() return if self.recipients_by_language: for language, emails in self.recipients_by_language.items(): self._get_and_send_report(language, emails) def _get_or_create_uuid(self): if not self.uuid: self.uuid = uuid.uuid4().hex self.save() return self.uuid def _get_and_send_report(self, language, emails): from corehq.apps.reports.views import get_scheduled_report_response, render_full_report_notification with localize(language): title = (_(DEFAULT_REPORT_NOTIF_SUBJECT) if self.email_subject == DEFAULT_REPORT_NOTIF_SUBJECT else self.email_subject) attach_excel = getattr(self, 'attach_excel', False) try: content, excel_files = get_scheduled_report_response( self.owner, self.domain, self._id, attach_excel=attach_excel, send_only_active=True) # Will be False if ALL the ReportConfigs in the ReportNotification # have a start_date in the future. if content is False: return for email in emails: body = render_full_report_notification( None, content, email, self).content send_html_email_async( title, email, body, email_from=settings.DEFAULT_FROM_EMAIL, file_attachments=excel_files, smtp_exception_skip_list=LARGE_FILE_SIZE_ERROR_CODES) except Exception as er: notify_exception( None, message= "Encountered error while generating report or sending email", details={ 'subject': title, 'recipients': str(emails), 'error': er, }) if getattr(er, 'smtp_code', None) in LARGE_FILE_SIZE_ERROR_CODES or type( er) == ESError: # If the email doesn't work because it is too large to fit in the HTML body, # send it as an excel attachment, by creating a mock request with the right data. for report_config in self.configs: mock_request = HttpRequest() mock_request.couch_user = self.owner mock_request.user = self.owner.get_django_user() mock_request.domain = self.domain mock_request.couch_user.current_domain = self.domain mock_request.couch_user.language = self.language mock_request.method = 'GET' mock_request.bypass_two_factor = True mock_query_string_parts = [ report_config.query_string, 'filterSet=true' ] if report_config.is_configurable_report: mock_query_string_parts.append( urlencode(report_config.filters, True)) mock_query_string_parts.append( urlencode(report_config.get_date_range(), True)) mock_request.GET = QueryDict( '&'.join(mock_query_string_parts)) date_range = report_config.get_date_range() start_date = datetime.strptime(date_range['startdate'], '%Y-%m-%d') end_date = datetime.strptime(date_range['enddate'], '%Y-%m-%d') datespan = DateSpan(start_date, end_date) request_data = vars(mock_request) request_data[ 'couch_user'] = mock_request.couch_user.userID request_data['datespan'] = datespan full_request = { 'request': request_data, 'domain': request_data['domain'], 'context': {}, 'request_params': json_request(request_data['GET']) } export_all_rows_task(report_config.report, full_request, emails, title) def remove_recipient(self, email): try: if email == self.owner.get_email(): self.send_to_owner = False self.recipient_emails.remove(email) except ValueError: pass def update_attributes(self, items): for k, v in items: if k == 'start_date': self.verify_start_date(v) self.__setattr__(k, v) def verify_start_date(self, start_date): if start_date != self.start_date and start_date < datetime.today( ).date(): raise ValidationError( "You can not specify a start date in the past.")
class ReportNotification(CachedCouchDocumentMixin, Document): domain = StringProperty() owner_id = StringProperty() recipient_emails = StringListProperty() config_ids = StringListProperty() send_to_owner = BooleanProperty() attach_excel = BooleanProperty() # language is only used if some of the config_ids refer to UCRs. language = StringProperty() email_subject = StringProperty(default=DEFAULT_REPORT_NOTIF_SUBJECT) hour = IntegerProperty(default=8) minute = IntegerProperty(default=0) day = IntegerProperty(default=1) interval = StringProperty(choices=["daily", "weekly", "monthly"]) uuid = StringProperty() start_date = DateProperty(default=None) @property def is_editable(self): try: self.report_slug return False except AttributeError: return True @classmethod def by_domain_and_owner(cls, domain, owner_id, stale=True, **kwargs): if stale: kwargs['stale'] = settings.COUCH_STALE_QUERY key = [domain, owner_id] db = cls.get_db() result = cache_core.cached_view(db, "reportconfig/user_notifications", reduce=False, include_docs=True, startkey=key, endkey=key + [{}], wrapper=cls.wrap, **kwargs) return result @property @memoized def all_recipient_emails(self): # handle old documents if not self.owner_id: return frozenset([self.owner.get_email()]) emails = frozenset(self.recipient_emails) if self.send_to_owner and self.owner_email: emails |= {self.owner_email} return emails @property @memoized def owner_email(self): if self.owner.is_web_user(): return self.owner.username email = self.owner.get_email() try: validate_email(email) return email except Exception: pass @property @memoized def owner(self): id = self.owner_id return CouchUser.get_by_user_id(id) @property @memoized def configs(self): """ Access the notification's associated configs as a list, transparently returning an appropriate dummy for old notifications which have `report_slug` instead of `config_ids`. """ if self.config_ids: configs = [] for config_doc in iter_docs(ReportConfig.get_db(), self.config_ids): config = ReportConfig.wrap(config_doc) if not hasattr(config, 'deleted'): configs.append(config) def _sort_key(config_id): if config_id in self.config_ids: return self.config_ids.index(config_id) else: return len(self.config_ids) configs = sorted(configs, key=_sort_key) elif self.report_slug == 'admin_domains': raise UnsupportedScheduledReportError( "admin_domains is no longer " "supported as a schedulable report for the time being") else: # create a new ReportConfig object, useful for its methods and # calculated properties, but don't save it class ReadonlyReportConfig(ReportConfig): def save(self, *args, **kwargs): pass config = ReadonlyReportConfig() object.__setattr__(config, '_id', 'dummy') config.report_type = ProjectReportDispatcher.prefix config.report_slug = self.report_slug config.domain = self.domain config.owner_id = self.owner_id configs = [config] return tuple(configs) @property def day_name(self): if self.interval == 'weekly': return calendar.day_name[self.day] return { "daily": _("Every day"), "monthly": _("Day %s of every month" % self.day), }[self.interval] @classmethod def day_choices(cls): """Tuples for day of week number and human-readable day of week""" return tuple([(val, calendar.day_name[val]) for val in range(7)]) @classmethod def hour_choices(cls): """Tuples for hour number and human-readable hour""" return tuple([(val, "%s:00" % val) for val in range(24)]) @property @memoized def recipients_by_language(self): user_languages = { user['username']: user['language'] for user in get_user_docs_by_username(self.all_recipient_emails) if 'username' in user and 'language' in user } fallback_language = user_languages.get(self.owner_email, 'en') recipients = defaultdict(list) for email in self.all_recipient_emails: language = user_languages.get(email, fallback_language) recipients[language].append(email) return immutabledict(recipients) def get_secret(self, email): uuid = self._get_or_create_uuid() return hashlib.sha1((uuid + email).encode('utf-8')).hexdigest()[:20] def send(self): # Scenario: user has been removed from the domain that they # have scheduled reports for. Delete this scheduled report if not self.owner.is_member_of(self.domain): self.delete() return if self.owner.is_deleted(): self.delete() return if self.recipients_by_language: for language, emails in self.recipients_by_language.items(): self._get_and_send_report(language, emails) def _get_or_create_uuid(self): if not self.uuid: self.uuid = uuid.uuid4().hex self.save() return self.uuid def _get_and_send_report(self, language, emails): from corehq.apps.reports.views import get_scheduled_report_response, render_full_report_notification with localize(language): title = (_(DEFAULT_REPORT_NOTIF_SUBJECT) if self.email_subject == DEFAULT_REPORT_NOTIF_SUBJECT else self.email_subject) attach_excel = getattr(self, 'attach_excel', False) content, excel_files = get_scheduled_report_response( self.owner, self.domain, self._id, attach_excel=attach_excel, send_only_active=True) # Will be False if ALL the ReportConfigs in the ReportNotification # have a start_date in the future. if content is False: return for email in emails: body = render_full_report_notification(None, content, email, self).content send_html_email_async.delay( title, email, body, email_from=settings.DEFAULT_FROM_EMAIL, file_attachments=excel_files) def remove_recipient(self, email): try: if email == self.owner.get_email(): self.send_to_owner = False self.recipient_emails.remove(email) except ValueError: pass def update_attributes(self, items): for k, v in items: if k == 'start_date': self.verify_start_date(v) self.__setattr__(k, v) def verify_start_date(self, start_date): if start_date != self.start_date and start_date < datetime.today( ).date(): raise ValidationError( "You can not specify a start date in the past.")
class FixtureDataType(Document): domain = StringProperty() is_global = BooleanProperty(default=False) tag = StringProperty() fields = SchemaListProperty(FixtureTypeField) item_attributes = StringListProperty() @classmethod def wrap(cls, obj): if not obj["doc_type"] == "FixtureDataType": raise ResourceNotFound # Migrate fixtures without attributes on item-fields to fields with attributes if obj["fields"] and isinstance(obj['fields'][0], basestring): obj['fields'] = [{'field_name': f, 'properties': []} for f in obj['fields']] # Migrate fixtures without attributes on items to items with attributes if 'item_attributes' not in obj: obj['item_attributes'] = [] return super(FixtureDataType, cls).wrap(obj) # support for old fields @property def fields_without_attributes(self): fields_without_attributes = [] for fixt_field in self.fields: fields_without_attributes.append(fixt_field.field_name) return fields_without_attributes @classmethod def total_by_domain(cls, domain): num_fixtures = FixtureDataType.get_db().view( 'fixtures/data_types_by_domain', reduce=True, key=domain, ).first() return num_fixtures['value'] if num_fixtures is not None else 0 @classmethod def by_domain(cls, domain): return cls.view('fixtures/data_types_by_domain', key=domain, reduce=False, include_docs=True, descending=True) @classmethod def by_domain_tag(cls, domain, tag): return cls.view('fixtures/data_types_by_domain_tag', key=[domain, tag], reduce=False, include_docs=True, descending=True) @classmethod def fixture_tag_exists(cls, domain, tag): fdts = FixtureDataType.by_domain(domain) for fdt in fdts: if tag == fdt.tag: return fdt return False def recursive_delete(self, transaction): item_ids = [] for item in FixtureDataItem.by_data_type(self.domain, self.get_id): transaction.delete(item) item_ids.append(item.get_id) transaction.delete_all(FixtureOwnership.for_all_item_ids(item_ids, self.domain)) transaction.delete(self) @classmethod def delete_fixtures_by_domain(cls, domain, transaction): for type in FixtureDataType.by_domain(domain): type.recursive_delete(transaction)
class FixtureTypeField(DocumentSchema): field_name = StringProperty() properties = StringListProperty()
class OpmReportSnapshot(Document): """ Represents a snapshot of a report, to be taken at the end of each month """ domain = StringProperty() month = IntegerProperty() year = IntegerProperty() report_class = StringProperty() headers = StringListProperty() slugs = StringListProperty() rows = ListProperty() block = StringProperty() visible_cols = ListProperty() @classmethod def by_month(cls, month, year, report_class, block=None): return cls.view('opm_tasks/opm_snapshots', key=[DOMAIN, month, year, report_class, block], reduce=False, include_docs=True).first() @classmethod def filtered(cls, snapshot, report): filtered_rows = [] need_filering = False filters = [] for key, field in report.filter_fields: keys = report.filter_data.get(field, []) if keys: need_filering = True filters.append((key, field)) if need_filering: for row in snapshot.rows: def key_finder(key): index = snapshot.slugs.index(key) return row[index] try: report.filter(fn=key_finder, filter_fields=filters) except InvalidRow: pass else: filtered_rows.append(row) snapshot.rows = filtered_rows return snapshot @classmethod def from_view(cls, report): block = None if report.block: block = report.block.lower() snapshot = cls.view('opm_tasks/opm_snapshots', key=[ DOMAIN, report.month, report.year, report.__class__.__name__, block ], reduce=False, include_docs=True).first() if not snapshot: return None return cls.filtered(snapshot, report)
class Domain(Document, HQBillingDomainMixin, SnapshotMixin): """Domain is the highest level collection of people/stuff in the system. Pretty much everything happens at the domain-level, including user membership, permission to see data, reports, charts, etc.""" name = StringProperty() is_active = BooleanProperty() is_public = BooleanProperty(default=False) date_created = DateTimeProperty() default_timezone = StringProperty(default=getattr(settings, "TIME_ZONE", "UTC")) case_sharing = BooleanProperty(default=False) organization = StringProperty() hr_name = StringProperty() # the human-readable name for this project within an organization eula = SchemaProperty(LicenseAgreement) creating_user = StringProperty() # username of the user who created this domain # domain metadata project_type = StringProperty() # e.g. MCH, HIV customer_type = StringProperty() # plus, full, etc. is_test = BooleanProperty(default=True) description = StringProperty() short_description = StringProperty() is_shared = BooleanProperty(default=False) commtrack_enabled = BooleanProperty(default=False) call_center_config = SchemaProperty(CallCenterProperties) case_display = SchemaProperty(CaseDisplaySettings) # CommConnect settings survey_management_enabled = BooleanProperty(default=False) sms_case_registration_enabled = BooleanProperty(default=False) # Whether or not a case can register via sms sms_case_registration_type = StringProperty() # Case type to apply to cases registered via sms sms_case_registration_owner_id = StringProperty() # Owner to apply to cases registered via sms sms_case_registration_user_id = StringProperty() # Submitting user to apply to cases registered via sms sms_mobile_worker_registration_enabled = BooleanProperty(default=False) # Whether or not a mobile worker can register via sms default_sms_backend_id = StringProperty() # exchange/domain copying stuff is_snapshot = BooleanProperty(default=False) is_approved = BooleanProperty(default=False) snapshot_time = DateTimeProperty() published = BooleanProperty(default=False) license = StringProperty(choices=LICENSES, default='cc') title = StringProperty() cda = SchemaProperty(LicenseAgreement) multimedia_included = BooleanProperty(default=True) downloads = IntegerProperty(default=0) author = StringProperty() phone_model = StringProperty() attribution_notes = StringProperty() publisher = StringProperty(choices=["organization", "user"], default="user") yt_id = StringProperty() deployment = SchemaProperty(Deployment) image_path = StringProperty() image_type = StringProperty() migrations = SchemaProperty(DomainMigrations) cached_properties = DictProperty() internal = SchemaProperty(InternalProperties) # extra user specified properties tags = StringListProperty() area = StringProperty(choices=AREA_CHOICES) sub_area = StringProperty(choices=SUB_AREA_CHOICES) launch_date = DateTimeProperty # to be eliminated from projects and related documents when they are copied for the exchange _dirty_fields = ('admin_password', 'admin_password_charset', 'city', 'country', 'region', 'customer_type') @classmethod def wrap(cls, data): # for domains that still use original_doc should_save = False if 'original_doc' in data: original_doc = data['original_doc'] del data['original_doc'] should_save = True if original_doc: original_doc = Domain.get_by_name(original_doc) data['copy_history'] = [original_doc._id] # for domains that have a public domain license if 'license' in data: if data.get("license", None) == "public": data["license"] = "cc" should_save = True # if not 'creating_user' in data: # should_save = True # from corehq.apps.users.models import CouchUser # admins = CouchUser.view("users/admins_by_domain", key=data["name"], reduce=False, include_docs=True).all() # if len(admins) == 1: # data["creating_user"] = admins[0].username # else: # data["creating_user"] = None if 'slug' in data and data["slug"]: data["hr_name"] = data["slug"] del data["slug"] self = super(Domain, cls).wrap(data) if self.get_id: self.apply_migrations() if should_save: self.save() return self @staticmethod def active_for_user(user, is_active=True): if isinstance(user, AnonymousUser): return [] from corehq.apps.users.models import CouchUser if isinstance(user, CouchUser): couch_user = user else: couch_user = CouchUser.from_django_user(user) if couch_user: domain_names = couch_user.get_domains() return Domain.view("domain/by_status", keys=[[is_active, d] for d in domain_names], reduce=False, include_docs=True, stale=settings.COUCH_STALE_QUERY, ).all() else: return [] @classmethod def field_by_prefix(cls, field, prefix='', is_approved=True): # unichr(0xfff8) is something close to the highest character available res = cls.view("domain/fields_by_prefix", group=True, startkey=[field, is_approved, prefix], endkey=[field, is_approved, "%s%c" % (prefix, unichr(0xfff8)), {}]) vals = [(d['value'], d['key'][2]) for d in res] vals.sort(reverse=True) return [(v[1], v[0]) for v in vals] @classmethod def get_by_field(cls, field, value, is_approved=True): return cls.view('domain/fields_by_prefix', key=[field, is_approved, value], reduce=False, include_docs=True).all() def apply_migrations(self): self.migrations.apply(self) @staticmethod def all_for_user(user): if not hasattr(user,'get_profile'): # this had better be an anonymous user return [] from corehq.apps.users.models import CouchUser couch_user = CouchUser.from_django_user(user) if couch_user: domain_names = couch_user.get_domains() return Domain.view("domain/domains", keys=domain_names, reduce=False, include_docs=True).all() else: return [] def add(self, model_instance, is_active=True): """ Add something to this domain, through the generic relation. Returns the created membership object """ # Add membership info to Couch couch_user = model_instance.get_profile().get_couch_user() couch_user.add_domain_membership(self.name) couch_user.save() def applications(self): from corehq.apps.app_manager.models import ApplicationBase return ApplicationBase.view('app_manager/applications_brief', startkey=[self.name], endkey=[self.name, {}]).all() def full_applications(self, include_builds=True): from corehq.apps.app_manager.models import Application, RemoteApp WRAPPERS = {'Application': Application, 'RemoteApp': RemoteApp} def wrap_application(a): return WRAPPERS[a['doc']['doc_type']].wrap(a['doc']) if include_builds: startkey = [self.name] endkey = [self.name, {}] else: startkey = [self.name, None] endkey = [self.name, None, {}] return get_db().view('app_manager/applications', startkey=startkey, endkey=endkey, include_docs=True, wrapper=wrap_application).all() @cached_property def versions(self): apps = self.applications() return list(set(a.application_version for a in apps)) @cached_property def has_case_management(self): for app in self.full_applications(): if app.doc_type == 'Application': if app.has_case_management(): return True return False @cached_property def has_media(self): for app in self.full_applications(): if app.doc_type == 'Application' and app.has_media(): return True return False def all_users(self): from corehq.apps.users.models import CouchUser return CouchUser.by_domain(self.name) def has_shared_media(self): return False def recent_submissions(self): from corehq.apps.reports.util import make_form_couch_key key = make_form_couch_key(self.name) res = get_db().view('reports_forms/all_forms', startkey=key+[{}], endkey=key, descending=True, reduce=False, include_docs=False, limit=1).all() if len(res) > 0: # if there have been any submissions in the past 30 days return (datetime.now() <= datetime.strptime(res[0]['value']['submission_time'], "%Y-%m-%dT%H:%M:%SZ") + timedelta(days=30)) else: return False @cached_property def languages(self): apps = self.applications() return set(chain.from_iterable([a.langs for a in apps])) def readable_languages(self): return ', '.join(lang_lookup[lang] or lang for lang in self.languages()) def __unicode__(self): return self.name @classmethod def get_by_name(cls, name, strict=False): if not name: # get_by_name should never be called with name as None (or '', etc) # I fixed the code in such a way that if I raise a ValueError # all tests pass and basic pages load, # but in order not to break anything in the wild, # I'm opting to notify by email if/when this happens # but fall back to the previous behavior of returning None try: raise ValueError('%r is not a valid domain name' % name) except ValueError: if settings.DEBUG: raise else: notify_exception(None, '%r is not a valid domain name' % name) return None extra_args = {'stale': settings.COUCH_STALE_QUERY} if not strict else {} result = cls.view("domain/domains", key=name, reduce=False, include_docs=True, **extra_args ).first() if result is None and not strict: # on the off chance this is a brand new domain, try with strict return cls.get_by_name(name, strict=True) return result @classmethod def get_by_organization(cls, organization): result = cls.view("domain/by_organization", startkey=[organization], endkey=[organization, {}], reduce=False, include_docs=True) return result @classmethod def get_by_organization_and_hrname(cls, organization, hr_name): result = cls.view("domain/by_organization", key=[organization, hr_name], reduce=False, include_docs=True) return result @classmethod def get_or_create_with_name(cls, name, is_active=False): result = cls.view("domain/domains", key=name, reduce=False, include_docs=True).first() if result: return result else: new_domain = Domain(name=name, is_active=is_active, date_created=datetime.utcnow()) new_domain.save(**get_safe_write_kwargs()) return new_domain def password_format(self): """ This was a performance hit, so for now we'll just return 'a' no matter what # If a single application is alphanumeric, return alphanumeric; otherwise, return numeric """ # for app in self.full_applications(): # if hasattr(app, 'profile'): # format = app.profile.get('properties', {}).get('password_format', 'n') # if format == 'a': # return 'a' # return 'n' return 'a' @classmethod def get_all(cls, include_docs=True): return Domain.view("domain/not_snapshots", include_docs=include_docs).all() def case_sharing_included(self): return self.case_sharing or reduce(lambda x, y: x or y, [getattr(app, 'case_sharing', False) for app in self.applications()], False) def save_copy(self, new_domain_name=None, user=None): from corehq.apps.app_manager.models import get_app if new_domain_name is not None and Domain.get_by_name(new_domain_name): return None db = get_db() new_id = db.copy_doc(self.get_id)['id'] if new_domain_name is None: new_domain_name = new_id new_domain = Domain.get(new_id) new_domain.name = new_domain_name new_domain.copy_history = self.get_updated_history() new_domain.is_snapshot = False new_domain.snapshot_time = None new_domain.organization = None # TODO: use current user's organization (?) # reset the cda new_domain.cda.signed = False new_domain.cda.date = None new_domain.cda.type = None new_domain.cda.user_id = None new_domain.cda.user_ip = None for field in self._dirty_fields: if hasattr(new_domain, field): delattr(new_domain, field) for res in db.view('domain/related_to_domain', key=[self.name, True]): if not self.is_snapshot and res['value']['doc_type'] in ('Application', 'RemoteApp'): app = get_app(self.name, res['value']['_id']).get_latest_saved() if app: self.copy_component(app.doc_type, app._id, new_domain_name, user=user) else: self.copy_component(res['value']['doc_type'], res['value']['_id'], new_domain_name, user=user) else: self.copy_component(res['value']['doc_type'], res['value']['_id'], new_domain_name, user=user) new_domain.save() if user: def add_dom_to_user(user): user.add_domain_membership(new_domain_name, is_admin=True) apply_update(user, add_dom_to_user) return new_domain def copy_component(self, doc_type, id, new_domain_name, user=None): from corehq.apps.app_manager.models import import_app from corehq.apps.users.models import UserRole str_to_cls = { 'UserRole': UserRole, } db = get_db() if doc_type in ('Application', 'RemoteApp'): new_doc = import_app(id, new_domain_name) new_doc.copy_history.append(id) else: cls = str_to_cls[doc_type] new_id = db.copy_doc(id)['id'] new_doc = cls.get(new_id) for field in self._dirty_fields: if hasattr(new_doc, field): delattr(new_doc, field) if hasattr(cls, '_meta_fields'): for field in cls._meta_fields: if not field.startswith('_') and hasattr(new_doc, field): delattr(new_doc, field) new_doc.domain = new_domain_name if self.is_snapshot and doc_type == 'Application': new_doc.prepare_multimedia_for_exchange() new_doc.save() return new_doc def save_snapshot(self): if self.is_snapshot: return self else: copy = self.save_copy() if copy is None: return None copy.is_snapshot = True copy.organization = self.organization # i don't think we want this? copy.snapshot_time = datetime.now() del copy.deployment copy.save() return copy def from_snapshot(self): return not self.is_snapshot and self.original_doc is not None def snapshots(self): return Domain.view('domain/snapshots', startkey=[self._id, {}], endkey=[self._id], include_docs=True, descending=True) @memoized def published_snapshot(self): snapshots = self.snapshots().all() for snapshot in snapshots: if snapshot.published: return snapshot return None @classmethod def published_snapshots(cls, include_unapproved=False, page=None, per_page=10): skip = None limit = None if page: skip = (page - 1) * per_page limit = per_page if include_unapproved: return cls.view('domain/published_snapshots', startkey=[False, {}], include_docs=True, descending=True, limit=limit, skip=skip) else: return cls.view('domain/published_snapshots', endkey=[True], include_docs=True, descending=True, limit=limit, skip=skip) @classmethod def snapshot_search(cls, query, page=None, per_page=10): skip = None limit = None if page: skip = (page - 1) * per_page limit = per_page results = get_db().search('domain/snapshot_search', q=json.dumps(query), limit=limit, skip=skip, stale='ok') return map(cls.get, [r['id'] for r in results]), results.total_rows @memoized def get_organization(self): from corehq.apps.orgs.models import Organization return Organization.get_by_name(self.organization) @memoized def organization_title(self): if self.organization: return self.get_organization().title else: return '' def update_deployment(self, **kwargs): self.deployment.update(kwargs) self.save() def update_internal(self, **kwargs): self.internal.update(kwargs) self.save() def display_name(self): if self.is_snapshot: return "Snapshot of %s" % self.copied_from.display_name() if self.hr_name and self.organization: return self.hr_name else: return self.name def long_display_name(self): if self.is_snapshot: return format_html( "Snapshot of {0} > {1}", self.get_organization().title, self.copied_from.display_name() ) if self.organization: return format_html( '{0} > {1}', self.get_organization().title, self.hr_name or self.name ) else: return self.name __str__ = long_display_name def get_license_display(self): return LICENSES.get(self.license) def copies(self): return Domain.view('domain/copied_from_snapshot', key=self._id, include_docs=True) def copies_of_parent(self): return Domain.view('domain/copied_from_snapshot', keys=[s._id for s in self.copied_from.snapshots()], include_docs=True) def delete(self): # delete all associated objects db = get_db() related_docs = db.view('domain/related_to_domain', startkey=[self.name], endkey=[self.name, {}], include_docs=True) for doc in related_docs: db.delete_doc(doc['doc']) super(Domain, self).delete() def all_media(self, from_apps=None): #todo add documentation or refactor from corehq.apps.hqmedia.models import CommCareMultimedia dom_with_media = self if not self.is_snapshot else self.copied_from if self.is_snapshot: app_ids = [app.copied_from.get_id for app in self.full_applications()] if from_apps: from_apps = set([a_id for a_id in app_ids if a_id in from_apps]) else: from_apps = app_ids if from_apps: media = [] media_ids = set() apps = [app for app in dom_with_media.full_applications() if app.get_id in from_apps] for app in apps: for _, m in app.get_media_objects(): if m.get_id not in media_ids: media.append(m) media_ids.add(m.get_id) return media return CommCareMultimedia.view('hqmedia/by_domain', key=dom_with_media.name, include_docs=True).all() def most_restrictive_licenses(self, apps_to_check=None): from corehq.apps.hqmedia.utils import most_restrictive licenses = [m.license['type'] for m in self.all_media(from_apps=apps_to_check) if m.license] return most_restrictive(licenses) @classmethod def popular_sort(cls, domains): sorted_list = [] MIN_REVIEWS = 1.0 domains = [(domain, Review.get_average_rating_by_app(domain.copied_from._id), Review.get_num_ratings_by_app(domain.copied_from._id)) for domain in domains] domains = [(domain, avg or 0.0, num or 0) for domain, avg, num in domains] total_average_sum = sum(avg for domain, avg, num in domains) total_average_count = len(domains) if not total_average_count: return [] total_average = (total_average_sum / total_average_count) for domain, average_rating, num_ratings in domains: if num_ratings == 0: sorted_list.append((0.0, domain)) else: weighted_rating = ((num_ratings / (num_ratings + MIN_REVIEWS)) * average_rating + (MIN_REVIEWS / (num_ratings + MIN_REVIEWS)) * total_average) sorted_list.append((weighted_rating, domain)) sorted_list = [domain for weighted_rating, domain in sorted(sorted_list, key=lambda domain: domain[0], reverse=True)] return sorted_list @classmethod def hit_sort(cls, domains): domains = list(domains) domains = sorted(domains, key=lambda domain: domain.downloads, reverse=True) return domains @classmethod def public_deployments(cls): return Domain.view('domain/with_deployment', include_docs=True).all() @classmethod def get_module_by_name(cls, domain_name): """ import and return the python module corresponding to domain_name, or None if it doesn't exist. """ module_name = get_domain_module_map().get(domain_name, domain_name) try: return __import__(module_name) if module_name else None except ImportError: return None @property def commtrack_settings(self): # this import causes some dependency issues so lives in here from corehq.apps.commtrack.models import CommtrackConfig if self.commtrack_enabled: return CommtrackConfig.for_domain(self.name) else: return None def get_case_display(self, case): """Get the properties display definition for a given case""" return self.case_display.case_details.get(case.type) def get_form_display(self, form): """Get the properties display definition for a given XFormInstance""" return self.case_display.form_details.get(form.xmlns)
class Domain(Document, HQBillingDomainMixin, SnapshotMixin): """Domain is the highest level collection of people/stuff in the system. Pretty much everything happens at the domain-level, including user membership, permission to see data, reports, charts, etc.""" name = StringProperty() is_active = BooleanProperty() is_public = BooleanProperty(default=False) date_created = DateTimeProperty() default_timezone = StringProperty( default=getattr(settings, "TIME_ZONE", "UTC")) case_sharing = BooleanProperty(default=False) secure_submissions = BooleanProperty(default=False) ota_restore_caching = BooleanProperty(default=False) cloudcare_releases = StringProperty( choices=['stars', 'nostars', 'default'], default='default') organization = StringProperty() hr_name = StringProperty( ) # the human-readable name for this project within an organization creating_user = StringProperty( ) # username of the user who created this domain # domain metadata project_type = StringProperty() # e.g. MCH, HIV customer_type = StringProperty() # plus, full, etc. is_test = StringProperty(choices=["true", "false", "none"], default="none") description = StringProperty() short_description = StringProperty() is_shared = BooleanProperty(default=False) commtrack_enabled = BooleanProperty(default=False) call_center_config = SchemaProperty(CallCenterProperties) has_careplan = BooleanProperty(default=False) restrict_superusers = BooleanProperty(default=False) location_restriction_for_users = BooleanProperty(default=True) case_display = SchemaProperty(CaseDisplaySettings) # CommConnect settings commconnect_enabled = BooleanProperty(default=False) survey_management_enabled = BooleanProperty(default=False) sms_case_registration_enabled = BooleanProperty( default=False) # Whether or not a case can register via sms sms_case_registration_type = StringProperty( ) # Case type to apply to cases registered via sms sms_case_registration_owner_id = StringProperty( ) # Owner to apply to cases registered via sms sms_case_registration_user_id = StringProperty( ) # Submitting user to apply to cases registered via sms sms_mobile_worker_registration_enabled = BooleanProperty( default=False) # Whether or not a mobile worker can register via sms default_sms_backend_id = StringProperty() use_default_sms_response = BooleanProperty(default=False) default_sms_response = StringProperty() chat_message_count_threshold = IntegerProperty() custom_chat_template = StringProperty( ) # See settings.CUSTOM_CHAT_TEMPLATES custom_case_username = StringProperty( ) # Case property to use when showing the case's name in a chat window # If empty, sms can be sent at any time. Otherwise, only send during # these windows of time. SMS_QUEUE_ENABLED must be True in localsettings # for this be considered. restricted_sms_times = SchemaListProperty(DayTimeWindow) # If empty, this is ignored. Otherwise, the framework will make sure # that during these days/times, no automated outbound sms will be sent # to someone if they have sent in an sms within sms_conversation_length # minutes. Outbound sms sent from a user in a chat window, however, will # still be sent. This is meant to prevent chat conversations from being # interrupted by automated sms reminders. # SMS_QUEUE_ENABLED must be True in localsettings for this to be # considered. sms_conversation_times = SchemaListProperty(DayTimeWindow) # In minutes, see above. sms_conversation_length = IntegerProperty(default=10) # Set to True to prevent survey questions and answers form being seen in # SMS chat windows. filter_surveys_from_chat = BooleanProperty(default=False) # The below option only matters if filter_surveys_from_chat = True. # If set to True, invalid survey responses will still be shown in the chat # window, while questions and valid responses will be filtered out. show_invalid_survey_responses_in_chat = BooleanProperty(default=False) # If set to True, if a message is read by anyone it counts as being read by # everyone. Set to False so that a message is only counted as being read # for a user if only that user has read it. count_messages_as_read_by_anyone = BooleanProperty(default=False) # Set to True to allow sending sms and all-label surveys to cases whose # phone number is duplicated with another contact send_to_duplicated_case_numbers = BooleanProperty(default=False) # exchange/domain copying stuff is_snapshot = BooleanProperty(default=False) is_approved = BooleanProperty(default=False) snapshot_time = DateTimeProperty() published = BooleanProperty(default=False) license = StringProperty(choices=LICENSES, default='cc') title = StringProperty() cda = SchemaProperty(LicenseAgreement) multimedia_included = BooleanProperty(default=True) downloads = IntegerProperty( default=0) # number of downloads for this specific snapshot full_downloads = IntegerProperty( default=0) # number of downloads for all snapshots from this domain author = StringProperty() phone_model = StringProperty() attribution_notes = StringProperty() publisher = StringProperty(choices=["organization", "user"], default="user") yt_id = StringProperty() deployment = SchemaProperty(Deployment) image_path = StringProperty() image_type = StringProperty() migrations = SchemaProperty(DomainMigrations) cached_properties = DictProperty() internal = SchemaProperty(InternalProperties) dynamic_reports = SchemaListProperty(DynamicReportSet) # extra user specified properties tags = StringListProperty() area = StringProperty(choices=AREA_CHOICES) sub_area = StringProperty(choices=SUB_AREA_CHOICES) launch_date = DateTimeProperty # to be eliminated from projects and related documents when they are copied for the exchange _dirty_fields = ('admin_password', 'admin_password_charset', 'city', 'country', 'region', 'customer_type') @property def domain_type(self): """ The primary type of this domain. Used to determine site-specific branding. """ if self.commtrack_enabled: return 'commtrack' else: return 'commcare' @classmethod def wrap(cls, data): # for domains that still use original_doc should_save = False if 'original_doc' in data: original_doc = data['original_doc'] del data['original_doc'] should_save = True if original_doc: original_doc = Domain.get_by_name(original_doc) data['copy_history'] = [original_doc._id] # for domains that have a public domain license if 'license' in data: if data.get("license", None) == "public": data["license"] = "cc" should_save = True if 'slug' in data and data["slug"]: data["hr_name"] = data["slug"] del data["slug"] if 'is_test' in data and isinstance(data["is_test"], bool): data["is_test"] = "true" if data["is_test"] else "false" should_save = True if 'cloudcare_releases' not in data: data['cloudcare_releases'] = 'nostars' # legacy default setting self = super(Domain, cls).wrap(data) if self.deployment is None: self.deployment = Deployment() if self.get_id: self.apply_migrations() if should_save: self.save() return self @staticmethod def active_for_user(user, is_active=True): if isinstance(user, AnonymousUser): return [] from corehq.apps.users.models import CouchUser if isinstance(user, CouchUser): couch_user = user else: couch_user = CouchUser.from_django_user(user) if couch_user: domain_names = couch_user.get_domains() return cache_core.cached_view(Domain.get_db(), "domain/by_status", keys=[[is_active, d] for d in domain_names], reduce=False, include_docs=True, wrapper=Domain.wrap) else: return [] @classmethod def field_by_prefix(cls, field, prefix='', is_approved=True): # unichr(0xfff8) is something close to the highest character available res = cls.view( "domain/fields_by_prefix", group=True, startkey=[field, is_approved, prefix], endkey=[field, is_approved, "%s%c" % (prefix, unichr(0xfff8)), {}]) vals = [(d['value'], d['key'][2]) for d in res] vals.sort(reverse=True) return [(v[1], v[0]) for v in vals] @classmethod def get_by_field(cls, field, value, is_approved=True): return cls.view('domain/fields_by_prefix', key=[field, is_approved, value], reduce=False, include_docs=True).all() def apply_migrations(self): self.migrations.apply(self) @staticmethod def all_for_user(user): if not hasattr(user, 'get_profile'): # this had better be an anonymous user return [] from corehq.apps.users.models import CouchUser couch_user = CouchUser.from_django_user(user) if couch_user: domain_names = couch_user.get_domains() return Domain.view("domain/domains", keys=domain_names, reduce=False, include_docs=True).all() else: return [] def add(self, model_instance, is_active=True): """ Add something to this domain, through the generic relation. Returns the created membership object """ # Add membership info to Couch couch_user = model_instance.get_profile().get_couch_user() couch_user.add_domain_membership(self.name) couch_user.save() def applications(self): from corehq.apps.app_manager.models import ApplicationBase return ApplicationBase.view('app_manager/applications_brief', startkey=[self.name], endkey=[self.name, {}]).all() def full_applications(self, include_builds=True): from corehq.apps.app_manager.models import Application, RemoteApp WRAPPERS = {'Application': Application, 'RemoteApp': RemoteApp} def wrap_application(a): return WRAPPERS[a['doc']['doc_type']].wrap(a['doc']) if include_builds: startkey = [self.name] endkey = [self.name, {}] else: startkey = [self.name, None] endkey = [self.name, None, {}] return get_db().view('app_manager/applications', startkey=startkey, endkey=endkey, include_docs=True, wrapper=wrap_application).all() @cached_property def versions(self): apps = self.applications() return list(set(a.application_version for a in apps)) @cached_property def has_case_management(self): for app in self.full_applications(): if app.doc_type == 'Application': if app.has_case_management(): return True return False @cached_property def has_media(self): for app in self.full_applications(): if app.doc_type == 'Application' and app.has_media(): return True return False @property def use_cloudcare_releases(self): return self.cloudcare_releases != 'nostars' def all_users(self): from corehq.apps.users.models import CouchUser return CouchUser.by_domain(self.name) def has_shared_media(self): return False def recent_submissions(self): from corehq.apps.reports.util import make_form_couch_key key = make_form_couch_key(self.name) res = get_db().view('reports_forms/all_forms', startkey=key + [{}], endkey=key, descending=True, reduce=False, include_docs=False, limit=1).all() if len(res ) > 0: # if there have been any submissions in the past 30 days return (datetime.now() <= datetime.strptime(res[0]['key'][2], "%Y-%m-%dT%H:%M:%SZ") + timedelta(days=30)) else: return False @cached_property def languages(self): apps = self.applications() return set(chain.from_iterable([a.langs for a in apps])) def readable_languages(self): return ', '.join(lang_lookup[lang] or lang for lang in self.languages()) def __unicode__(self): return self.name @classmethod def get_by_name(cls, name, strict=False): if not name: # get_by_name should never be called with name as None (or '', etc) # I fixed the code in such a way that if I raise a ValueError # all tests pass and basic pages load, # but in order not to break anything in the wild, # I'm opting to notify by email if/when this happens # but fall back to the previous behavior of returning None try: raise ValueError('%r is not a valid domain name' % name) except ValueError: if settings.DEBUG: raise else: notify_exception(None, '%r is not a valid domain name' % name) return None cache_key = _domain_cache_key(name) MISSING = object() res = cache.get(cache_key, MISSING) if res != MISSING: return res else: domain = cls._get_by_name(name, strict) # 30 mins, so any unforeseen invalidation bugs aren't too bad. cache.set(cache_key, domain, 30 * 60) return domain @classmethod def _get_by_name(cls, name, strict=False): extra_args = { 'stale': settings.COUCH_STALE_QUERY } if not strict else {} db = cls.get_db() res = cache_core.cached_view(db, "domain/domains", key=name, reduce=False, include_docs=True, wrapper=cls.wrap, force_invalidate=strict, **extra_args) if len(res) > 0: result = res[0] else: result = None if result is None and not strict: # on the off chance this is a brand new domain, try with strict return cls.get_by_name(name, strict=True) return result @classmethod def get_by_organization(cls, organization): result = cache_core.cached_view(cls.get_db(), "domain/by_organization", startkey=[organization], endkey=[organization, {}], reduce=False, include_docs=True, wrapper=cls.wrap) from corehq.apps.accounting.utils import domain_has_privilege from corehq import privileges result = filter( lambda x: domain_has_privilege(x.name, privileges. CROSS_PROJECT_REPORTS), result) return result @classmethod def get_by_organization_and_hrname(cls, organization, hr_name): result = cls.view("domain/by_organization", key=[organization, hr_name], reduce=False, include_docs=True) return result @classmethod def get_or_create_with_name(cls, name, is_active=False, secure_submissions=True): result = cls.view("domain/domains", key=name, reduce=False, include_docs=True).first() if result: return result else: new_domain = Domain( name=name, is_active=is_active, date_created=datetime.utcnow(), secure_submissions=secure_submissions, ) new_domain.migrations = DomainMigrations( has_migrated_permissions=True) new_domain.save(**get_safe_write_kwargs()) return new_domain def password_format(self): """ This was a performance hit, so for now we'll just return 'a' no matter what If a single application is alphanumeric, return alphanumeric; otherwise, return numeric """ return 'a' @classmethod def get_all(cls, include_docs=True): # todo: this should use iter_docs return Domain.view("domain/not_snapshots", include_docs=include_docs).all() def case_sharing_included(self): return self.case_sharing or reduce(lambda x, y: x or y, [ getattr(app, 'case_sharing', False) for app in self.applications() ], False) def save(self, **params): super(Domain, self).save(**params) cache.delete(_domain_cache_key(self.name)) from corehq.apps.domain.signals import commcare_domain_post_save results = commcare_domain_post_save.send_robust(sender='domain', domain=self) for result in results: # Second argument is None if there was no error if result[1]: notify_exception( None, message="Error occured during domain post_save %s: %s" % (self.name, str(result[1]))) def save_copy(self, new_domain_name=None, user=None, ignore=None): from corehq.apps.app_manager.models import get_app from corehq.apps.reminders.models import CaseReminderHandler ignore = ignore if ignore is not None else [] if new_domain_name is not None and Domain.get_by_name(new_domain_name): return None db = get_db() new_id = db.copy_doc(self.get_id)['id'] if new_domain_name is None: new_domain_name = new_id new_domain = Domain.get(new_id) new_domain.name = new_domain_name new_domain.copy_history = self.get_updated_history() new_domain.is_snapshot = False new_domain.snapshot_time = None new_domain.organization = None # TODO: use current user's organization (?) # reset stuff new_domain.cda.signed = False new_domain.cda.date = None new_domain.cda.type = None new_domain.cda.user_id = None new_domain.cda.user_ip = None new_domain.is_test = "none" new_domain.internal = InternalProperties() new_domain.creating_user = user.username if user else None for field in self._dirty_fields: if hasattr(new_domain, field): delattr(new_domain, field) new_comps = {} # a mapping of component's id to it's copy for res in db.view('domain/related_to_domain', key=[self.name, True]): if not self.is_snapshot and res['value']['doc_type'] in ( 'Application', 'RemoteApp'): app = get_app(self.name, res['value']['_id']).get_latest_saved() if app: comp = self.copy_component(app.doc_type, app._id, new_domain_name, user=user) else: comp = self.copy_component(res['value']['doc_type'], res['value']['_id'], new_domain_name, user=user) elif res['value']['doc_type'] not in ignore: comp = self.copy_component(res['value']['doc_type'], res['value']['_id'], new_domain_name, user=user) else: comp = None if comp: new_comps[res['value']['_id']] = comp new_domain.save() if user: def add_dom_to_user(user): user.add_domain_membership(new_domain_name, is_admin=True) apply_update(user, add_dom_to_user) def update_events(handler): """ Change the form_unique_id to the proper form for each event in a newly copied CaseReminderHandler """ from corehq.apps.app_manager.models import FormBase for event in handler.events: if not event.form_unique_id: continue form = FormBase.get_form(event.form_unique_id) form_app = form.get_app() m_index, f_index = form_app.get_form_location(form.unique_id) form_copy = new_comps[form_app._id].get_module( m_index).get_form(f_index) event.form_unique_id = form_copy.unique_id def update_for_copy(handler): handler.active = False update_events(handler) if 'CaseReminderHandler' not in ignore: for handler in CaseReminderHandler.get_handlers(new_domain_name): apply_update(handler, update_for_copy) return new_domain def reminder_should_be_copied(self, handler): from corehq.apps.reminders.models import ON_DATETIME return (handler.start_condition_type != ON_DATETIME and handler.user_group_id is None) def copy_component(self, doc_type, id, new_domain_name, user=None): from corehq.apps.app_manager.models import import_app from corehq.apps.users.models import UserRole from corehq.apps.reminders.models import CaseReminderHandler str_to_cls = { 'UserRole': UserRole, 'CaseReminderHandler': CaseReminderHandler, } db = get_db() if doc_type in ('Application', 'RemoteApp'): new_doc = import_app(id, new_domain_name) new_doc.copy_history.append(id) else: cls = str_to_cls[doc_type] if doc_type == 'CaseReminderHandler': cur_doc = cls.get(id) if not self.reminder_should_be_copied(cur_doc): return None new_id = db.copy_doc(id)['id'] new_doc = cls.get(new_id) for field in self._dirty_fields: if hasattr(new_doc, field): delattr(new_doc, field) if hasattr(cls, '_meta_fields'): for field in cls._meta_fields: if not field.startswith('_') and hasattr(new_doc, field): delattr(new_doc, field) new_doc.domain = new_domain_name if self.is_snapshot and doc_type == 'Application': new_doc.prepare_multimedia_for_exchange() new_doc.save() return new_doc def save_snapshot(self, ignore=None): if self.is_snapshot: return self else: copy = self.save_copy(ignore=ignore) if copy is None: return None copy.is_snapshot = True copy.snapshot_time = datetime.now() del copy.deployment copy.save() return copy def from_snapshot(self): return not self.is_snapshot and self.original_doc is not None def snapshots(self): return Domain.view('domain/snapshots', startkey=[self._id, {}], endkey=[self._id], include_docs=True, reduce=False, descending=True) @memoized def published_snapshot(self): snapshots = self.snapshots().all() for snapshot in snapshots: if snapshot.published: return snapshot return None @classmethod def published_snapshots(cls, include_unapproved=False, page=None, per_page=10): skip = None limit = None if page: skip = (page - 1) * per_page limit = per_page if include_unapproved: return cls.view('domain/published_snapshots', startkey=[False, {}], include_docs=True, descending=True, limit=limit, skip=skip) else: return cls.view('domain/published_snapshots', endkey=[True], include_docs=True, descending=True, limit=limit, skip=skip) @classmethod def snapshot_search(cls, query, page=None, per_page=10): skip = None limit = None if page: skip = (page - 1) * per_page limit = per_page results = get_db().search( 'domain/snapshot_search', q=json.dumps(query), limit=limit, skip=skip, #stale='ok', ) return map(cls.get, [r['id'] for r in results]), results.total_rows @memoized def get_organization(self): from corehq.apps.orgs.models import Organization return Organization.get_by_name(self.organization) @memoized def organization_title(self): if self.organization: return self.get_organization().title else: return '' def update_deployment(self, **kwargs): self.deployment.update(kwargs) self.save() def update_internal(self, **kwargs): self.internal.update(kwargs) self.save() def display_name(self): if self.is_snapshot: return "Snapshot of %s" % self.copied_from.display_name() if self.hr_name and self.organization: return self.hr_name else: return self.name def long_display_name(self): if self.is_snapshot: return format_html("Snapshot of {0} > {1}", self.get_organization().title, self.copied_from.display_name()) if self.organization: return format_html('{0} > {1}', self.get_organization().title, self.hr_name or self.name) else: return self.name __str__ = long_display_name def get_license_display(self): return LICENSES.get(self.license) def copies(self): return Domain.view('domain/copied_from_snapshot', key=self._id, include_docs=True) def copies_of_parent(self): return Domain.view('domain/copied_from_snapshot', keys=[s._id for s in self.copied_from.snapshots()], include_docs=True) def delete(self): # delete all associated objects db = get_db() related_docs = db.view('domain/related_to_domain', startkey=[self.name], endkey=[self.name, {}], include_docs=True) for doc in related_docs: db.delete_doc(doc['doc']) super(Domain, self).delete() def all_media(self, from_apps=None): #todo add documentation or refactor from corehq.apps.hqmedia.models import CommCareMultimedia dom_with_media = self if not self.is_snapshot else self.copied_from if self.is_snapshot: app_ids = [ app.copied_from.get_id for app in self.full_applications() ] if from_apps: from_apps = set( [a_id for a_id in app_ids if a_id in from_apps]) else: from_apps = app_ids if from_apps: media = [] media_ids = set() apps = [ app for app in dom_with_media.full_applications() if app.get_id in from_apps ] for app in apps: if app.doc_type != 'Application': continue for _, m in app.get_media_objects(): if m.get_id not in media_ids: media.append(m) media_ids.add(m.get_id) return media return CommCareMultimedia.view('hqmedia/by_domain', key=dom_with_media.name, include_docs=True).all() def most_restrictive_licenses(self, apps_to_check=None): from corehq.apps.hqmedia.utils import most_restrictive licenses = [ m.license['type'] for m in self.all_media(from_apps=apps_to_check) if m.license ] return most_restrictive(licenses) @classmethod def popular_sort(cls, domains): sorted_list = [] MIN_REVIEWS = 1.0 domains = [(domain, Review.get_average_rating_by_app(domain.copied_from._id), Review.get_num_ratings_by_app(domain.copied_from._id)) for domain in domains] domains = [(domain, avg or 0.0, num or 0) for domain, avg, num in domains] total_average_sum = sum(avg for domain, avg, num in domains) total_average_count = len(domains) if not total_average_count: return [] total_average = (total_average_sum / total_average_count) for domain, average_rating, num_ratings in domains: if num_ratings == 0: sorted_list.append((0.0, domain)) else: weighted_rating = ( (num_ratings / (num_ratings + MIN_REVIEWS)) * average_rating + (MIN_REVIEWS / (num_ratings + MIN_REVIEWS)) * total_average) sorted_list.append((weighted_rating, domain)) sorted_list = [ domain for weighted_rating, domain in sorted( sorted_list, key=lambda domain: domain[0], reverse=True) ] return sorted_list @classmethod def hit_sort(cls, domains): domains = list(domains) domains = sorted(domains, key=lambda domain: domain.download_count, reverse=True) return domains @classmethod def public_deployments(cls): return Domain.view('domain/with_deployment', include_docs=True).all() @classmethod def get_module_by_name(cls, domain_name): """ import and return the python module corresponding to domain_name, or None if it doesn't exist. """ from corehq.apps.domain.utils import get_domain_module_map module_name = get_domain_module_map().get(domain_name, domain_name) try: return import_module(module_name) if module_name else None except ImportError: return None @property @memoized def commtrack_settings(self): # this import causes some dependency issues so lives in here from corehq.apps.commtrack.models import CommtrackConfig if self.commtrack_enabled: return CommtrackConfig.for_domain(self.name) else: return None @property def has_custom_logo(self): return (self['_attachments'] and LOGO_ATTACHMENT in self['_attachments']) def get_custom_logo(self): if not self.has_custom_logo: return None return (self.fetch_attachment(LOGO_ATTACHMENT), self['_attachments'][LOGO_ATTACHMENT]['content_type']) def get_case_display(self, case): """Get the properties display definition for a given case""" return self.case_display.case_details.get(case.type) def get_form_display(self, form): """Get the properties display definition for a given XFormInstance""" return self.case_display.form_details.get(form.xmlns) @property def total_downloads(self): """ Returns the total number of downloads from every snapshot created from this domain """ return get_db().view( "domain/snapshots", startkey=[self.get_id], endkey=[self.get_id, {}], reduce=True, include_docs=False, ).one()["value"] @property @memoized def download_count(self): """ Updates and returns the total number of downloads from every sister snapshot. """ if self.is_snapshot: self.full_downloads = self.copied_from.total_downloads return self.full_downloads @property @memoized def published_by(self): from corehq.apps.users.models import CouchUser pb_id = self.cda.user_id return CouchUser.get_by_user_id(pb_id) if pb_id else None @property def name_of_publisher(self): return self.published_by.human_friendly_name if self.published_by else ""
class ReportNotification(CachedCouchDocumentMixin, Document): domain = StringProperty() owner_id = StringProperty() recipient_emails = StringListProperty() config_ids = StringListProperty() send_to_owner = BooleanProperty() attach_excel = BooleanProperty() # language is only used if some of the config_ids refer to UCRs or custom reports language = StringProperty() email_subject = StringProperty(default=DEFAULT_REPORT_NOTIF_SUBJECT) hour = IntegerProperty(default=8) minute = IntegerProperty(default=0) day = IntegerProperty(default=1) interval = StringProperty(choices=["hourly", "daily", "weekly", "monthly"]) uuid = StringProperty() start_date = DateProperty(default=None) addedToBulk = BooleanProperty(default=False) @property def is_editable(self): try: self.report_slug return False except AttributeError: return True @classmethod def get_report(cls, report_id): try: notification = ReportNotification.get(report_id) except ResourceNotFound: notification = None else: if notification.doc_type != 'ReportNotification': notification = None return notification @classmethod def by_domain(cls, domain, stale=True, **kwargs): if stale: kwargs['stale'] = settings.COUCH_STALE_QUERY key = [domain] return cls._get_view_by_key(key, **kwargs) @classmethod def by_domain_and_owner(cls, domain, owner_id, stale=True, **kwargs): if stale: kwargs['stale'] = settings.COUCH_STALE_QUERY key = [domain, owner_id] return cls._get_view_by_key(key, **kwargs) @classmethod def _get_view_by_key(cls, key, **kwargs): db = cls.get_db() result = cache_core.cached_view(db, "reportconfig/user_notifications", reduce=False, include_docs=True, startkey=key, endkey=key + [{}], wrapper=cls.wrap, **kwargs) return result @property @memoized def all_recipient_emails(self): emails = frozenset(self.recipient_emails) if self.send_to_owner and self.owner_email: emails |= {self.owner_email} return emails @property @memoized def owner_email(self): if self.owner.is_web_user(): return self.owner.username email = self.owner.get_email() try: validate_email(email) return email except Exception: pass @property @memoized def owner(self): id = self.owner_id return CouchUser.get_by_user_id(id) @property @memoized def configs(self): """ Access the notification's associated configs as a list, transparently returning an appropriate dummy for old notifications which have `report_slug` instead of `config_ids`. """ if self.config_ids: configs = [] for config_doc in iter_docs(ReportConfig.get_db(), self.config_ids): config = ReportConfig.wrap(config_doc) if not hasattr(config, 'deleted'): configs.append(config) def _sort_key(config_id): if config_id in self.config_ids: return self.config_ids.index(config_id) else: return len(self.config_ids) configs = sorted(configs, key=_sort_key) elif self.report_slug == 'admin_domains': raise UnsupportedScheduledReportError( "admin_domains is no longer " "supported as a schedulable report for the time being") else: # create a new ReportConfig object, useful for its methods and # calculated properties, but don't save it class ReadonlyReportConfig(ReportConfig): def save(self, *args, **kwargs): pass config = ReadonlyReportConfig() object.__setattr__(config, '_id', 'dummy') config.report_type = ProjectReportDispatcher.prefix config.report_slug = self.report_slug config.domain = self.domain config.owner_id = self.owner_id configs = [config] return tuple(configs) @property def day_name(self): if self.interval == 'hourly': return _("Every hour") if self.interval == 'weekly': return calendar.day_name[self.day] return { "daily": _("Every day"), "monthly": _("Day %s of every month" % self.day), }[self.interval] @classmethod def day_choices(cls): """Tuples for day of week number and human-readable day of week""" return tuple([(val, calendar.day_name[val]) for val in range(7)]) @classmethod def hour_choices(cls): """Tuples for hour number and human-readable hour""" return tuple([(val, "%s:00" % val) for val in range(24)]) @property @memoized def recipients_by_language(self): user_languages = { user['username']: user['language'] for user in get_user_docs_by_username(self.all_recipient_emails) if 'username' in user and 'language' in user } if self.language: fallback_language = self.language else: fallback_language = user_languages.get(self.owner_email, 'en') recipients = defaultdict(list) for email in self.all_recipient_emails: language = user_languages.get( email, fallback_language) or fallback_language recipients[language].append(email) return immutabledict(recipients) def get_secret(self, email): uuid = self._get_or_create_uuid() return hashlib.sha1((uuid + email).encode('utf-8')).hexdigest()[:20] def send(self): # Scenario: user has been removed from the domain that they # have scheduled reports for. Delete this scheduled report if not self.owner.is_member_of(self.domain, allow_enterprise=True): self.delete() return if self.owner.is_deleted(): self.delete() return if self.recipients_by_language: for language, emails in self.recipients_by_language.items(): self._get_and_send_report(language, emails) def _get_or_create_uuid(self): if not self.uuid: self.uuid = uuid.uuid4().hex self.save() return self.uuid def _get_and_send_report(self, language, emails): with localize(language): title = self._get_title(self.email_subject) attach_excel = getattr(self, 'attach_excel', False) report_text, excel_files = self._generate_report( attach_excel, title, emails) # Both are empty if ALL the ReportConfigs in the ReportNotification # have a start_date in the future (or an exception occurred) if not report_text and not excel_files: return self._send_emails(title, report_text, emails, excel_files) @staticmethod def _get_title(subject): # only translate the default subject return (_(DEFAULT_REPORT_NOTIF_SUBJECT) if subject == DEFAULT_REPORT_NOTIF_SUBJECT else subject) def _generate_report(self, attach_excel, title, emails): from corehq.apps.reports.views import get_scheduled_report_response report_text = None excel_files = None try: report_text, excel_files = get_scheduled_report_response( self.owner, self.domain, self._id, attach_excel=attach_excel, send_only_active=True) # TODO: Be more specific with our error-handling. If building the report could fail, # we should have a reasonable idea of why except Exception as er: notify_exception( None, message="Encountered error while generating report", details={ 'subject': title, 'recipients': str(emails), 'error': er, }) if isinstance(er, ESError): # An ElasticSearch error could indicate that the report itself is too large. # Try exporting the report instead, as that builds the report in chunks, # rather than all at once. # TODO: narrow down this handling so that we don't try this if ElasticSearch is simply down, # for example self._export_report(emails, title) return report_text, excel_files def _send_emails(self, title, report_text, emails, excel_files): from corehq.apps.reports.views import render_full_report_notification email_is_too_large = False for email in emails: body = render_full_report_notification(None, report_text, email, self).content try: self._send_email(title, email, body, excel_files) except Exception as er: if isinstance(er, SMTPSenderRefused) and ( er.smtp_code in LARGE_FILE_SIZE_ERROR_CODES): email_is_too_large = True break else: ScheduledReportLogger.log_email_failure( self, email, body, er) else: ScheduledReportLogger.log_email_success(self, email, body) if email_is_too_large: # TODO: Because different domains may have different size thresholds, # one of the middle addresses could have triggered this, causing us to send # both the original email and the retried email to some users. # This is likely best handled by treating each address separately. ScheduledReportLogger.log_email_size_failure( self, email, emails, body) # The body is too large -- attempt to resend the report as attachments. if excel_files: # If the attachments already exist, just send them self._send_only_attachments(title, emails, excel_files) else: # Otherwise we're forced to trigger a process to create them self._export_report(emails, title) def _send_email(self, title, email, body, excel_files): send_HTML_email(title, email, body, email_from=settings.DEFAULT_FROM_EMAIL, file_attachments=excel_files, smtp_exception_skip_list=LARGE_FILE_SIZE_ERROR_CODES) def _send_only_attachments(self, title, emails, excel_files): message = _( "Unable to generate email report. Excel files are attached.") send_HTML_email(title, emails, message, email_from=settings.DEFAULT_FROM_EMAIL, file_attachments=excel_files) def _export_report(self, emails, title): from corehq.apps.reports.standard.deployments import ApplicationStatusReport for report_config in self.configs: mock_request = HttpRequest() mock_request.couch_user = self.owner mock_request.user = self.owner.get_django_user() mock_request.domain = self.domain mock_request.couch_user.current_domain = self.domain mock_request.couch_user.language = self.language mock_request.method = 'GET' mock_request.bypass_two_factor = True mock_query_string_parts = [ report_config.query_string, 'filterSet=true' ] mock_request.GET = QueryDict('&'.join(mock_query_string_parts)) request_data = vars(mock_request) request_data['couch_user'] = mock_request.couch_user.userID if report_config.report_slug != ApplicationStatusReport.slug: # ApplicationStatusReport doesn't have date filter date_range = report_config.get_date_range() start_date = datetime.strptime(date_range['startdate'], '%Y-%m-%d') end_date = datetime.strptime(date_range['enddate'], '%Y-%m-%d') datespan = DateSpan(start_date, end_date) request_data['datespan'] = datespan full_request = { 'request': request_data, 'domain': request_data['domain'], 'context': {}, 'request_params': json_request(request_data['GET']) } export_all_rows_task(report_config.report, full_request, emails, title) def remove_recipient(self, email): try: if email == self.owner.get_email(): self.send_to_owner = False self.recipient_emails.remove(email) except ValueError: pass def update_attributes(self, items): for k, v in items: if k == 'start_date': self.verify_start_date(v) self.__setattr__(k, v) def verify_start_date(self, start_date): if start_date != self.start_date and start_date < datetime.today( ).date(): raise ValidationError( "You can not specify a start date in the past.") def can_be_viewed_by(self, user): return ((user._id == self.owner_id) or (user.is_domain_admin(self.domain) or (user.get_email() in self.all_recipient_emails)))
class OpmReportSnapshot(Document): """ Represents a snapshot of a report, to be taken at the end of each month """ domain = StringProperty() month = IntegerProperty() year = IntegerProperty() report_class = StringProperty() headers = StringListProperty() slugs = StringListProperty() rows = ListProperty() block = StringProperty() visible_cols = ListProperty() @classmethod def by_month(cls, month, year, report_class, block=None): return cls.view('opm_tasks/opm_snapshots', key=[DOMAIN, month, year, report_class, block], reduce=False, include_docs=True).first() @classmethod def filtered(cls, snapshot, report): filtered_rows = [] need_filering = False filters = [] keys_list = [] for key, field in report.filter_fields: keys = report.filter_data.get(field, []) if keys: need_filering = True filters.append((key, field)) if field == 'gp': keys_list.append([ user._id for user in report.users if 'user_data' in user and 'gp' in user.user_data and user.user_data['gp'] and user.user_data['gp'] in keys ]) else: keys_list.append(keys) if need_filering: def get_slug(key): if key in snapshot.slugs: return snapshot.slugs.index(key) return None filters = filter(lambda x: x is not None, [get_slug(key) for key, value in filters]) get_element = lambda row, i: row[i] if row[i] else "" for row in snapshot.rows: values = [(bool(keys_list[i]) and get_element(row, f) in keys_list[i]) for i, f in enumerate(filters)] if all(values): filtered_rows.append(row) snapshot.rows = filtered_rows return snapshot @classmethod def from_view(cls, report): block = None if report.block: block = report.block.lower() snapshot = cls.view('opm_tasks/opm_snapshots', key=[ DOMAIN, report.month, report.year, report.__class__.__name__, block ], reduce=False, include_docs=True).first() if not snapshot: return None return cls.filtered(snapshot, report)
class GroupExportConfiguration(Document): """ An export configuration allows you to setup a collection of exports that all run together. Used by the management command or a scheduled job to run a bunch of exports on a schedule. """ full_exports = SchemaListProperty(ExportConfiguration) custom_export_ids = StringListProperty() def get_custom_exports(self): for custom in list(self.custom_export_ids): custom_export = self._get_custom(custom) if custom_export: yield custom_export def _get_custom(self, custom_id): """ Get a custom export, or delete it's reference if not found """ try: return SavedExportSchema.get(custom_id) except ResourceNotFound: try: self.custom_export_ids.remove(custom_id) self.save() except ValueError: pass @property @memoized def saved_exports(self): return self._saved_exports_from_configs(self.all_configs) def _saved_exports_from_configs(self, configs): exports = SavedBasicExport.view( "couchexport/saved_exports", keys=[json.dumps(config.index) for config in configs], include_docs=True, reduce=False, ).all() export_map = dict((json.dumps(export.configuration.index), export) for export in exports) return [(config, export_map.get(json.dumps(config.index), None)) for config in configs] @property def all_configs(self): """ Return an iterator of config-like objects that include the main configs + the custom export configs. """ for full in self.full_exports: yield full for custom in self.get_custom_exports(): yield custom.to_export_config() @property def all_export_schemas(self): """ Return an iterator of ExportSchema-like objects that include the main configs + the custom export configs. """ for full in self.full_exports: yield FakeSavedExportSchema(index=full.index, type=full.type) for custom in self.get_custom_exports(): yield custom @property @memoized def all_exports(self): """ Returns an iterator of tuples consisting of the export config and an ExportSchema-like document that can be used to get at the data. """ return zip(self.all_configs, self.all_export_schemas)