class DataSourceConfiguration(UnicodeMixIn, CachedCouchDocumentMixin, Document): """ A data source configuration. These map 1:1 with database tables that get created. Each data source can back an arbitrary number of reports. """ domain = StringProperty(required=True) engine_id = StringProperty(default=UCR_ENGINE_ID) es_index_settings = SchemaProperty(ElasticSearchIndexSettings) backend_id = StringProperty(default=UCR_SQL_BACKEND) referenced_doc_type = StringProperty(required=True) table_id = StringProperty(required=True) display_name = StringProperty() base_item_expression = DictProperty() configured_filter = DictProperty() configured_indicators = ListProperty() named_expressions = DictProperty() named_filters = DictProperty() meta = SchemaProperty(DataSourceMeta) is_deactivated = BooleanProperty(default=False) last_modified = DateTimeProperty() asynchronous = BooleanProperty(default=False) class Meta(object): # prevent JsonObject from auto-converting dates etc. string_conversions = () def __unicode__(self): return u'{} - {}'.format(self.domain, self.display_name) def save(self, **params): self.last_modified = datetime.utcnow() super(DataSourceConfiguration, self).save(**params) def filter(self, document): filter_fn = self._get_main_filter() return filter_fn(document, EvaluationContext(document, 0)) def deleted_filter(self, document): filter_fn = self._get_deleted_filter() return filter_fn and filter_fn(document, EvaluationContext(document, 0)) @memoized def _get_main_filter(self): return self._get_filter([self.referenced_doc_type]) @memoized def _get_deleted_filter(self): return self._get_filter(get_deleted_doc_types(self.referenced_doc_type), include_configured=False) def _get_filter(self, doc_types, include_configured=True): if not doc_types: return None extras = ( [self.configured_filter] if include_configured and self.configured_filter else [] ) built_in_filters = [ self._get_domain_filter_spec(), { 'type': 'or', 'filters': [ { 'type': 'property_match', 'property_name': 'doc_type', 'property_value': doc_type, } for doc_type in doc_types ], }, ] return FilterFactory.from_spec( { 'type': 'and', 'filters': built_in_filters + extras, }, context=self.get_factory_context(), ) def _get_domain_filter_spec(self): return { 'type': 'property_match', 'property_name': 'domain', 'property_value': self.domain, } @property @memoized def named_expression_objects(self): named_expression_specs = deepcopy(self.named_expressions) named_expressions = {} spec_error = None while named_expression_specs: number_generated = 0 for name, expression in named_expression_specs.items(): try: named_expressions[name] = ExpressionFactory.from_spec( expression, FactoryContext(named_expressions=named_expressions, named_filters={}) ) number_generated += 1 del named_expression_specs[name] except BadSpecError as spec_error: # maybe a nested name resolution issue, try again on the next pass pass if number_generated == 0 and named_expression_specs: # we unsuccessfully generated anything on this pass and there are still unresolved # references. we have to fail. assert spec_error is not None raise spec_error return named_expressions @property @memoized def named_filter_objects(self): return {name: FilterFactory.from_spec(filter, FactoryContext(self.named_expression_objects, {})) for name, filter in self.named_filters.items()} def get_factory_context(self): return FactoryContext(self.named_expression_objects, self.named_filter_objects) @property @memoized def default_indicators(self): default_indicators = [IndicatorFactory.from_spec({ "column_id": "doc_id", "type": "expression", "display_name": "document id", "datatype": "string", "is_nullable": False, "is_primary_key": True, "expression": { "type": "root_doc", "expression": { "type": "property_name", "property_name": "_id" } } }, self.get_factory_context())] default_indicators.append(IndicatorFactory.from_spec({ "type": "inserted_at", }, self.get_factory_context())) if self.base_item_expression: default_indicators.append(IndicatorFactory.from_spec({ "type": "repeat_iteration", }, self.get_factory_context())) return default_indicators @property @memoized def indicators(self): return CompoundIndicator( self.display_name, self.default_indicators + [ IndicatorFactory.from_spec(indicator, self.get_factory_context()) for indicator in self.configured_indicators ] ) @property @memoized def parsed_expression(self): if self.base_item_expression: return ExpressionFactory.from_spec(self.base_item_expression, context=self.get_factory_context()) return None def get_columns(self): return self.indicators.get_columns() def get_items(self, document, eval_context=None): if self.filter(document): if not self.base_item_expression: return [document] else: result = self.parsed_expression(document, eval_context) if result is None: return [] elif isinstance(result, list): return result else: return [result] else: return [] def get_all_values(self, doc, eval_context=None): if not eval_context: eval_context = EvaluationContext(doc) rows = [] for item in self.get_items(doc, eval_context): indicators = self.indicators.get_values(item, eval_context) rows.append(indicators) eval_context.increment_iteration() return rows def get_report_count(self): """ Return the number of ReportConfigurations that reference this data source. """ return ReportConfiguration.count_by_data_source(self.domain, self._id) def validate(self, required=True): super(DataSourceConfiguration, self).validate(required) # these two properties implicitly call other validation self._get_main_filter() self._get_deleted_filter() # validate indicators and column uniqueness columns = [c.id for c in self.indicators.get_columns()] unique_columns = set(columns) if len(columns) != len(unique_columns): for column in set(columns): columns.remove(column) raise BadSpecError(_('Report contains duplicate column ids: {}').format(', '.join(set(columns)))) if self.referenced_doc_type not in VALID_REFERENCED_DOC_TYPES: raise BadSpecError( _('Report contains invalid referenced_doc_type: {}').format(self.referenced_doc_type)) self.parsed_expression @classmethod def by_domain(cls, domain): return get_datasources_for_domain(domain) @classmethod def all_ids(cls): return [res['id'] for res in cls.get_db().view('userreports/data_sources_by_build_info', reduce=False, include_docs=False)] @classmethod def all(cls): for result in iter_docs(cls.get_db(), cls.all_ids()): yield cls.wrap(result) @property def is_static(self): return id_is_static(self._id) def deactivate(self): if not self.is_static: self.is_deactivated = True self.save() get_indicator_adapter(self).drop_table() def get_es_index_settings(self): es_index_settings = self.es_index_settings.to_json() es_index_settings.pop('doc_type') return {"settings": es_index_settings}
class Repeater(QuickCachedDocumentMixin, Document): """ Represents the configuration of a repeater. Will specify the URL to forward to and other properties of the configuration. """ base_doc = 'Repeater' domain = StringProperty() connection_settings_id = IntegerProperty(required=False, default=None) # TODO: Delete the following properties once all Repeaters have been # migrated to ConnectionSettings. (2020-05-16) url = StringProperty() auth_type = StringProperty(choices=(BASIC_AUTH, DIGEST_AUTH, OAUTH1, BEARER_AUTH), required=False) username = StringProperty() password = StringProperty() # See also plaintext_password() skip_cert_verify = BooleanProperty(default=False) # See also verify() notify_addresses_str = StringProperty( default="") # See also notify_addresses() format = StringProperty() friendly_name = _("Data") paused = BooleanProperty(default=False) # TODO: Use to collect stats to determine whether remote endpoint is valid started_at = DateTimeProperty(default=datetime.utcnow) last_success_at = DateTimeProperty(required=False, default=None) failure_streak = IntegerProperty(default=0) payload_generator_classes = () _has_config = False def __str__(self): return f'{self.__class__.__name__}: {self.name}' def __repr__(self): return f"<{self.__class__.__name__} {self._id} {self.name!r}>" @cached_property def connection_settings(self): if not self.connection_settings_id: return self.create_connection_settings() return ConnectionSettings.objects.get(pk=self.connection_settings_id) @property def name(self): return self.connection_settings.name @classmethod def available_for_domain(cls, domain): """Returns whether this repeater can be used by a particular domain """ return True def get_pending_record_count(self): return get_pending_repeat_record_count(self.domain, self._id) def get_failure_record_count(self): return get_failure_repeat_record_count(self.domain, self._id) def get_success_record_count(self): return get_success_repeat_record_count(self.domain, self._id) def get_cancelled_record_count(self): return get_cancelled_repeat_record_count(self.domain, self._id) def _format_or_default_format(self): from corehq.motech.repeaters.repeater_generators import RegisterGenerator return self.format or RegisterGenerator.default_format_by_repeater( self.__class__) def _get_payload_generator(self, payload_format): from corehq.motech.repeaters.repeater_generators import RegisterGenerator gen = RegisterGenerator.generator_class_by_repeater_format( self.__class__, payload_format) return gen(self) @property @memoized def generator(self): return self._get_payload_generator(self._format_or_default_format()) def payload_doc(self, repeat_record): raise NotImplementedError @memoized def get_payload(self, repeat_record): return self.generator.get_payload(repeat_record, self.payload_doc(repeat_record)) def get_attempt_info(self, repeat_record): return None def register(self, payload, next_check=None): if not self.allowed_to_forward(payload): return now = datetime.utcnow() repeat_record = RepeatRecord(repeater_id=self.get_id, repeater_type=self.doc_type, domain=self.domain, registered_on=now, next_check=next_check or now, payload_id=payload.get_id) repeat_record.save() if next_check is None: repeat_record.attempt_forward_now() return repeat_record def allowed_to_forward(self, payload): """ Return True/False depending on whether the payload meets forawrding criteria or not """ return True def clear_caches(self): super(Repeater, self).clear_caches() # Also expire for cases repeater is fetched using Repeater class. # The quick cache called in clear_cache also check on relies of doc class # so in case the class is set as Repeater it is not expired like in edit forms. # So expire it explicitly here with Repeater class as well. Repeater.get.clear(Repeater, self._id) if self.__class__ == Repeater: cls = self.get_class_from_doc_type(self.doc_type) else: cls = self.__class__ # clear cls.by_domain (i.e. filtered by doc type) Repeater.by_domain.clear(cls, self.domain) # clear Repeater.by_domain (i.e. not filtered by doc type) Repeater.by_domain.clear(Repeater, self.domain) @classmethod @quickcache(['cls.__name__', 'domain'], timeout=5 * 60, memoize_timeout=10) def by_domain(cls, domain): key = [domain] if cls.__name__ in get_all_repeater_types(): key.append(cls.__name__) elif cls.__name__ == Repeater.__name__: # In this case the wrap function delegates to the # appropriate sub-repeater types. pass else: # Any repeater type can be posted to the API, and the installed apps # determine whether we actually know about it. # But if we do not know about it, then may as well return nothing now return [] raw_docs = cls.view('repeaters/repeaters', startkey=key, endkey=key + [{}], include_docs=True, reduce=False, wrap_doc=False) return [ cls.wrap(repeater_doc['doc']) for repeater_doc in raw_docs if cls.get_class_from_doc_type(repeater_doc['doc']['doc_type']) ] @classmethod def wrap(cls, data): data.pop('name', None) if cls.__name__ == Repeater.__name__: cls_ = cls.get_class_from_doc_type(data['doc_type']) if cls_: return cls_.wrap(data) else: raise ResourceNotFound('Unknown repeater type: %s' % data) else: return super(Repeater, cls).wrap(data) @staticmethod def get_class_from_doc_type(doc_type): doc_type = doc_type.replace(DELETED_SUFFIX, '') repeater_types = get_all_repeater_types() if doc_type in repeater_types: return repeater_types[doc_type] else: return None def retire(self): if DELETED_SUFFIX not in self['doc_type']: self['doc_type'] += DELETED_SUFFIX if DELETED_SUFFIX not in self['base_doc']: self['base_doc'] += DELETED_SUFFIX self.paused = False self.save() def pause(self): self.paused = True self.save() def resume(self): self.paused = False self.save() def get_url(self, repeat_record): # to be overridden return self.connection_settings.url def allow_retries(self, response): """Whether to requeue the repeater when it fails """ return True def get_headers(self, repeat_record): # to be overridden return self.generator.get_headers() @property def plaintext_password(self): if self.password is None: return '' if self.password.startswith('${algo}$'.format(algo=ALGO_AES)): ciphertext = self.password.split('$', 2)[2] return b64_aes_decrypt(ciphertext) return self.password @property def verify(self): return not self.skip_cert_verify def send_request(self, repeat_record, payload): url = self.get_url(repeat_record) return simple_post( self.domain, url, payload, headers=self.get_headers(repeat_record), auth_manager=self.connection_settings.get_auth_manager(), verify=self.verify, notify_addresses=self.connection_settings.notify_addresses, payload_id=repeat_record.payload_id, ) def fire_for_record(self, repeat_record): payload = self.get_payload(repeat_record) try: response = self.send_request(repeat_record, payload) except (Timeout, ConnectionError) as error: log_repeater_timeout_in_datadog(self.domain) return self.handle_response(RequestConnectionError(error), repeat_record) except Exception as e: return self.handle_response(e, repeat_record) else: return self.handle_response(response, repeat_record) def handle_response(self, result, repeat_record): """ route the result to the success, failure, or exception handlers result may be either a response object or an exception """ if isinstance(result, Exception): attempt = repeat_record.handle_exception(result) elif _is_response( result) and 200 <= result.status_code < 300 or result is True: attempt = repeat_record.handle_success(result) else: attempt = repeat_record.handle_failure(result) return attempt @property def form_class_name(self): """ Return the name of the class whose edit form this class uses. (Most classes that extend CaseRepeater, and all classes that extend FormRepeater, use the same form.) """ return self.__class__.__name__ def create_connection_settings(self): if self.connection_settings_id: return # Nothing to do conn = ConnectionSettings( domain=self.domain, name=self.url, url=self.url, auth_type=self.auth_type, username=self.username, skip_cert_verify=self.skip_cert_verify, notify_addresses_str=self.notify_addresses_str, ) # Allow ConnectionSettings to encrypt old Repeater passwords: conn.plaintext_password = self.plaintext_password conn.save() self.connection_settings_id = conn.id self.save() return conn
class CommCareMultimedia(BlobMixin, SafeSaveDocument): """ The base object of all CommCare Multimedia """ file_hash = StringProperty() # use this to search for multimedia in couch aux_media = SchemaListProperty(AuxMedia) last_modified = DateTimeProperty() valid_domains = StringListProperty( ) # A list of domains that uses this file owners = StringListProperty( default=[]) # list of domains that uploaded this file licenses = SchemaListProperty(HQMediaLicense, default=[]) shared_by = StringListProperty( default=[]) # list of domains that can share this file tags = DictProperty(default={}) # dict of string lists _blobdb_type_code = CODES.multimedia @classmethod def get(cls, docid, rev=None, db=None, dynamic_properties=True): # copied and tweaked from the superclass's method if not db: db = cls.get_db() cls._allow_dynamic_properties = dynamic_properties # on cloudant don't get the doc back until all nodes agree # on the copy, to avoid race conditions extras = get_safe_read_kwargs() return db.get(docid, rev=rev, wrapper=cls.wrap, **extras) @property def is_shared(self): return len(self.shared_by) > 0 @property def license(self): return self.licenses[0] if self.licenses else None @retry_resource(3) def update_or_add_license(self, domain, type="", author="", attribution_notes="", org="", should_save=True): for license in self.licenses: if license.domain == domain: license.type = type or license.type license.author = author or license.author license.organization = org or license.organization license.attribution_notes = attribution_notes or license.attribution_notes break else: license = HQMediaLicense(domain=domain, type=type, author=author, attribution_notes=attribution_notes, organization=org) self.licenses.append(license) if should_save: self.save() def url(self): return reverse("hqmedia_download", args=[self.doc_type, self._id]) def attach_data(self, data, original_filename=None, username=None, attachment_id=None, media_meta=None): """ This creates the auxmedia attachment with the downloaded data. """ self.last_modified = datetime.utcnow() if not attachment_id: attachment_id = self.file_hash if not self.blobs or attachment_id not in self.blobs: if not getattr(self, '_id'): # put_attchment blows away existing data, so make sure an id has been assigned # to this guy before we do it. This is the usual path; remote apps are the exception. self.save() self.put_attachment( data, attachment_id, content_type=self.get_mime_type(data, filename=original_filename), domain=SHARED_DOMAIN, ) new_media = AuxMedia() new_media.uploaded_date = datetime.utcnow() new_media.attachment_id = attachment_id new_media.uploaded_filename = original_filename new_media.uploaded_by = username new_media.checksum = self.file_hash if media_meta: new_media.media_meta = media_meta self.aux_media.append(new_media) self.save() return True def add_domain(self, domain, owner=None, **kwargs): if len(self.owners) == 0: # this is intended to simulate migration--if it happens that a media file somehow gets no more owners # (which should be impossible) it will transfer ownership to all copiers... not necessarily a bad thing, # just something to be aware of self.owners = self.valid_domains if owner and domain not in self.owners: self.owners.append(domain) elif not owner and domain in self.owners: self.owners.remove(domain) if domain in self.owners: shared = kwargs.get('shared', '') if shared and domain not in self.shared_by: self.shared_by.append(domain) elif not shared and shared != '' and domain in self.shared_by: self.shared_by.remove(domain) if kwargs.get('tags', ''): self.tags[domain] = kwargs['tags'] if domain not in self.valid_domains: self.valid_domains.append(domain) self.save() def get_display_file(self, return_type=True): if self.attachment_id: data = self.fetch_attachment(self.attachment_id, stream=True).read() if return_type: content_type = self.blobs[self.attachment_id].content_type return data, content_type else: return data return None def get_file_extension(self): extension = '' if self.aux_media: extension = '.{}'.format( self.aux_media[-1]['uploaded_filename'].split(".")[-1]) return extension def get_media_info(self, path, is_updated=False, original_path=None): return { "path": path, "uid": self.file_hash, "m_id": self._id, "url": reverse("hqmedia_download", args=[self.__class__.__name__, self._id]), "updated": is_updated, "original_path": original_path, "icon_class": self.get_icon_class(), "media_type": self.get_nice_name(), "humanized_content_length": filesizeformat(self.content_length), } @property def attachment_id(self): if not self.aux_media: return None ids = set([aux.attachment_id for aux in self.aux_media]) assert len(ids) == 1 return ids.pop() @property def content_length(self): if self.attachment_id: return self.blobs[self.attachment_id].content_length @classmethod def get_mime_type(cls, data, filename=None): mime = magic.Magic(mime=True) mime_type = mime.from_buffer(data) if mime_type.startswith('application') and filename is not None: guessed_type = mimetypes.guess_type(filename) mime_type = guessed_type[0] if guessed_type[0] else mime_type return mime_type @classmethod def get_base_mime_type(cls, data, filename=None): mime_type = cls.get_mime_type(data, filename=filename) return mime_type.split('/')[0] if mime_type else None @classmethod def generate_hash(cls, data): return hashlib.md5(data).hexdigest() @classmethod def get_by_hash(cls, file_hash): result = cls.view('hqmedia/by_hash', key=file_hash, include_docs=True).first() if not result: result = cls() result.file_hash = file_hash return result @classmethod def get_by_data(cls, data): file_hash = cls.generate_hash(data) return cls.get_by_hash(file_hash) @classmethod def get_doc_types(cls): return ('CommCareImage', 'CommCareAudio', 'CommCareVideo') @classmethod def get_doc_class(cls, doc_type): return { 'CommCareImage': CommCareImage, 'CommCareAudio': CommCareAudio, 'CommCareVideo': CommCareVideo, 'CommCareMultimedia': CommCareMultimedia, }[doc_type] @classmethod def get_class_by_data(cls, data, filename=None): return { 'image': CommCareImage, 'audio': CommCareAudio, 'video': CommCareVideo, }.get(cls.get_base_mime_type(data, filename=filename)) @classmethod def get_form_path(cls, path, lowercase=False): path = path.strip() if lowercase: path = path.lower() if path.startswith(MULTIMEDIA_PREFIX): return path if path.startswith('/'): path = path[1:] return "%s%s" % (MULTIMEDIA_PREFIX, path) @classmethod def get_standard_path(cls, path): return path.replace(MULTIMEDIA_PREFIX, "") @classmethod def wrap(cls, data): should_save = False if data.get('tags') == []: data['tags'] = {} if not data.get('owners'): data['owners'] = data.get('valid_domains', []) if isinstance(data.get('licenses', ''), dict): # need to migrate licncses from old format to new format # old: {"mydomain": "public", "yourdomain": "cc"} migrated = [HQMediaLicense(domain=domain, type=type)._doc \ for domain, type in data["licenses"].items()] data['licenses'] = migrated # deprecating support for public domain license if isinstance(data.get("licenses", ""), list) and len(data["licenses"]) > 0: if data["licenses"][0].get("type", "") == "public": data["licenses"][0]["type"] = "cc" should_save = True self = super(CommCareMultimedia, cls).wrap(data) if should_save: self.save() return self @classmethod def get_nice_name(cls): return _("Generic Multimedia") @classmethod def get_icon_class(cls): return "fa fa-desktop"
class Product(Document): """ A product, e.g. "coartem" or "tylenol" """ domain = StringProperty() name = StringProperty() unit = StringProperty() code_ = StringProperty() # todo: why the hell is this code_ and not code description = StringProperty() category = StringProperty() program_id = StringProperty() cost = DecimalProperty() product_data = DictProperty() is_archived = BooleanProperty(default=False) last_modified = DateTimeProperty() @classmethod def wrap(cls, data): from corehq.apps.groups.models import dt_no_Z_re # If "Z" is missing because of the Aug 2014 migration, then add it. # cf. Group class last_modified = data.get('last_modified') if last_modified and dt_no_Z_re.match(last_modified): data['last_modified'] += 'Z' return super(Product, cls).wrap(data) @classmethod def save_docs(cls, docs, use_uuids=True, all_or_nothing=False, codes_by_domain=None): from corehq.apps.commtrack.util import generate_code codes_by_domain = codes_by_domain or {} def get_codes(domain): if domain not in codes_by_domain: codes_by_domain[domain] = SQLProduct.objects.filter(domain=domain)\ .values_list('code', flat=True).distinct() return codes_by_domain[domain] for doc in docs: if not doc['code_']: doc['code_'] = generate_code(doc['name'], get_codes(doc['domain'])) super(Product, cls).save_docs(docs, use_uuids, all_or_nothing) bulk_save = save_docs def sync_to_sql(self): properties_to_sync = [ ('product_id', '_id'), 'domain', 'name', 'is_archived', ('code', 'code_'), 'description', 'category', 'program_id', 'cost', ('units', 'unit'), 'product_data', ] # sync properties to SQL version sql_product, _ = SQLProduct.objects.get_or_create(product_id=self._id) for prop in properties_to_sync: if isinstance(prop, tuple): sql_prop, couch_prop = prop else: sql_prop = couch_prop = prop if hasattr(self, couch_prop): setattr(sql_product, sql_prop, getattr(self, couch_prop)) sql_product.save() def save(self, *args, **kwargs): """ Saving a couch version of Product will trigger one way syncing to the SQLProduct version of this product. """ # mark modified time stamp for selective syncing self.last_modified = datetime.utcnow() # generate code if user didn't specify one if not self.code: from corehq.apps.commtrack.util import generate_code self.code = generate_code( self.name, SQLProduct.objects.filter(domain=self.domain).values_list( 'code', flat=True).distinct()) result = super(Product, self).save(*args, **kwargs) self.sync_to_sql() return result @property def code(self): return self.code_ @code.setter def code(self, val): self.code_ = val.lower() if val else None @classmethod def get_by_code(cls, domain, code): if not code: return None result = cls.view("commtrack/product_by_code", key=[domain, code.lower()], include_docs=True).first() return result @classmethod def by_program_id(cls, domain, prog_id, wrap=True, **kwargs): kwargs.update( dict(view_name='commtrack/product_by_program_id', key=[domain, prog_id], include_docs=True)) if wrap: return Product.view(**kwargs) else: return [ row["doc"] for row in Product.view(wrap_doc=False, **kwargs) ] @classmethod def by_domain(cls, domain, wrap=True, include_archived=False, **kwargs): """ Gets all products in a domain. By default this filters out any archived products. WARNING: this doesn't paginate correctly; it filters after the query If you need pagination, use SQLProduct instead """ kwargs.update( dict(view_name='commtrack/products', startkey=[domain], endkey=[domain, {}], include_docs=True)) if wrap: products = Product.view(**kwargs) if not include_archived: return filter(lambda p: not p.is_archived, products) else: return products else: if not include_archived: return [ row["doc"] for row in Product.view(wrap_doc=False, **kwargs) if not row["doc"].get('is_archived', False) ] else: return [ row["doc"] for row in Product.view(wrap_doc=False, **kwargs) ] @classmethod def archived_by_domain(cls, domain, wrap=True, **kwargs): products = cls.by_domain(domain, wrap, kwargs) if wrap: return filter(lambda p: p.is_archived, products) else: return [p for p in products if p.get('is_archived', False)] @classmethod def ids_by_domain(cls, domain): """ Gets all product ids in a domain. """ view_results = Product.get_db().view( 'commtrack/products', startkey=[domain], endkey=[domain, {}], include_docs=False, ) return [row['id'] for row in view_results] @classmethod def count_by_domain(cls, domain): """ Gets count of products in a domain """ # todo: we should add a reduce so we can get this out of couch return len(cls.ids_by_domain(domain)) @classmethod def _export_attrs(cls): return [ ('name', unicode), ('unit', unicode), 'description', 'category', ('program_id', str), ('cost', lambda a: Decimal(a) if a else None), ] def to_dict(self): from corehq.apps.commtrack.util import encode_if_needed product_dict = {} product_dict['id'] = self._id product_dict['product_id'] = self.code_ for attr in self._export_attrs(): real_attr = attr[0] if isinstance(attr, tuple) else attr product_dict[real_attr] = encode_if_needed(getattr( self, real_attr)) return product_dict def custom_property_dict(self): from corehq.apps.commtrack.util import encode_if_needed property_dict = {} for prop, val in self.product_data.iteritems(): property_dict['data: ' + prop] = encode_if_needed(val) return property_dict def archive(self): """ Mark a product as archived. This will cause it (and its data) to not show up in default Couch and SQL views. """ self.is_archived = True self.save() def unarchive(self): """ Unarchive a product, causing it (and its data) to show up in Couch and SQL views again. """ if self.code: if SQLProduct.objects.filter(code=self.code, is_archived=False).exists(): raise DuplicateProductCodeException() self.is_archived = False self.save() @classmethod def from_excel(cls, row, custom_data_validator): if not row: return None id = row.get('id') if id: try: p = cls.get(id) except ResourceNotFound: raise InvalidProductException( _("Product with ID '{product_id}' could not be found!"). format(product_id=id)) else: p = cls() p.code = str(row.get('product_id') or '') for attr in cls._export_attrs(): key = attr[0] if isinstance(attr, tuple) else attr if key in row: val = row[key] if val is None: val = '' if isinstance(attr, tuple): val = attr[1](val) setattr(p, key, val) else: break if not p.code: raise InvalidProductException( _('Product ID is a required field and cannot be blank!')) if not p.name: raise InvalidProductException( _('Product name is a required field and cannot be blank!')) custom_data = row.get('data', {}) error = custom_data_validator(custom_data) if error: raise InvalidProductException(error) p.product_data = custom_data p.product_data.update(row.get('uncategorized_data', {})) return p
class RepeatRecord(Document): """ An record of a particular instance of something that needs to be forwarded with a link to the proper repeater object """ domain = StringProperty() repeater_id = StringProperty() repeater_type = StringProperty() payload_id = StringProperty() overall_tries = IntegerProperty(default=0) max_possible_tries = IntegerProperty(default=3) attempts = ListProperty(RepeatRecordAttempt) cancelled = BooleanProperty(default=False) registered_on = DateTimeProperty() last_checked = DateTimeProperty() failure_reason = StringProperty() next_check = DateTimeProperty() succeeded = BooleanProperty(default=False) @property def record_id(self): return self._id @classmethod def wrap(cls, data): should_bootstrap_attempts = ('attempts' not in data) self = super(RepeatRecord, cls).wrap(data) if should_bootstrap_attempts and self.last_checked: assert not self.attempts self.attempts = [ RepeatRecordAttempt( cancelled=self.cancelled, datetime=self.last_checked, failure_reason=self.failure_reason, success_response=None, next_check=self.next_check, succeeded=self.succeeded, ) ] return self @property @memoized def repeater(self): try: return Repeater.get(self.repeater_id) except ResourceNotFound: return None @property def url(self): warnings.warn( "RepeatRecord.url is deprecated. Use Repeater.get_url instead", DeprecationWarning) if self.repeater: return self.repeater.get_url(self) @property def state(self): state = RECORD_PENDING_STATE if self.succeeded: state = RECORD_SUCCESS_STATE elif self.cancelled: state = RECORD_CANCELLED_STATE elif self.failure_reason: state = RECORD_FAILURE_STATE return state @classmethod def all(cls, domain=None, due_before=None, limit=None): json_now = json_format_datetime(due_before or datetime.utcnow()) repeat_records = RepeatRecord.view( "repeaters/repeat_records_by_next_check", startkey=[domain], endkey=[domain, json_now, {}], include_docs=True, reduce=False, limit=limit, ) return repeat_records @classmethod def count(cls, domain=None): results = RepeatRecord.view( "repeaters/repeat_records_by_next_check", startkey=[domain], endkey=[domain, {}], reduce=True, ).one() return results['value'] if results else 0 def add_attempt(self, attempt): self.attempts.append(attempt) self.last_checked = attempt.datetime self.next_check = attempt.next_check self.succeeded = attempt.succeeded self.cancelled = attempt.cancelled self.failure_reason = attempt.failure_reason def get_numbered_attempts(self): for i, attempt in enumerate(self.attempts): yield i + 1, attempt def postpone_by(self, duration): self.last_checked = datetime.utcnow() self.next_check = self.last_checked + duration self.save() def make_set_next_try_attempt(self, failure_reason): # we use an exponential back-off to avoid submitting to bad urls # too frequently. assert self.succeeded is False assert self.next_check is not None window = timedelta(minutes=0) if self.last_checked: window = self.next_check - self.last_checked window += (window // 2) # window *= 1.5 if window < MIN_RETRY_WAIT: window = MIN_RETRY_WAIT elif window > MAX_RETRY_WAIT: window = MAX_RETRY_WAIT now = datetime.utcnow() return RepeatRecordAttempt( cancelled=False, datetime=now, failure_reason=failure_reason, success_response=None, next_check=now + window, succeeded=False, ) def try_now(self): # try when we haven't succeeded and either we've # never checked, or it's time to check again return not self.succeeded def get_payload(self): return self.repeater.get_payload(self) def get_attempt_info(self): return self.repeater.get_attempt_info(self) def handle_payload_exception(self, exception): now = datetime.utcnow() return RepeatRecordAttempt( cancelled=True, datetime=now, failure_reason=six.text_type(exception), success_response=None, next_check=None, succeeded=False, ) def fire(self, force_send=False): if self.try_now() or force_send: self.overall_tries += 1 try: attempt = self.repeater.fire_for_record(self) except Exception as e: log_repeater_error_in_datadog(self.domain, status_code=None, repeater_type=self.repeater_type) attempt = self.handle_payload_exception(e) raise finally: # pycharm warns attempt might not be defined. # that'll only happen if fire_for_record raise a non-Exception exception (e.g. SIGINT) # or handle_payload_exception raises an exception. I'm okay with that. -DMR self.add_attempt(attempt) self.save() @staticmethod def _format_response(response): return '{}: {}.\n{}'.format(response.status_code, response.reason, getattr(response, 'content', None)) def handle_success(self, response): """Do something with the response if the repeater succeeds """ now = datetime.utcnow() log_repeater_success_in_datadog( self.domain, response.status_code if response else None, self.repeater_type) return RepeatRecordAttempt( cancelled=False, datetime=now, failure_reason=None, success_response=self._format_response(response) if response else None, next_check=None, succeeded=True, info=self.get_attempt_info(), ) def handle_failure(self, response): """Do something with the response if the repeater fails """ return self._make_failure_attempt(self._format_response(response), response) def handle_exception(self, exception): """handle internal exceptions """ return self._make_failure_attempt(six.text_type(exception), None) def _make_failure_attempt(self, reason, response): log_repeater_error_in_datadog( self.domain, response.status_code if response else None, self.repeater_type) if self.repeater.allow_retries( response) and self.overall_tries < self.max_possible_tries: return self.make_set_next_try_attempt(reason) else: now = datetime.utcnow() return RepeatRecordAttempt( cancelled=True, datetime=now, failure_reason=reason, success_response=None, next_check=None, succeeded=False, info=self.get_attempt_info(), ) def cancel(self): self.next_check = None self.cancelled = True def requeue(self): self.cancelled = False self.succeeded = False self.failure_reason = '' self.overall_tries = 0 self.next_check = datetime.utcnow()
class Domain(QuickCachedDocumentMixin, BlobMixin, Document, SnapshotMixin): """ Domain is the highest level collection of people/stuff in the system. Pretty much everything happens at the domain-level, including user membership, permission to see data, reports, charts, etc. Exceptions: accounting has some models that combine multiple domains, which make "enterprise" multi-domain features like the enterprise dashboard possible. Naming conventions: Most often, variables representing domain names are named `domain`, and variables representing domain objects are named `domain_obj`. New code should follow this convention, unless it's in an area that consistently uses `domain` for the object and `domain_name` for the string. There's a `project` attribute attached to requests that's a domain object. In spite of this, don't use `project` in new code. """ _blobdb_type_code = BLOB_CODES.domain name = StringProperty() is_active = BooleanProperty() date_created = DateTimeProperty() default_timezone = StringProperty( default=getattr(settings, "TIME_ZONE", "UTC")) case_sharing = BooleanProperty(default=False) secure_submissions = BooleanProperty(default=False) cloudcare_releases = StringProperty( choices=['stars', 'nostars', 'default'], default='default') organization = StringProperty() hr_name = StringProperty() # the human-readable name for this project project_description = StringProperty() # Brief description of the project creating_user = StringProperty( ) # username of the user who created this domain # domain metadata project_type = StringProperty() # e.g. MCH, HIV customer_type = StringProperty() # plus, full, etc. is_test = StringProperty(choices=["true", "false", "none"], default="none") description = StringProperty() short_description = StringProperty() is_shared = BooleanProperty(default=False) commtrack_enabled = BooleanProperty(default=False) call_center_config = SchemaProperty(CallCenterProperties) restrict_superusers = BooleanProperty(default=False) allow_domain_requests = BooleanProperty(default=False) location_restriction_for_users = BooleanProperty(default=False) usercase_enabled = BooleanProperty(default=False) hipaa_compliant = BooleanProperty(default=False) use_sql_backend = BooleanProperty(default=False) first_domain_for_user = BooleanProperty(default=False) case_display = SchemaProperty(CaseDisplaySettings) # CommConnect settings commconnect_enabled = BooleanProperty(default=False) survey_management_enabled = BooleanProperty(default=False) # Whether or not a case can register via sms sms_case_registration_enabled = BooleanProperty(default=False) # Case type to apply to cases registered via sms sms_case_registration_type = StringProperty() # Owner to apply to cases registered via sms sms_case_registration_owner_id = StringProperty() # Submitting user to apply to cases registered via sms sms_case_registration_user_id = StringProperty() # Whether or not a mobile worker can register via sms sms_mobile_worker_registration_enabled = BooleanProperty(default=False) use_default_sms_response = BooleanProperty(default=False) default_sms_response = StringProperty() chat_message_count_threshold = IntegerProperty() custom_chat_template = StringProperty( ) # See settings.CUSTOM_CHAT_TEMPLATES custom_case_username = StringProperty( ) # Case property to use when showing the case's name in a chat window # If empty, sms can be sent at any time. Otherwise, only send during # these windows of time. SMS_QUEUE_ENABLED must be True in localsettings # for this be considered. restricted_sms_times = SchemaListProperty(DayTimeWindow) # If empty, this is ignored. Otherwise, the framework will make sure # that during these days/times, no automated outbound sms will be sent # to someone if they have sent in an sms within sms_conversation_length # minutes. Outbound sms sent from a user in a chat window, however, will # still be sent. This is meant to prevent chat conversations from being # interrupted by automated sms reminders. # SMS_QUEUE_ENABLED must be True in localsettings for this to be # considered. sms_conversation_times = SchemaListProperty(DayTimeWindow) # In minutes, see above. sms_conversation_length = IntegerProperty(default=10) # Set to True to prevent survey questions and answers form being seen in # SMS chat windows. filter_surveys_from_chat = BooleanProperty(default=False) # The below option only matters if filter_surveys_from_chat = True. # If set to True, invalid survey responses will still be shown in the chat # window, while questions and valid responses will be filtered out. show_invalid_survey_responses_in_chat = BooleanProperty(default=False) # If set to True, if a message is read by anyone it counts as being read by # everyone. Set to False so that a message is only counted as being read # for a user if only that user has read it. count_messages_as_read_by_anyone = BooleanProperty(default=False) enable_registration_welcome_sms_for_case = BooleanProperty(default=False) enable_registration_welcome_sms_for_mobile_worker = BooleanProperty( default=False) sms_survey_date_format = StringProperty() granted_messaging_access = BooleanProperty(default=False) # Allowed outbound SMS per day # If this is None, then the default is applied. See get_daily_outbound_sms_limit() custom_daily_outbound_sms_limit = IntegerProperty() # Allowed number of case updates or closes from automatic update rules in the daily rule run. # If this value is None, the value in settings.MAX_RULE_UPDATES_IN_ONE_RUN is used. auto_case_update_limit = IntegerProperty() # Allowed number of max OData feeds that this domain can create. # If this value is None, the value in settings.DEFAULT_ODATA_FEED_LIMIT is used odata_feed_limit = IntegerProperty() # exchange/domain copying stuff is_snapshot = BooleanProperty(default=False) is_approved = BooleanProperty(default=False) snapshot_time = DateTimeProperty() published = BooleanProperty(default=False) license = StringProperty(choices=LICENSES, default='cc') title = StringProperty() cda = SchemaProperty(LicenseAgreement) multimedia_included = BooleanProperty(default=True) downloads = IntegerProperty( default=0) # number of downloads for this specific snapshot full_downloads = IntegerProperty( default=0) # number of downloads for all snapshots from this domain author = StringProperty() phone_model = StringProperty() attribution_notes = StringProperty() publisher = StringProperty(choices=["organization", "user"], default="user") yt_id = StringProperty() snapshot_head = BooleanProperty(default=False) deployment = SchemaProperty(Deployment) image_path = StringProperty() image_type = StringProperty() cached_properties = DictProperty() internal = SchemaProperty(InternalProperties) dynamic_reports = SchemaListProperty(DynamicReportSet) # extra user specified properties tags = StringListProperty() area = StringProperty(choices=AREA_CHOICES) sub_area = StringProperty(choices=SUB_AREA_CHOICES) launch_date = DateTimeProperty last_modified = DateTimeProperty(default=datetime(2015, 1, 1)) # when turned on, use SECURE_TIMEOUT for sessions of users who are members of this domain secure_sessions = BooleanProperty(default=False) two_factor_auth = BooleanProperty(default=False) strong_mobile_passwords = BooleanProperty(default=False) # There is no longer a way to request a report builder trial, so this property should be removed in the near # future. (Keeping it for now in case a user has requested a trial and but has not yet been granted it) requested_report_builder_trial = StringListProperty() requested_report_builder_subscription = StringListProperty() report_whitelist = StringListProperty() # seconds between sending mobile UCRs to users. Can be overridden per user default_mobile_ucr_sync_interval = IntegerProperty() @classmethod def wrap(cls, data): # for domains that still use original_doc should_save = False if 'original_doc' in data: original_doc = data['original_doc'] del data['original_doc'] should_save = True if original_doc: original_doc = Domain.get_by_name(original_doc) data['copy_history'] = [original_doc._id] # for domains that have a public domain license if 'license' in data: if data.get("license", None) == "public": data["license"] = "cc" should_save = True if 'slug' in data and data["slug"]: data["hr_name"] = data["slug"] del data["slug"] if 'is_test' in data and isinstance(data["is_test"], bool): data["is_test"] = "true" if data["is_test"] else "false" should_save = True if 'cloudcare_releases' not in data: data['cloudcare_releases'] = 'nostars' # legacy default setting # Don't actually remove location_types yet. We can migrate fully and # remove this after everything's hunky-dory in production. 2015-03-06 if 'location_types' in data: data['obsolete_location_types'] = data.pop('location_types') if 'granted_messaging_access' not in data: # enable messaging for domains created before this flag was added data['granted_messaging_access'] = True self = super(Domain, cls).wrap(data) if self.deployment is None: self.deployment = Deployment() if should_save: self.save() return self def get_default_timezone(self): """return a timezone object from self.default_timezone""" import pytz return pytz.timezone(self.default_timezone) @staticmethod @quickcache(['name'], timeout=24 * 60 * 60) def is_secure_session_required(name): domain_obj = Domain.get_by_name(name) return domain_obj and domain_obj.secure_sessions @staticmethod @quickcache(['couch_user._id', 'is_active'], timeout=5 * 60, memoize_timeout=10) def active_for_couch_user(couch_user, is_active=True): domain_names = couch_user.get_domains() return Domain.view( "domain/by_status", keys=[[is_active, d] for d in domain_names], reduce=False, include_docs=True, ).all() @staticmethod def active_for_user(user, is_active=True): if isinstance(user, AnonymousUser): return [] from corehq.apps.users.models import CouchUser if isinstance(user, CouchUser): couch_user = user else: couch_user = CouchUser.from_django_user(user) if couch_user: return Domain.active_for_couch_user(couch_user, is_active=is_active) else: return [] def add(self, model_instance, is_active=True): """ Add something to this domain, through the generic relation. Returns the created membership object """ # Add membership info to Couch couch_user = model_instance.get_profile().get_couch_user() couch_user.add_domain_membership(self.name) couch_user.save() def applications(self): return get_brief_apps_in_domain(self.name) def full_applications(self, include_builds=True): from corehq.apps.app_manager.util import get_correct_app_class from corehq.apps.app_manager.models import Application def wrap_application(a): return get_correct_app_class(a['doc']).wrap(a['doc']) if include_builds: startkey = [self.name] endkey = [self.name, {}] else: startkey = [self.name, None] endkey = [self.name, None, {}] return Application.get_db().view('app_manager/applications', startkey=startkey, endkey=endkey, include_docs=True, wrapper=wrap_application).all() @cached_property def versions(self): apps = self.applications() return list(set(a.application_version for a in apps)) @cached_property def has_media(self): from corehq.apps.app_manager.util import is_remote_app for app in self.full_applications(): if not is_remote_app(app) and app.has_media(): return True return False @property def use_cloudcare_releases(self): return self.cloudcare_releases != 'nostars' def all_users(self): from corehq.apps.users.models import CouchUser return CouchUser.by_domain(self.name) def recent_submissions(self): return domain_has_submission_in_last_30_days(self.name) @cached_property def languages(self): apps = self.applications() return set(chain.from_iterable([a.langs for a in apps])) @classmethod @quickcache(['name'], skip_arg='strict', timeout=30 * 60, session_function=icds_conditional_session_key()) def get_by_name(cls, name, strict=False): if not name: # get_by_name should never be called with name as None (or '', etc) # I fixed the code in such a way that if I raise a ValueError # all tests pass and basic pages load, # but in order not to break anything in the wild, # I'm opting to notify by email if/when this happens # but fall back to the previous behavior of returning None if settings.DEBUG: raise ValueError('%r is not a valid domain name' % name) else: _assert = soft_assert(notify_admins=True, exponential_backoff=False) _assert(False, '%r is not a valid domain name' % name) return None def _get_by_name(stale=False): extra_args = {'stale': settings.COUCH_STALE_QUERY} if stale else {} result = cls.view("domain/domains", key=name, reduce=False, include_docs=True, **extra_args).first() if not isinstance(result, Domain): # A stale view may return a result with no doc if the doc has just been deleted. # In this case couchdbkit just returns the raw view result as a dict return None else: return result domain = _get_by_name(stale=(not strict)) if domain is None and not strict: # on the off chance this is a brand new domain, try with strict domain = _get_by_name(stale=False) return domain @classmethod def get_or_create_with_name(cls, name, is_active=False, secure_submissions=True, use_sql_backend=False): result = cls.view("domain/domains", key=name, reduce=False, include_docs=True).first() if result: return result else: new_domain = Domain( name=name, is_active=is_active, date_created=datetime.utcnow(), secure_submissions=secure_submissions, use_sql_backend=use_sql_backend, ) new_domain.save(**get_safe_write_kwargs()) return new_domain @classmethod def generate_name(cls, hr_name, max_length=25): ''' Generate a URL-friendly name based on a given human-readable name. Normalizes given name, then looks for conflicting domains, addressing conflicts by adding "-1", "-2", etc. May return None if it fails to generate a new, unique name. Throws exception if it can't figure out a name, which shouldn't happen unless max_length is absurdly short. ''' from corehq.apps.domain.utils import get_domain_url_slug from corehq.apps.domain.dbaccessors import domain_or_deleted_domain_exists name = get_domain_url_slug(hr_name, max_length=max_length) if not name: raise NameUnavailableException if domain_or_deleted_domain_exists(name): prefix = name while len(prefix): name = next_available_name( prefix, Domain.get_names_by_prefix(prefix + '-')) if domain_or_deleted_domain_exists(name): # should never happen raise NameUnavailableException if len(name) <= max_length: return name prefix = prefix[:-1] raise NameUnavailableException return name @classmethod def get_all(cls, include_docs=True): domains = Domain.view("domain/not_snapshots", include_docs=False).all() if not include_docs: return domains else: return map(cls.wrap, iter_docs(cls.get_db(), [d['id'] for d in domains])) @classmethod def get_all_names(cls): return sorted({d['key'] for d in cls.get_all(include_docs=False)}) @classmethod def get_all_ids(cls): return [d['id'] for d in cls.get_all(include_docs=False)] @classmethod def get_names_by_prefix(cls, prefix): return [ d['key'] for d in Domain.view("domain/domains", startkey=prefix, endkey=prefix + "zzz", reduce=False, include_docs=False).all() ] + [ d['key'] for d in Domain.view("domain/deleted_domains", startkey=prefix, endkey=prefix + "zzz", reduce=False, include_docs=False).all() ] def case_sharing_included(self): return self.case_sharing or reduce(lambda x, y: x or y, [ getattr(app, 'case_sharing', False) for app in self.applications() ], False) def save(self, **params): from corehq.apps.domain.dbaccessors import domain_or_deleted_domain_exists self.last_modified = datetime.utcnow() if not self._rev: if domain_or_deleted_domain_exists(self.name): raise NameUnavailableException(self.name) # mark any new domain as timezone migration complete set_tz_migration_complete(self.name) super(Domain, self).save(**params) from corehq.apps.domain.signals import commcare_domain_post_save results = commcare_domain_post_save.send_robust(sender='domain', domain=self) log_signal_errors(results, "Error occurred during domain post_save (%s)", {'domain': self.name}) def snapshots(self, **view_kwargs): return Domain.view('domain/snapshots', startkey=[self._id, {}], endkey=[self._id], include_docs=True, reduce=False, descending=True, **view_kwargs) def update_deployment(self, **kwargs): self.deployment.update(kwargs) self.save() def update_internal(self, **kwargs): self.internal.update(kwargs) self.save() def display_name(self): if self.is_snapshot: return "Snapshot of %s" % self.copied_from.display_name() return self.hr_name or self.name def long_display_name(self): if self.is_snapshot: return format_html("Snapshot of {}", self.copied_from.display_name()) return self.hr_name or self.name __str__ = long_display_name def get_license_display(self): return LICENSES.get(self.license) def get_license_url(self): return LICENSE_LINKS.get(self.license) def copies(self): return Domain.view('domain/copied_from_snapshot', key=self._id, include_docs=True) def copies_of_parent(self): return Domain.view('domain/copied_from_snapshot', keys=[s._id for s in self.copied_from.snapshots()], include_docs=True) def delete(self, leave_tombstone=False): if not leave_tombstone and not settings.UNIT_TESTING: raise ValueError( 'Cannot delete domain without leaving a tombstone except during testing' ) self._pre_delete() if leave_tombstone: domain = self.get(self._id) if not domain.doc_type.endswith('-Deleted'): domain.doc_type = '{}-Deleted'.format(domain.doc_type) domain.save() else: super().delete() # The save signals can undo effect of clearing the cache within the save # because they query the stale view (but attaches the up to date doc). # This is only a problem on delete/soft-delete, # because these change the presence in the index, not just the doc content. # Since this is rare, I'm opting to just re-clear the cache here # rather than making the signals use a strict lookup or something like that. self.clear_caches() def _pre_delete(self): from corehq.apps.domain.deletion import apply_deletion_operations # delete SQL models first because UCR tables are indexed by configs in couch apply_deletion_operations(self.name) # delete couch docs for db, related_doc_ids in get_all_doc_ids_for_domain_grouped_by_db( self.name): iter_bulk_delete(db, related_doc_ids, chunksize=500) @classmethod def get_module_by_name(cls, domain_name): """ import and return the python module corresponding to domain_name, or None if it doesn't exist. """ module_name = settings.DOMAIN_MODULE_MAP.get(domain_name, domain_name) try: return import_module(module_name) if module_name else None except ImportError: return None @property @memoized def commtrack_settings(self): # this import causes some dependency issues so lives in here from corehq.apps.commtrack.models import CommtrackConfig if self.commtrack_enabled: return CommtrackConfig.for_domain(self.name) else: return None @property def has_custom_logo(self): return self.has_attachment(LOGO_ATTACHMENT) def get_custom_logo(self): if not self.has_custom_logo: return None return (self.fetch_attachment(LOGO_ATTACHMENT), self.blobs[LOGO_ATTACHMENT].content_type) def put_attachment(self, *args, **kw): return super(Domain, self).put_attachment(domain=self.name, *args, **kw) def get_case_display(self, case): """Get the properties display definition for a given case""" return self.case_display.case_details.get(case.type) def get_form_display(self, form): """Get the properties display definition for a given XFormInstance""" return self.case_display.form_details.get(form.xmlns) @property def location_types(self): from corehq.apps.locations.models import LocationType return LocationType.objects.filter(domain=self.name).all() @memoized def has_privilege(self, privilege): from corehq.apps.accounting.utils import domain_has_privilege return domain_has_privilege(self, privilege) @property @memoized def uses_locations(self): from corehq import privileges from corehq.apps.locations.models import LocationType return (self.has_privilege(privileges.LOCATIONS) and (self.commtrack_enabled or LocationType.objects.filter(domain=self.name).exists())) def convert_to_commtrack(self): """ One-stop-shop to make a domain CommTrack """ from corehq.apps.commtrack.util import make_domain_commtrack make_domain_commtrack(self) def clear_caches(self): from .utils import domain_restricts_superusers super(Domain, self).clear_caches() self.get_by_name.clear(self.__class__, self.name) self.is_secure_session_required.clear(self.name) domain_restricts_superusers.clear(self.name) def get_daily_outbound_sms_limit(self): if self.custom_daily_outbound_sms_limit: return self.custom_daily_outbound_sms_limit # https://manage.dimagi.com/default.asp?274299 return 50000
class RegistrationRequest(Document): tos_confirmed = BooleanProperty(default=False) request_time = DateTimeProperty() request_ip = StringProperty() activation_guid = StringProperty() confirm_time = DateTimeProperty() confirm_ip = StringProperty() domain = StringProperty() new_user_username = StringProperty() requesting_user_username = StringProperty() @property @memoized def project(self): return Domain.get_by_name(self.domain) @classmethod def get_by_guid(cls, guid): result = cls.view("registration/requests_by_guid", key=guid, reduce=False, include_docs=True).first() return result @classmethod def get_requests_today(cls): today = datetime.datetime.utcnow() yesterday = today - datetime.timedelta(1) result = cls.view("registration/requests_by_time", startkey=yesterday.isoformat(), endkey=today.isoformat(), reduce=True).all() if not result: return 0 return result[0]['value'] @classmethod def get_requests_24hrs_ago(cls): today = datetime.datetime.utcnow() yesterday = today - datetime.timedelta(1) join_on_start = datetime.datetime(yesterday.year, yesterday.month, yesterday.day, yesterday.hour, 0, 0, 0) join_on_end = datetime.datetime(yesterday.year, yesterday.month, yesterday.day, yesterday.hour, 59, 59, 59) result = cls.view("registration/requests_by_time", startkey=join_on_start.isoformat(), endkey=join_on_end.isoformat(), include_docs=True, reduce=False).all() return [ doc for doc in result if (doc.new_user_username == doc.requesting_user_username and doc.confirm_time is None) ] @classmethod def get_request_for_username(cls, username): result = cls.view("registration/requests_by_username", key=username, reduce=False, include_docs=True).first() return result
class AtomFeedStatus(DocumentSchema): last_polled_at = DateTimeProperty(default=None) last_page = StringProperty(default=None)
class Program(Document): """ A program, e.g. "hiv" or "tb" """ domain = StringProperty() name = StringProperty() code = StringProperty() last_modified = DateTimeProperty() default = BooleanProperty(default=False) is_archived = BooleanProperty(default=False) @classmethod def wrap(cls, data): # If "Z" is missing because of the Aug 2014 migration, then add it. # cf. Group class last_modified = data.get('last_modified') if last_modified and dt_no_Z_re.match(last_modified): data['last_modified'] += 'Z' return super(Program, cls).wrap(data) def save(self, *args, **kwargs): self.last_modified = datetime.utcnow() super(Program, self).save(*args, **kwargs) self.clear_caches(self.domain) @classmethod def by_domain(cls, domain, wrap=True): """ Gets all programs in a domain. """ kwargs = dict( view_name='program_by_code/view', startkey=[domain], endkey=[domain, {}], include_docs=True ) if wrap: return Program.view(**kwargs) else: return [row["doc"] for row in Program.view(wrap_doc=False, **kwargs)] @classmethod def default_for_domain(cls, domain): programs = cls.by_domain(domain) for p in programs: if p.default: return p def delete(self): # you cannot delete the default program if self.default: raise Exception(_('You cannot delete the default program')) default = Program.default_for_domain(self.domain) sql_products = SQLProduct.objects.filter(domain=self.domain, program_id=self.get_id) to_save = [] for product in sql_products.couch_products(): product['program_id'] = default._id to_save.append(product) # break up saving in case there are many products if len(to_save) > 500: Product.bulk_save(to_save) to_save = [] Product.bulk_save(to_save) # bulk update sqlproducts sql_products.update(program_id=default._id) super(Program, self).delete() self.clear_caches(self.domain) def unarchive(self): """ Unarchive a program, causing it (and its data) to show up in Couch and SQL views again. """ self.is_archived = False self.save() @classmethod def get_by_code(cls, domain, code): result = cls.view("program_by_code/view", key=[domain, code], include_docs=True, limit=1).first() return result def get_products_count(self): return (SQLProduct.objects .filter(domain=self.domain, program_id=self.get_id) .count()) @classmethod def clear_caches(cls, domain): from casexml.apps.phone.utils import clear_fixture_cache from corehq.apps.programs.fixtures import PROGRAM_FIXTURE_BUCKET clear_fixture_cache(domain, PROGRAM_FIXTURE_BUCKET)
class RegistrationRequest(SyncCouchToSQLMixin, Document, RegistrationRequestMixin): tos_confirmed = BooleanProperty(default=False) request_time = DateTimeProperty() request_ip = StringProperty() activation_guid = StringProperty() confirm_time = DateTimeProperty() confirm_ip = StringProperty() domain = StringProperty() new_user_username = StringProperty() requesting_user_username = StringProperty() @classmethod def get_by_guid(cls, guid): return SQLRegistrationRequest.objects.filter( activation_guid=guid).first() @classmethod def get_requests_today(cls): today = datetime.datetime.utcnow() yesterday = today - datetime.timedelta(1) return SQLRegistrationRequest.objects.filter( request_time__gte=yesterday.isoformat(), request_time__lte=today.isoformat(), ).count() @classmethod def get_requests_24hrs_ago(cls): today = datetime.datetime.utcnow() yesterday = today - datetime.timedelta(1) join_on_start = datetime.datetime(yesterday.year, yesterday.month, yesterday.day, yesterday.hour, 0, 0, 0) join_on_end = datetime.datetime(yesterday.year, yesterday.month, yesterday.day, yesterday.hour, 59, 59, 59) requests = SQLRegistrationRequest.objects.filter( request_time__gte=join_on_start, request_time__lte=join_on_end, confirm_time__isnull=True) return [ req for req in requests if req.new_user_username == req.requesting_user_username ] @classmethod def get_request_for_username(cls, username): return SQLRegistrationRequest.objects.filter( new_user_username=username).first() @classmethod def _migration_get_fields(cls): return [ "tos_confirmed", "request_time", "request_ip", "activation_guid", "confirm_time", "confirm_ip", "domain", "new_user_username", "requesting_user_username", ] @classmethod def _migration_get_sql_model_class(cls): return SQLRegistrationRequest
class Domain(QuickCachedDocumentMixin, Document, SnapshotMixin): """Domain is the highest level collection of people/stuff in the system. Pretty much everything happens at the domain-level, including user membership, permission to see data, reports, charts, etc.""" name = StringProperty() is_active = BooleanProperty() date_created = DateTimeProperty() default_timezone = StringProperty(default=getattr(settings, "TIME_ZONE", "UTC")) case_sharing = BooleanProperty(default=False) secure_submissions = BooleanProperty(default=False) cloudcare_releases = StringProperty(choices=['stars', 'nostars', 'default'], default='default') organization = StringProperty() hr_name = StringProperty() # the human-readable name for this project creating_user = StringProperty() # username of the user who created this domain # domain metadata project_type = StringProperty() # e.g. MCH, HIV customer_type = StringProperty() # plus, full, etc. is_test = StringProperty(choices=["true", "false", "none"], default="none") description = StringProperty() short_description = StringProperty() is_shared = BooleanProperty(default=False) commtrack_enabled = BooleanProperty(default=False) call_center_config = SchemaProperty(CallCenterProperties) has_careplan = BooleanProperty(default=False) restrict_superusers = BooleanProperty(default=False) allow_domain_requests = BooleanProperty(default=False) location_restriction_for_users = BooleanProperty(default=False) usercase_enabled = BooleanProperty(default=False) hipaa_compliant = BooleanProperty(default=False) use_sql_backend = BooleanProperty(default=False) case_display = SchemaProperty(CaseDisplaySettings) # CommConnect settings commconnect_enabled = BooleanProperty(default=False) survey_management_enabled = BooleanProperty(default=False) # Whether or not a case can register via sms sms_case_registration_enabled = BooleanProperty(default=False) # Case type to apply to cases registered via sms sms_case_registration_type = StringProperty() # Owner to apply to cases registered via sms sms_case_registration_owner_id = StringProperty() # Submitting user to apply to cases registered via sms sms_case_registration_user_id = StringProperty() # Whether or not a mobile worker can register via sms sms_mobile_worker_registration_enabled = BooleanProperty(default=False) use_default_sms_response = BooleanProperty(default=False) default_sms_response = StringProperty() chat_message_count_threshold = IntegerProperty() custom_chat_template = StringProperty() # See settings.CUSTOM_CHAT_TEMPLATES custom_case_username = StringProperty() # Case property to use when showing the case's name in a chat window # If empty, sms can be sent at any time. Otherwise, only send during # these windows of time. SMS_QUEUE_ENABLED must be True in localsettings # for this be considered. restricted_sms_times = SchemaListProperty(DayTimeWindow) # If empty, this is ignored. Otherwise, the framework will make sure # that during these days/times, no automated outbound sms will be sent # to someone if they have sent in an sms within sms_conversation_length # minutes. Outbound sms sent from a user in a chat window, however, will # still be sent. This is meant to prevent chat conversations from being # interrupted by automated sms reminders. # SMS_QUEUE_ENABLED must be True in localsettings for this to be # considered. sms_conversation_times = SchemaListProperty(DayTimeWindow) # In minutes, see above. sms_conversation_length = IntegerProperty(default=10) # Set to True to prevent survey questions and answers form being seen in # SMS chat windows. filter_surveys_from_chat = BooleanProperty(default=False) # The below option only matters if filter_surveys_from_chat = True. # If set to True, invalid survey responses will still be shown in the chat # window, while questions and valid responses will be filtered out. show_invalid_survey_responses_in_chat = BooleanProperty(default=False) # If set to True, if a message is read by anyone it counts as being read by # everyone. Set to False so that a message is only counted as being read # for a user if only that user has read it. count_messages_as_read_by_anyone = BooleanProperty(default=False) # Set to True to allow sending sms and all-label surveys to cases whose # phone number is duplicated with another contact send_to_duplicated_case_numbers = BooleanProperty(default=True) enable_registration_welcome_sms_for_case = BooleanProperty(default=False) enable_registration_welcome_sms_for_mobile_worker = BooleanProperty(default=False) sms_survey_date_format = StringProperty() # exchange/domain copying stuff is_snapshot = BooleanProperty(default=False) is_approved = BooleanProperty(default=False) snapshot_time = DateTimeProperty() published = BooleanProperty(default=False) license = StringProperty(choices=LICENSES, default='cc') title = StringProperty() cda = SchemaProperty(LicenseAgreement) multimedia_included = BooleanProperty(default=True) downloads = IntegerProperty(default=0) # number of downloads for this specific snapshot full_downloads = IntegerProperty(default=0) # number of downloads for all snapshots from this domain author = StringProperty() phone_model = StringProperty() attribution_notes = StringProperty() publisher = StringProperty(choices=["organization", "user"], default="user") yt_id = StringProperty() snapshot_head = BooleanProperty(default=False) deployment = SchemaProperty(Deployment) image_path = StringProperty() image_type = StringProperty() cached_properties = DictProperty() internal = SchemaProperty(InternalProperties) dynamic_reports = SchemaListProperty(DynamicReportSet) # extra user specified properties tags = StringListProperty() area = StringProperty(choices=AREA_CHOICES) sub_area = StringProperty(choices=SUB_AREA_CHOICES) launch_date = DateTimeProperty # to be eliminated from projects and related documents when they are copied for the exchange _dirty_fields = ('admin_password', 'admin_password_charset', 'city', 'countries', 'region', 'customer_type') last_modified = DateTimeProperty(default=datetime(2015, 1, 1)) # when turned on, use SECURE_TIMEOUT for sessions of users who are members of this domain secure_sessions = BooleanProperty(default=False) two_factor_auth = BooleanProperty(default=False) strong_mobile_passwords = BooleanProperty(default=False) # There is no longer a way to request a report builder trial, so this property should be removed in the near # future. (Keeping it for now in case a user has requested a trial and but has not yet been granted it) requested_report_builder_trial = StringListProperty() requested_report_builder_subscription = StringListProperty() @classmethod def wrap(cls, data): # for domains that still use original_doc should_save = False if 'original_doc' in data: original_doc = data['original_doc'] del data['original_doc'] should_save = True if original_doc: original_doc = Domain.get_by_name(original_doc) data['copy_history'] = [original_doc._id] # for domains that have a public domain license if 'license' in data: if data.get("license", None) == "public": data["license"] = "cc" should_save = True if 'slug' in data and data["slug"]: data["hr_name"] = data["slug"] del data["slug"] if 'is_test' in data and isinstance(data["is_test"], bool): data["is_test"] = "true" if data["is_test"] else "false" should_save = True if 'cloudcare_releases' not in data: data['cloudcare_releases'] = 'nostars' # legacy default setting # Don't actually remove location_types yet. We can migrate fully and # remove this after everything's hunky-dory in production. 2015-03-06 if 'location_types' in data: data['obsolete_location_types'] = data.pop('location_types') self = super(Domain, cls).wrap(data) if self.deployment is None: self.deployment = Deployment() if should_save: self.save() return self def get_default_timezone(self): """return a timezone object from self.default_timezone""" import pytz return pytz.timezone(self.default_timezone) @staticmethod @quickcache(['name'], timeout=24 * 60 * 60) def is_secure_session_required(name): domain = Domain.get_by_name(name) return domain and domain.secure_sessions @staticmethod @skippable_quickcache(['couch_user._id', 'is_active'], skip_arg='strict', timeout=5*60, memoize_timeout=10) def active_for_couch_user(couch_user, is_active=True, strict=False): domain_names = couch_user.get_domains() return Domain.view( "domain/by_status", keys=[[is_active, d] for d in domain_names], reduce=False, include_docs=True, stale=settings.COUCH_STALE_QUERY if not strict else None, ).all() @staticmethod def active_for_user(user, is_active=True, strict=False): if isinstance(user, AnonymousUser): return [] from corehq.apps.users.models import CouchUser if isinstance(user, CouchUser): couch_user = user else: couch_user = CouchUser.from_django_user(user) if couch_user: return Domain.active_for_couch_user( couch_user, is_active=is_active, strict=strict) else: return [] @classmethod def field_by_prefix(cls, field, prefix=''): # unichr(0xfff8) is something close to the highest character available res = cls.view("domain/fields_by_prefix", group=True, startkey=[field, True, prefix], endkey=[field, True, "%s%c" % (prefix, unichr(0xfff8)), {}]) vals = [(d['value'], d['key'][2]) for d in res] vals.sort(reverse=True) return [(v[1], v[0]) for v in vals] def add(self, model_instance, is_active=True): """ Add something to this domain, through the generic relation. Returns the created membership object """ # Add membership info to Couch couch_user = model_instance.get_profile().get_couch_user() couch_user.add_domain_membership(self.name) couch_user.save() def applications(self): return get_brief_apps_in_domain(self.name) def full_applications(self, include_builds=True): from corehq.apps.app_manager.models import Application, RemoteApp WRAPPERS = {'Application': Application, 'RemoteApp': RemoteApp} def wrap_application(a): return WRAPPERS[a['doc']['doc_type']].wrap(a['doc']) if include_builds: startkey = [self.name] endkey = [self.name, {}] else: startkey = [self.name, None] endkey = [self.name, None, {}] return Application.get_db().view('app_manager/applications', startkey=startkey, endkey=endkey, include_docs=True, wrapper=wrap_application).all() @cached_property def versions(self): apps = self.applications() return list(set(a.application_version for a in apps)) @cached_property def has_case_management(self): for app in self.full_applications(): if app.doc_type == 'Application': if app.has_case_management(): return True return False @cached_property def has_media(self): for app in self.full_applications(): if app.doc_type == 'Application' and app.has_media(): return True return False @property def use_cloudcare_releases(self): return self.cloudcare_releases != 'nostars' def all_users(self): from corehq.apps.users.models import CouchUser return CouchUser.by_domain(self.name) def recent_submissions(self): return domain_has_submission_in_last_30_days(self.name) @cached_property def languages(self): apps = self.applications() return set(chain.from_iterable([a.langs for a in apps])) def readable_languages(self): return ', '.join(lang_lookup[lang] or lang for lang in self.languages()) def __unicode__(self): return self.name @classmethod @skippable_quickcache(['name'], skip_arg='strict', timeout=30*60) def get_by_name(cls, name, strict=False): if not name: # get_by_name should never be called with name as None (or '', etc) # I fixed the code in such a way that if I raise a ValueError # all tests pass and basic pages load, # but in order not to break anything in the wild, # I'm opting to notify by email if/when this happens # but fall back to the previous behavior of returning None if settings.DEBUG: raise ValueError('%r is not a valid domain name' % name) else: _assert = soft_assert(notify_admins=True, exponential_backoff=False) _assert(False, '%r is not a valid domain name' % name) return None def _get_by_name(stale=False): extra_args = {'stale': settings.COUCH_STALE_QUERY} if stale else {} result = cls.view("domain/domains", key=name, reduce=False, include_docs=True, **extra_args).first() if not isinstance(result, Domain): # A stale view may return a result with no doc if the doc has just been deleted. # In this case couchdbkit just returns the raw view result as a dict return None else: return result domain = _get_by_name(stale=(not strict)) if domain is None and not strict: # on the off chance this is a brand new domain, try with strict domain = _get_by_name(stale=False) return domain @classmethod def get_or_create_with_name(cls, name, is_active=False, secure_submissions=True, use_sql_backend=False): result = cls.view("domain/domains", key=name, reduce=False, include_docs=True).first() if result: return result else: new_domain = Domain( name=name, is_active=is_active, date_created=datetime.utcnow(), secure_submissions=secure_submissions, use_sql_backend=use_sql_backend, ) new_domain.save(**get_safe_write_kwargs()) return new_domain @classmethod def generate_name(cls, hr_name, max_length=25): ''' Generate a URL-friendly name based on a given human-readable name. Normalizes given name, then looks for conflicting domains, addressing conflicts by adding "-1", "-2", etc. May return None if it fails to generate a new, unique name. Throws exception if it can't figure out a name, which shouldn't happen unless max_length is absurdly short. ''' from corehq.apps.domain.utils import get_domain_url_slug name = get_domain_url_slug(hr_name, max_length=max_length) if not name: raise NameUnavailableException if Domain.get_by_name(name): prefix = name while len(prefix): name = next_available_name(prefix, Domain.get_names_by_prefix(prefix + '-')) if Domain.get_by_name(name): # should never happen raise NameUnavailableException if len(name) <= max_length: return name prefix = prefix[:-1] raise NameUnavailableException return name @classmethod def get_all(cls, include_docs=True): domains = Domain.view("domain/not_snapshots", include_docs=False).all() if not include_docs: return domains else: return imap(cls.wrap, iter_docs(cls.get_db(), [d['id'] for d in domains])) @classmethod def get_all_names(cls): return [d['key'] for d in cls.get_all(include_docs=False)] @classmethod def get_all_ids(cls): return [d['id'] for d in cls.get_all(include_docs=False)] @classmethod def get_names_by_prefix(cls, prefix): return [d['key'] for d in Domain.view( "domain/domains", startkey=prefix, endkey=prefix + u"zzz", reduce=False, include_docs=False ).all()] def case_sharing_included(self): return self.case_sharing or reduce(lambda x, y: x or y, [getattr(app, 'case_sharing', False) for app in self.applications()], False) def save(self, **params): self.last_modified = datetime.utcnow() if not self._rev: # mark any new domain as timezone migration complete set_migration_complete(self.name) super(Domain, self).save(**params) from corehq.apps.domain.signals import commcare_domain_post_save results = commcare_domain_post_save.send_robust(sender='domain', domain=self) for result in results: # Second argument is None if there was no error if result[1]: notify_exception( None, message="Error occured during domain post_save %s: %s" % (self.name, str(result[1])) ) def save_copy(self, new_domain_name=None, new_hr_name=None, user=None, copy_by_id=None, share_reminders=True, share_user_roles=True): from corehq.apps.app_manager.dbaccessors import get_app from corehq.apps.reminders.models import CaseReminderHandler from corehq.apps.fixtures.models import FixtureDataItem from corehq.apps.app_manager.dbaccessors import get_brief_apps_in_domain from corehq.apps.domain.dbaccessors import get_doc_ids_in_domain_by_class from corehq.apps.fixtures.models import FixtureDataType from corehq.apps.users.models import UserRole db = Domain.get_db() new_id = db.copy_doc(self.get_id)['id'] if new_domain_name is None: new_domain_name = new_id with CriticalSection(['request_domain_name_{}'.format(new_domain_name)]): new_domain_name = Domain.generate_name(new_domain_name) new_domain = Domain.get(new_id) new_domain.name = new_domain_name new_domain.hr_name = new_hr_name new_domain.copy_history = self.get_updated_history() new_domain.is_snapshot = False new_domain.snapshot_time = None new_domain.organization = None # TODO: use current user's organization (?) # reset stuff new_domain.cda.signed = False new_domain.cda.date = None new_domain.cda.type = None new_domain.cda.user_id = None new_domain.cda.user_ip = None new_domain.is_test = "none" new_domain.internal = InternalProperties() new_domain.creating_user = user.username if user else None new_domain.date_created = datetime.utcnow() for field in self._dirty_fields: if hasattr(new_domain, field): delattr(new_domain, field) # Saving the domain should happen before we import any apps since # importing apps can update the domain object (for example, if user # as a case needs to be enabled) new_domain.save() new_app_components = {} # a mapping of component's id to its copy def copy_data_items(old_type_id, new_type_id): for item in FixtureDataItem.by_data_type(self.name, old_type_id): comp = self.copy_component( item.doc_type, item._id, new_domain_name, user=user) comp.data_type_id = new_type_id comp.save() def get_latest_app_id(doc_id): app = get_app(self.name, doc_id).get_latest_saved() if app: return app._id, app.doc_type for app in get_brief_apps_in_domain(self.name): doc_id, doc_type = app.get_id, app.doc_type original_doc_id = doc_id if copy_by_id and doc_id not in copy_by_id: continue if not self.is_snapshot: doc_id, doc_type = get_latest_app_id(doc_id) or (doc_id, doc_type) component = self.copy_component(doc_type, doc_id, new_domain_name, user=user) if component: new_app_components[original_doc_id] = component for doc_id in get_doc_ids_in_domain_by_class(self.name, FixtureDataType): if copy_by_id and doc_id not in copy_by_id: continue component = self.copy_component( 'FixtureDataType', doc_id, new_domain_name, user=user) copy_data_items(doc_id, component._id) if share_reminders: for doc_id in get_doc_ids_in_domain_by_class(self.name, CaseReminderHandler): self.copy_component( 'CaseReminderHandler', doc_id, new_domain_name, user=user) if share_user_roles: for doc_id in get_doc_ids_in_domain_by_class(self.name, UserRole): self.copy_component('UserRole', doc_id, new_domain_name, user=user) if user: def add_dom_to_user(user): user.add_domain_membership(new_domain_name, is_admin=True) apply_update(user, add_dom_to_user) def update_events(handler): """ Change the form_unique_id to the proper form for each event in a newly copied CaseReminderHandler """ from corehq.apps.app_manager.models import FormBase for event in handler.events: if not event.form_unique_id: continue form = FormBase.get_form(event.form_unique_id) form_app = form.get_app() m_index, f_index = form_app.get_form_location(form.unique_id) form_copy = new_app_components[form_app._id].get_module(m_index).get_form(f_index) event.form_unique_id = form_copy.unique_id def update_for_copy(handler): handler.active = False update_events(handler) if share_reminders: for handler in CaseReminderHandler.get_handlers(new_domain_name): apply_update(handler, update_for_copy) return new_domain def reminder_should_be_copied(self, handler): from corehq.apps.reminders.models import ON_DATETIME return (handler.start_condition_type != ON_DATETIME and handler.user_group_id is None) def copy_component(self, doc_type, id, new_domain_name, user=None): from corehq.apps.app_manager.models import import_app from corehq.apps.users.models import UserRole from corehq.apps.reminders.models import CaseReminderHandler from corehq.apps.fixtures.models import FixtureDataType, FixtureDataItem str_to_cls = { 'UserRole': UserRole, 'CaseReminderHandler': CaseReminderHandler, 'FixtureDataType': FixtureDataType, 'FixtureDataItem': FixtureDataItem, } if doc_type in ('Application', 'RemoteApp'): new_doc = import_app(id, new_domain_name) new_doc.copy_history.append(id) new_doc.case_sharing = False # when copying from app-docs that don't have # unique_id attribute on Modules new_doc.ensure_module_unique_ids(should_save=False) else: cls = str_to_cls[doc_type] db = cls.get_db() if doc_type == 'CaseReminderHandler': cur_doc = cls.get(id) if not self.reminder_should_be_copied(cur_doc): return None new_id = db.copy_doc(id)['id'] new_doc = cls.get(new_id) for field in self._dirty_fields: if hasattr(new_doc, field): delattr(new_doc, field) if hasattr(cls, '_meta_fields'): for field in cls._meta_fields: if not field.startswith('_') and hasattr(new_doc, field): delattr(new_doc, field) new_doc.domain = new_domain_name if doc_type == 'FixtureDataType': new_doc.copy_from = id new_doc.is_global = True if self.is_snapshot and doc_type == 'Application': new_doc.prepare_multimedia_for_exchange() new_doc.save() return new_doc def save_snapshot(self, share_reminders, copy_by_id=None): if self.is_snapshot: return self else: try: copy = self.save_copy( copy_by_id=copy_by_id, share_reminders=share_reminders, share_user_roles=False) except NameUnavailableException: return None copy.is_snapshot = True head = self.snapshots(limit=1).first() if head and head.snapshot_head: head.snapshot_head = False head.save() copy.snapshot_head = True copy.snapshot_time = datetime.utcnow() del copy.deployment copy.save() return copy def snapshots(self, **view_kwargs): return Domain.view('domain/snapshots', startkey=[self._id, {}], endkey=[self._id], include_docs=True, reduce=False, descending=True, **view_kwargs ) @memoized def published_snapshot(self): snapshots = self.snapshots().all() for snapshot in snapshots: if snapshot.published: return snapshot return None def update_deployment(self, **kwargs): self.deployment.update(kwargs) self.save() def update_internal(self, **kwargs): self.internal.update(kwargs) self.save() def display_name(self): if self.is_snapshot: return "Snapshot of %s" % self.copied_from.display_name() return self.hr_name or self.name def long_display_name(self): if self.is_snapshot: return format_html("Snapshot of {}", self.copied_from.display_name()) return self.hr_name or self.name __str__ = long_display_name def get_license_display(self): return LICENSES.get(self.license) def get_license_url(self): return LICENSE_LINKS.get(self.license) def copies(self): return Domain.view('domain/copied_from_snapshot', key=self._id, include_docs=True) def copies_of_parent(self): return Domain.view('domain/copied_from_snapshot', keys=[s._id for s in self.copied_from.snapshots()], include_docs=True) def delete(self): self._pre_delete() super(Domain, self).delete() def _pre_delete(self): from corehq.apps.domain.signals import commcare_domain_pre_delete from corehq.apps.domain.deletion import apply_deletion_operations dynamic_deletion_operations = [] results = commcare_domain_pre_delete.send_robust(sender='domain', domain=self) for result in results: response = result[1] if isinstance(response, Exception): raise DomainDeleteException(u"Error occurred during domain pre_delete {}: {}".format(self.name, str(response))) elif response: assert isinstance(response, list) dynamic_deletion_operations.extend(response) # delete all associated objects for db, related_doc_ids in get_all_doc_ids_for_domain_grouped_by_db(self.name): iter_bulk_delete(db, related_doc_ids, chunksize=500) apply_deletion_operations(self.name, dynamic_deletion_operations) def all_media(self, from_apps=None): # todo add documentation or refactor from corehq.apps.hqmedia.models import CommCareMultimedia dom_with_media = self if not self.is_snapshot else self.copied_from if self.is_snapshot: app_ids = [app.copied_from.get_id for app in self.full_applications()] if from_apps: from_apps = set([a_id for a_id in app_ids if a_id in from_apps]) else: from_apps = app_ids if from_apps: media = [] media_ids = set() apps = [app for app in dom_with_media.full_applications() if app.get_id in from_apps] for app in apps: if app.doc_type != 'Application': continue for _, m in app.get_media_objects(): if m.get_id not in media_ids: media.append(m) media_ids.add(m.get_id) return media return CommCareMultimedia.view('hqmedia/by_domain', key=dom_with_media.name, include_docs=True).all() def most_restrictive_licenses(self, apps_to_check=None): from corehq.apps.hqmedia.utils import most_restrictive licenses = [m.license['type'] for m in self.all_media(from_apps=apps_to_check) if m.license] return most_restrictive(licenses) @classmethod def get_module_by_name(cls, domain_name): """ import and return the python module corresponding to domain_name, or None if it doesn't exist. """ from corehq.apps.domain.utils import get_domain_module_map module_name = get_domain_module_map().get(domain_name, domain_name) try: return import_module(module_name) if module_name else None except ImportError: return None @property @memoized def commtrack_settings(self): # this import causes some dependency issues so lives in here from corehq.apps.commtrack.models import CommtrackConfig if self.commtrack_enabled: return CommtrackConfig.for_domain(self.name) else: return None @property def has_custom_logo(self): return (self['_attachments'] and LOGO_ATTACHMENT in self['_attachments']) def get_custom_logo(self): if not self.has_custom_logo: return None return ( self.fetch_attachment(LOGO_ATTACHMENT), self['_attachments'][LOGO_ATTACHMENT]['content_type'] ) def get_case_display(self, case): """Get the properties display definition for a given case""" return self.case_display.case_details.get(case.type) def get_form_display(self, form): """Get the properties display definition for a given XFormInstance""" return self.case_display.form_details.get(form.xmlns) @property def total_downloads(self): """ Returns the total number of downloads from every snapshot created from this domain """ from corehq.apps.domain.dbaccessors import count_downloads_for_all_snapshots return count_downloads_for_all_snapshots(self.get_id) @property @memoized def download_count(self): """ Updates and returns the total number of downloads from every sister snapshot. """ if self.is_snapshot: self.full_downloads = self.copied_from.total_downloads return self.full_downloads @property @memoized def published_by(self): from corehq.apps.users.models import CouchUser pb_id = self.cda.user_id return CouchUser.get_by_user_id(pb_id) if pb_id else None @property def name_of_publisher(self): return self.published_by.human_friendly_name if self.published_by else "" @property def location_types(self): from corehq.apps.locations.models import LocationType return LocationType.objects.filter(domain=self.name).all() @memoized def has_privilege(self, privilege): from corehq.apps.accounting.utils import domain_has_privilege return domain_has_privilege(self, privilege) @property @memoized def uses_locations(self): from corehq import privileges from corehq.apps.locations.models import LocationType return (self.has_privilege(privileges.LOCATIONS) and (self.commtrack_enabled or LocationType.objects.filter(domain=self.name).exists())) @property def supports_multiple_locations_per_user(self): """ This method is a wrapper around the toggle that enables multiple location functionality. Callers of this method should know that this is special functionality left around for special applications, and not a feature flag that should be set normally. """ return toggles.MULTIPLE_LOCATIONS_PER_USER.enabled(self.name) def convert_to_commtrack(self): """ One-stop-shop to make a domain CommTrack """ from corehq.apps.commtrack.util import make_domain_commtrack make_domain_commtrack(self) def clear_caches(self): from .utils import domain_restricts_superusers super(Domain, self).clear_caches() self.get_by_name.clear(self.__class__, self.name) self.is_secure_session_required.clear(self.name) domain_restricts_superusers.clear(self.name)
class RequisitionCase(CommCareCase): """ A wrapper around CommCareCases to get more built in functionality specific to requisitions. """ class Meta: # This is necessary otherwise syncdb will confuse this app with casexml app_label = "commtrack" requisition_status = StringProperty() # TODO none of these properties are supported on mobile currently # we need to discuss what will be eventually so we know what we need # to support here requested_on = DateTimeProperty() approved_on = DateTimeProperty() fulfilled_on = DateTimeProperty() received_on = DateTimeProperty() requested_by = StringProperty() approved_by = StringProperty() fulfilled_by = StringProperty() received_by = StringProperty() @memoized def get_location(self): try: return SupplyPointCase.get(self.indices[0].referenced_id).location except ResourceNotFound: return None @memoized def get_requester(self): # TODO this doesn't get set by mobile yet # if self.requested_by: # return CommCareUser.get(self.requested_by) return None def sms_format(self): if self.requisition_status == RequisitionStatus.REQUESTED: section = 'ct-requested' elif self.requisition_status == RequisitionStatus.APPROVED: section = 'ct-approved' else: section = 'stock' formatted_strings = [] states = StockState.objects.filter(case_id=self._id, section_id=section) for state in states: product = Product.get(state.product_id) formatted_strings.append('%s:%d' % (product.code, state.stock_on_hand)) return ' '.join(sorted(formatted_strings)) def get_next_action(self): req_config = CommtrackConfig.for_domain(self.domain).requisition_config return req_config.get_next_action( RequisitionStatus.to_action_type(self.requisition_status)) @classmethod def get_by_external_id(cls, domain, external_id): # only used by openlmis raise NotImplementedError() # return cls.view('hqcase/by_domain_external_id', # key=[domain, external_id], # include_docs=True, reduce=False, # classes={'CommCareCase': RequisitionCase} # ).all() @classmethod def get_display_config(cls): return [{ "layout": [ [{ "name": _("Status"), "expr": "requisition_status" }], ] }, { "layout": [[{ "name": _("Requested On"), "expr": "requested_on", "parse_date": True }, { "name": _("requested_by"), "expr": "requested_by" }], [{ "name": _("Approved On"), "expr": "approved_on", "parse_date": True }, { "name": _("approved_by"), "expr": "approved_by" }], [{ "name": _("Received On"), "expr": "received_on", "parse_date": True }, { "name": _("received_by"), "expr": "received_by" }]] }]
class AbstractSyncLog(SafeSaveDocument): date = DateTimeProperty() domain = StringProperty() user_id = StringProperty() request_user_id = StringProperty() # ID of user making request build_id = StringProperty() # only works with app-aware sync app_id = StringProperty() # only works with app-aware sync previous_log_id = StringProperty() # previous sync log, forming a chain duration = IntegerProperty() # in seconds log_format = StringProperty() # owner_ids_on_phone stores the ids the phone thinks it's the owner of. # This typically includes the user id, # as well as all groups that that user is a member of. owner_ids_on_phone = StringListProperty() # for debugging / logging previous_log_rev = StringProperty( ) # rev of the previous log at the time of creation last_submitted = DateTimeProperty( ) # last time a submission caused this to be modified rev_before_last_submitted = StringProperty( ) # rev when the last submission was saved last_cached = DateTimeProperty( ) # last time this generated a cached response hash_at_last_cached = StringProperty( ) # the state hash of this when it was last cached # save state errors and hashes here had_state_error = BooleanProperty(default=False) error_date = DateTimeProperty() error_hash = StringProperty() cache_payload_paths = DictProperty() last_ucr_sync_times = SchemaListProperty(UCRSyncLog) strict = True # for asserts @classmethod def wrap(cls, data): ret = super(AbstractSyncLog, cls).wrap(data) if hasattr(ret, 'has_assert_errors'): ret.strict = False return ret def save(self): self._synclog_sql = save_synclog_to_sql(self) def delete(self): if getattr(self, '_synclog_sql', None): self._synclog_sql.delete() def case_count(self): """ How many cases are associated with this. Used in reports. """ raise NotImplementedError() def phone_is_holding_case(self, case_id): raise NotImplementedError() def get_footprint_of_cases_on_phone(self): """ Gets the phone's flat list of all case ids on the phone, owned or not owned but relevant. """ raise NotImplementedError() def get_state_hash(self): return CaseStateHash( Checksum(self.get_footprint_of_cases_on_phone()).hexdigest()) def update_phone_lists(self, xform, case_list): """ Given a form an list of touched cases, update this sync log to reflect the updated state on the phone. """ raise NotImplementedError() @classmethod def from_other_format(cls, other_sync_log): """ Convert to an instance of a subclass from another subclass. Subclasses can override this to provide conversion functions. """ raise IncompatibleSyncLogType('Unable to convert from {} to {}'.format( type(other_sync_log), cls, ))
class UCRSyncLog(Document): report_uuid = StringProperty() datetime = DateTimeProperty()
class Group(QuickCachedDocumentMixin, UndoableDocument): """ The main use case for these 'groups' of users is currently so that we can break down reports by arbitrary regions. (Things like who sees what reports are determined by permissions.) """ domain = StringProperty() name = StringProperty() # a list of user ids for users users = ListProperty() # a list of user ids that have been removed from the Group. # This is recorded so that we can update the user at a later point removed_users = SetProperty() case_sharing = BooleanProperty() reporting = BooleanProperty(default=True) last_modified = DateTimeProperty() # custom data can live here metadata = DictProperty() @classmethod def wrap(cls, data): last_modified = data.get('last_modified') # if it's missing a Z because of the Aug. 2014 migration # that added this in iso_format() without Z, then add a Z if last_modified and dt_no_Z_re.match(last_modified): data['last_modified'] += 'Z' return super(Group, cls).wrap(data) def save(self, *args, **kwargs): self.last_modified = datetime.utcnow() super(Group, self).save(*args, **kwargs) refresh_group_views() @classmethod def save_docs(cls, docs, use_uuids=True): utcnow = datetime.utcnow() for doc in docs: doc['last_modified'] = utcnow super(Group, cls).save_docs(docs, use_uuids) refresh_group_views() bulk_save = save_docs def delete(self): super(Group, self).delete() refresh_group_views() @classmethod def delete_docs(cls, docs, **params): super(Group, cls).delete_docs(docs, **params) refresh_group_views() bulk_delete = delete_docs def clear_caches(self): super(Group, self).clear_caches() self.by_domain.clear(self.__class__, self.domain) self.ids_by_domain.clear(self.__class__, self.domain) def add_user(self, couch_user_id, save=True): if not isinstance(couch_user_id, str): couch_user_id = couch_user_id.user_id if couch_user_id not in self.users: self.users.append(couch_user_id) if couch_user_id in self.removed_users: self.removed_users.remove(couch_user_id) if save: self.save() def remove_user(self, couch_user_id): ''' Returns True if it removed a user, False otherwise ''' if not isinstance(couch_user_id, str): couch_user_id = couch_user_id.user_id if couch_user_id in self.users: for i in range(0, len(self.users)): if self.users[i] == couch_user_id: del self.users[i] self.removed_users.add(couch_user_id) return True return False def get_user_ids(self, is_active=True): return [user.user_id for user in self.get_users(is_active=is_active)] @memoized def get_users(self, is_active=True, only_commcare=False): def is_relevant_user(user): if user.is_deleted(): return False if only_commcare and user.__class__ != CommCareUser().__class__: return False if is_active and not user.is_active: return False return True users = map(CouchUser.wrap_correctly, iter_docs(self.get_db(), self.users)) return list(filter(is_relevant_user, users)) @memoized def get_static_user_ids(self, is_active=True): return [user.user_id for user in self.get_static_users(is_active)] @classmethod def get_static_user_ids_for_groups(cls, group_ids): static_user_ids = [] for group_id in group_ids: group = cls.get(group_id) static_user_ids.append(group.get_static_user_ids()) return static_user_ids @memoized def get_static_users(self, is_active=True): return self.get_users(is_active) @classmethod @quickcache(['cls.__name__', 'domain']) def by_domain(cls, domain): return group_by_domain(domain) @classmethod def choices_by_domain(cls, domain): group_ids = cls.ids_by_domain(domain) group_choices = [] for group_doc in iter_docs(cls.get_db(), group_ids): group_choices.append((group_doc['_id'], group_doc['name'])) return group_choices @classmethod @quickcache(['cls.__name__', 'domain']) def ids_by_domain(cls, domain): return get_group_ids_by_domain(domain) @classmethod def by_name(cls, domain, name, one=True): result = stale_group_by_name(domain, name) if one and result: return result[0] else: return result @classmethod def by_user_id(cls, user_id, wrap=True): results = cls.view('groups/by_user', key=user_id, include_docs=wrap) if wrap: return results else: return [r['id'] for r in results] @classmethod def get_case_sharing_accessible_locations(cls, domain, user): return [ location.case_sharing_group_object() for location in SQLLocation.objects.accessible_to_user( domain, user).filter(location_type__shares_cases=True) ] @classmethod def get_case_sharing_groups(cls, domain, wrap=True): all_groups = cls.by_domain(domain) if wrap: groups = [group for group in all_groups if group.case_sharing] groups.extend([ location.case_sharing_group_object() for location in SQLLocation.objects.filter( domain=domain, location_type__shares_cases=True) ]) return groups else: return [group._id for group in all_groups if group.case_sharing] @classmethod def get_reporting_groups(cls, domain): key = ['^Reporting', domain] return cls.view( 'groups/by_name', startkey=key, endkey=key + [{}], include_docs=True, stale=settings.COUCH_STALE_QUERY, ).all() def create_delete_record(self, *args, **kwargs): return DeleteGroupRecord(*args, **kwargs) @property def display_name(self): if self.name: return self.name else: return "[No Name]" @classmethod def user_in_group(cls, user_id, group_id): if not user_id or not group_id: return False c = cls.get_db().view('groups/by_user', key=user_id, startkey_docid=group_id, endkey_docid=group_id).count() if c == 0: return False elif c == 1: return True else: raise Exception( "This should just logically not be possible unless the group " "has the user in there twice") def is_member_of(self, domain): return self.domain == domain @property def is_deleted(self): return self.doc_type.endswith(DELETED_SUFFIX) def __repr__(self): return ("Group(domain={self.domain!r}, name={self.name!r}, " "case_sharing={self.case_sharing!r})").format(self=self)
class ComputedDocumentMixin(DocumentSchema): """ Use this mixin for things like CommCareCase or XFormInstance documents that take advantage of indicator definitions. computed_ is namespaced and may look like the following for indicators: computed_: { mvp_indicators: { indicator_slug: { version: 1, value: "foo" } } } """ computed_ = DictProperty() computed_modified_on_ = DateTimeProperty() # a flag for the indicator pillows so that there aren't any Document Update Conflicts initial_processing_complete = BooleanProperty() def update_indicator(self, indicator_def, save_on_update=True, logger=None): existing_indicators = self.computed_.get(indicator_def.namespace, {}) updated_indicators, is_update = indicator_def.update_computed_namespace( existing_indicators, self) if is_update: self.computed_[indicator_def.namespace] = updated_indicators self.computed_modified_on_ = datetime.datetime.utcnow() if logger: logger.info( "[INDICATOR %(namespace)s %(domain)s] Updating %(indicator_type)s:%(indicator_slug)s " "in %(document_type)s [%(document_id)s]." % { 'namespace': indicator_def.namespace, 'domain': indicator_def.domain, 'indicator_type': indicator_def.__class__.__name__, 'indicator_slug': indicator_def.slug, 'document_type': self.__class__.__name__, 'document_id': self._id, }) if save_on_update: self.save(**get_safe_write_kwargs()) if logger: logger.debug("Saved %s." % self._id) return is_update def update_indicators_in_bulk(self, indicators, save_on_update=True, logger=None): is_update = False for indicator in indicators: try: if self.update_indicator(indicator, save_on_update=False, logger=logger): is_update = True except Exception: logger.exception( "[INDICATOR %(namespace)s %(domain)s] Failed to update %(indicator_type)s: " "%(indicator_slug)s in %(document_type)s [%(document_id)s]." % { 'namespace': indicator.namespace, 'domain': indicator.domain, 'indicator_type': indicator.__class__.__name__, 'indicator_slug': indicator.slug, 'document_type': self.__class__.__name__, 'document_id': self._id, }) if is_update and save_on_update: try: self.save(**get_safe_write_kwargs()) if logger: logger.info("Saved %s." % self._id) except ResourceConflict: logger.error( "[INDICATOR %(domain)s] Resource conflict failed to save document indicators for " "%(document_type)s [%(document_id)s]." % { 'domain': self.domain, 'document_type': self.__class__.__name__, 'document_id': self._id, }) return is_update
class RepeatRecord(Document): """ An record of a particular instance of something that needs to be forwarded with a link to the proper repeater object """ domain = StringProperty() repeater_id = StringProperty() repeater_type = StringProperty() payload_id = StringProperty() overall_tries = IntegerProperty(default=0) max_possible_tries = IntegerProperty(default=6) attempts = ListProperty(RepeatRecordAttempt) cancelled = BooleanProperty(default=False) registered_on = DateTimeProperty() last_checked = DateTimeProperty() failure_reason = StringProperty() next_check = DateTimeProperty() succeeded = BooleanProperty(default=False) @property def record_id(self): return self._id @classmethod def wrap(cls, data): should_bootstrap_attempts = ('attempts' not in data) self = super(RepeatRecord, cls).wrap(data) if should_bootstrap_attempts and self.last_checked: assert not self.attempts self.attempts = [ RepeatRecordAttempt( cancelled=self.cancelled, datetime=self.last_checked, failure_reason=self.failure_reason, success_response=None, next_check=self.next_check, succeeded=self.succeeded, ) ] return self @property @memoized def repeater(self): try: return Repeater.get(self.repeater_id) except ResourceNotFound: return None @property def url(self): warnings.warn( "RepeatRecord.url is deprecated. Use Repeater.get_url instead", DeprecationWarning) if self.repeater: return self.repeater.get_url(self) @property def state(self): state = RECORD_PENDING_STATE if self.succeeded: state = RECORD_SUCCESS_STATE elif self.cancelled: state = RECORD_CANCELLED_STATE elif self.failure_reason: state = RECORD_FAILURE_STATE return state @classmethod def all(cls, domain=None, due_before=None, limit=None): json_now = json_format_datetime(due_before or datetime.utcnow()) repeat_records = RepeatRecord.view( "repeaters/repeat_records_by_next_check", startkey=[domain], endkey=[domain, json_now, {}], include_docs=True, reduce=False, limit=limit, ) return repeat_records @classmethod def count(cls, domain=None): results = RepeatRecord.view( "repeaters/repeat_records_by_next_check", startkey=[domain], endkey=[domain, {}], reduce=True, ).one() return results['value'] if results else 0 def add_attempt(self, attempt): self.attempts.append(attempt) self.last_checked = attempt.datetime self.next_check = attempt.next_check self.succeeded = attempt.succeeded self.cancelled = attempt.cancelled self.failure_reason = attempt.failure_reason def get_numbered_attempts(self): for i, attempt in enumerate(self.attempts): yield i + 1, attempt def postpone_by(self, duration): self.last_checked = datetime.utcnow() self.next_check = self.last_checked + duration self.save() def make_set_next_try_attempt(self, failure_reason): assert self.succeeded is False assert self.next_check is not None now = datetime.utcnow() return RepeatRecordAttempt( cancelled=False, datetime=now, failure_reason=failure_reason, success_response=None, next_check=now + _get_retry_interval(self.last_checked, now), succeeded=False, ) def try_now(self): # try when we haven't succeeded and either we've # never checked, or it's time to check again return not self.succeeded def get_payload(self): return self.repeater.get_payload(self) def get_attempt_info(self): return self.repeater.get_attempt_info(self) def handle_payload_exception(self, exception): now = datetime.utcnow() return RepeatRecordAttempt( cancelled=True, datetime=now, failure_reason=str(exception), success_response=None, next_check=None, succeeded=False, ) def fire(self, force_send=False): if self.try_now() or force_send: self.overall_tries += 1 try: attempt = self.repeater.fire_for_record(self) except Exception as e: log_repeater_error_in_datadog(self.domain, status_code=None, repeater_type=self.repeater_type) attempt = self.handle_payload_exception(e) raise finally: # pycharm warns attempt might not be defined. # that'll only happen if fire_for_record raise a non-Exception exception (e.g. SIGINT) # or handle_payload_exception raises an exception. I'm okay with that. -DMR self.add_attempt(attempt) self.save() @staticmethod def _format_response(response): if not _is_response(response): return None response_body = getattr(response, "text", "") return '{}: {}.\n{}'.format(response.status_code, response.reason, response_body) def handle_success(self, response): """ Log success in Datadog and return a success RepeatRecordAttempt. ``response`` can be a Requests response instance, or True if the payload did not result in an API call. """ now = datetime.utcnow() if _is_response(response): # ^^^ Don't bother logging success in Datadog if the payload # did not need to be sent. (This can happen with DHIS2 if # the form that triggered the forwarder doesn't contain data # for a DHIS2 Event.) log_repeater_success_in_datadog(self.domain, response.status_code, self.repeater_type) return RepeatRecordAttempt( cancelled=False, datetime=now, failure_reason=None, success_response=self._format_response(response), next_check=None, succeeded=True, info=self.get_attempt_info(), ) def handle_failure(self, response): """Do something with the response if the repeater fails """ return self._make_failure_attempt(self._format_response(response), response) def handle_exception(self, exception): """handle internal exceptions """ return self._make_failure_attempt(str(exception), None) def _make_failure_attempt(self, reason, response): log_repeater_error_in_datadog( self.domain, response.status_code if response else None, self.repeater_type) if self.repeater.allow_retries( response) and self.overall_tries < self.max_possible_tries: return self.make_set_next_try_attempt(reason) else: now = datetime.utcnow() return RepeatRecordAttempt( cancelled=True, datetime=now, failure_reason=reason, success_response=None, next_check=None, succeeded=False, info=self.get_attempt_info(), ) def cancel(self): self.next_check = None self.cancelled = True def attempt_forward_now(self): from corehq.motech.repeaters.tasks import process_repeat_record def is_ready(): return self.next_check < datetime.utcnow() def already_processed(): return self.succeeded or self.cancelled or self.next_check is None if already_processed() or not is_ready(): return # Set the next check to happen an arbitrarily long time from now so # if something goes horribly wrong with the delayed task it will not # be lost forever. A check at this time is expected to occur rarely, # if ever, because `process_repeat_record` will usually succeed or # reset the next check to sometime sooner. self.next_check = datetime.utcnow() + timedelta(hours=48) try: self.save() except ResourceConflict: # Another process beat us to the punch. This takes advantage # of Couch DB's optimistic locking, which prevents a process # with stale data from overwriting the work of another. return process_repeat_record.delay(self) def requeue(self): self.cancelled = False self.succeeded = False self.failure_reason = '' self.overall_tries = 0 self.next_check = datetime.utcnow()
class XFormInstance(DeferredBlobMixin, SafeSaveDocument, ComputedDocumentMixin, CouchDocLockableMixIn, AbstractXFormInstance): """An XForms instance.""" domain = StringProperty() app_id = StringProperty() xmlns = StringProperty() form = DictProperty() received_on = DateTimeProperty() server_modified_on = DateTimeProperty() # Used to tag forms that were forcefully submitted # without a touchforms session completing normally partial_submission = BooleanProperty(default=False) history = SchemaListProperty(XFormOperation) auth_context = DictProperty() submit_ip = StringProperty() path = StringProperty() openrosa_headers = DictProperty() last_sync_token = StringProperty() # almost always a datetime, but if it's not parseable it'll be a string date_header = DefaultProperty() build_id = StringProperty() export_tag = DefaultProperty(name='#export_tag') _blobdb_type_code = CODES.form_xml class Meta(object): app_label = 'couchforms' @classmethod def get(cls, docid, rev=None, db=None, dynamic_properties=True): # copied and tweaked from the superclass's method if not db: db = cls.get_db() cls._allow_dynamic_properties = dynamic_properties # on cloudant don't get the doc back until all nodes agree # on the copy, to avoid race conditions extras = get_safe_read_kwargs() try: if cls == XFormInstance: doc = db.get(docid, rev=rev, **extras) if doc['doc_type'] in doc_types(): return doc_types()[doc['doc_type']].wrap(doc) try: return XFormInstance.wrap(doc) except WrappingAttributeError: raise ResourceNotFound( "The doc with _id {} and doc_type {} can't be wrapped " "as an XFormInstance".format(docid, doc['doc_type'])) return db.get(docid, rev=rev, wrapper=cls.wrap, **extras) except ResourceNotFound: raise XFormNotFound(docid) @property def form_id(self): return self._id @form_id.setter def form_id(self, value): self._id = value @property def form_data(self): return DictProperty().unwrap(self.form)[1] @property def user_id(self): return getattr(self.metadata, 'userID', None) @property def is_error(self): return self.doc_type != 'XFormInstance' @property def is_duplicate(self): return self.doc_type == 'XFormDuplicate' @property def is_archived(self): return self.doc_type == 'XFormArchived' @property def is_deprecated(self): return self.doc_type == 'XFormDeprecated' @property def is_submission_error_log(self): return self.doc_type == 'SubmissionErrorLog' @property def is_deleted(self): return self.doc_type.endswith(DELETED_SUFFIX) @property def is_normal(self): return self.doc_type == 'XFormInstance' @property def deletion_id(self): return getattr(self, '-deletion_id', None) @property def deletion_date(self): return getattr(self, '-deletion_date', None) @property def metadata(self): if const.TAG_META in self.form: return Metadata.wrap( clean_metadata(self.to_json()[const.TAG_FORM][const.TAG_META])) return None @property def time_start(self): # Will be addressed in https://github.com/dimagi/commcare-hq/pull/19391/ return None @property def time_end(self): return None @property def commcare_version(self): return str(self.metadata.commcare_version) @property def app_version(self): return None def __str__(self): return "%s (%s)" % (self.type, self.xmlns) def save(self, **kwargs): # HACK: cloudant has a race condition when saving newly created forms # which throws errors here. use a try/retry loop here to get around # it until we find something more stable. RETRIES = 10 SLEEP = 0.5 # seconds tries = 0 self.server_modified_on = datetime.datetime.utcnow() while True: try: return super(XFormInstance, self).save(**kwargs) except PreconditionFailed: if tries == 0: logging.error('doc %s got a precondition failed', self._id) if tries < RETRIES: tries += 1 time.sleep(SLEEP) else: raise def get_data(self, path): """ Evaluates an xpath expression like: path/to/node and returns the value of that element, or None if there is no value. """ return safe_index(self, path.split("/")) def soft_delete(self): self.doc_type += DELETED_SUFFIX self.save() def get_xml(self): try: return self.fetch_attachment(ATTACHMENT_NAME) except ResourceNotFound: logging.warn("no xml found for %s, trying old attachment scheme." % self.get_id) try: return self[const.TAG_XML] except AttributeError: return None def get_attachment(self, attachment_name): return self.fetch_attachment(attachment_name) def get_xml_element(self): xml_string = self.get_xml() if not xml_string: return None return self._xml_string_to_element(xml_string) def _xml_string_to_element(self, xml_string): def _to_xml_element(payload): if isinstance(payload, six.text_type): payload = payload.encode('utf-8', errors='replace') return etree.fromstring(payload) try: return _to_xml_element(xml_string) except XMLSyntaxError: # there is a bug at least in pact code that double # saves a submission in a way that the attachments get saved in a base64-encoded format decoded_payload = base64.b64decode(xml_string) element = _to_xml_element(decoded_payload) # in this scenario resave the attachment properly in case future calls circumvent this method self.save() self.put_attachment(decoded_payload, ATTACHMENT_NAME) return element def put_attachment(self, content, name, **kw): if kw.get("type_code") is None: kw["type_code"] = (CODES.form_xml if name == ATTACHMENT_NAME else CODES.form_attachment) return super(XFormInstance, self).put_attachment(content, name, **kw) @property def attachments(self): """ Get the extra attachments for this form. This will not include the form itself """ def _meta_to_json(meta): is_image = False if meta.content_type is not None: is_image = True if meta.content_type.startswith( 'image/') else False meta_json = meta.to_json() meta_json['is_image'] = is_image return meta_json return { name: _meta_to_json(meta) for name, meta in six.iteritems(self.blobs) if name != ATTACHMENT_NAME } def xml_md5(self): return hashlib.md5(self.get_xml()).hexdigest() def archive(self, user_id=None): if self.is_archived: return # If this archive was initiated by a user, delete all other stubs for this action so that this action # isn't overridden from couchforms.models import UnfinishedArchiveStub UnfinishedArchiveStub.objects.filter( xform_id=self.form_id).all().delete() from corehq.form_processor.submission_process_tracker import unfinished_archive with unfinished_archive(instance=self, user_id=user_id, archive=True) as archive_stub: self.doc_type = "XFormArchived" self.history.append( XFormOperation( user=user_id, operation='archive', )) self.save() archive_stub.archive_history_updated() xform_archived.send(sender="couchforms", xform=self) def unarchive(self, user_id=None): if not self.is_archived: return # If this unarchive was initiated by a user, delete all other stubs for this action so that this action # isn't overridden from couchforms.models import UnfinishedArchiveStub UnfinishedArchiveStub.objects.filter( xform_id=self.form_id).all().delete() from corehq.form_processor.submission_process_tracker import unfinished_archive with unfinished_archive(instance=self, user_id=user_id, archive=False) as archive_stub: self.doc_type = "XFormInstance" self.history.append( XFormOperation( user=user_id, operation='unarchive', )) XFormInstance.save( self ) # subclasses explicitly set the doc type so force regular save archive_stub.archive_history_updated() xform_unarchived.send(sender="couchforms", xform=self) def publish_archive_action_to_kafka(self, user_id, archive): from couchforms.models import UnfinishedArchiveStub from corehq.form_processor.submission_process_tracker import unfinished_archive # Delete the original stub UnfinishedArchiveStub.objects.filter( xform_id=self.form_id).all().delete() with unfinished_archive(instance=self, user_id=user_id, archive=archive): if archive: xform_archived.send(sender="couchforms", xform=self) else: xform_unarchived.send(sender="couchforms", xform=self)
class DataSourceConfiguration(CachedCouchDocumentMixin, Document, AbstractUCRDataSource): """ A data source configuration. These map 1:1 with database tables that get created. Each data source can back an arbitrary number of reports. """ domain = StringProperty(required=True) engine_id = StringProperty(default=UCR_ENGINE_ID) backend_id = StringProperty(default=UCR_SQL_BACKEND) # no longer used referenced_doc_type = StringProperty(required=True) table_id = StringProperty(required=True) display_name = StringProperty() base_item_expression = DictProperty() configured_filter = DictProperty() configured_indicators = ListProperty() named_expressions = DictProperty() named_filters = DictProperty() meta = SchemaProperty(DataSourceMeta) is_deactivated = BooleanProperty(default=False) last_modified = DateTimeProperty() asynchronous = BooleanProperty(default=False) sql_column_indexes = SchemaListProperty(SQLColumnIndexes) disable_destructive_rebuild = BooleanProperty(default=False) sql_settings = SchemaProperty(SQLSettings) validations = SchemaListProperty(Validation) mirrored_engine_ids = ListProperty(default=[]) class Meta(object): # prevent JsonObject from auto-converting dates etc. string_conversions = () def __str__(self): return '{} - {}'.format(self.domain, self.display_name) def save(self, **params): self.last_modified = datetime.utcnow() super(DataSourceConfiguration, self).save(**params) @property def data_source_id(self): return self._id def filter(self, document, eval_context=None): if eval_context is None: eval_context = EvaluationContext(document) filter_fn = self._get_main_filter() return filter_fn(document, eval_context) def deleted_filter(self, document): filter_fn = self._get_deleted_filter() return filter_fn and filter_fn(document, EvaluationContext( document, 0)) @property def has_validations(self): return len(self.validations) > 0 def validate_document(self, document, eval_context=None): if eval_context is None: eval_context = EvaluationContext(document) errors = [] for validation in self._validations(): if validation.validation_function(document, eval_context) is False: errors.append((validation.name, validation.error_message)) if errors: raise ValidationError(errors) @memoized def _validations(self): return [ _Validation( validation.name, validation.error_message, FilterFactory.from_spec(validation.expression, context=self.get_factory_context())) for validation in self.validations ] @memoized def _get_main_filter(self): return self._get_filter([self.referenced_doc_type]) @memoized def _get_deleted_filter(self): return self._get_filter(get_deleted_doc_types( self.referenced_doc_type), include_configured=False) def _get_filter(self, doc_types, include_configured=True): if not doc_types: return None extras = ([self.configured_filter] if include_configured and self.configured_filter else []) built_in_filters = [ self._get_domain_filter_spec(), { 'type': 'or', 'filters': [{ "type": "boolean_expression", "expression": { "type": "property_name", "property_name": "doc_type", }, "operator": "eq", "property_value": doc_type, } for doc_type in doc_types], }, ] return FilterFactory.from_spec( { 'type': 'and', 'filters': built_in_filters + extras, }, context=self.get_factory_context(), ) def _get_domain_filter_spec(self): return { "type": "boolean_expression", "expression": { "type": "property_name", "property_name": "domain", }, "operator": "eq", "property_value": self.domain, } @property @memoized def named_expression_objects(self): named_expression_specs = deepcopy(self.named_expressions) named_expressions = {} spec_error = None while named_expression_specs: number_generated = 0 for name, expression in list(named_expression_specs.items()): try: named_expressions[name] = ExpressionFactory.from_spec( expression, FactoryContext(named_expressions=named_expressions, named_filters={})) number_generated += 1 del named_expression_specs[name] except BadSpecError as bad_spec_error: # maybe a nested name resolution issue, try again on the next pass spec_error = bad_spec_error if number_generated == 0 and named_expression_specs: # we unsuccessfully generated anything on this pass and there are still unresolved # references. we have to fail. assert spec_error is not None raise spec_error return named_expressions @property @memoized def named_filter_objects(self): return { name: FilterFactory.from_spec( filter, FactoryContext(self.named_expression_objects, {})) for name, filter in self.named_filters.items() } def get_factory_context(self): return FactoryContext(self.named_expression_objects, self.named_filter_objects) @property @memoized def default_indicators(self): default_indicators = [ IndicatorFactory.from_spec( { "column_id": "doc_id", "type": "expression", "display_name": "document id", "datatype": "string", "is_nullable": False, "is_primary_key": True, "expression": { "type": "root_doc", "expression": { "type": "property_name", "property_name": "_id" } } }, self.get_factory_context()) ] default_indicators.append( IndicatorFactory.from_spec({ "type": "inserted_at", }, self.get_factory_context())) if self.base_item_expression: default_indicators.append( IndicatorFactory.from_spec({ "type": "repeat_iteration", }, self.get_factory_context())) return default_indicators @property @memoized def indicators(self): return CompoundIndicator( self.display_name, self.default_indicators + [ IndicatorFactory.from_spec(indicator, self.get_factory_context()) for indicator in self.configured_indicators ], None, ) @property @memoized def parsed_expression(self): if self.base_item_expression: return ExpressionFactory.from_spec( self.base_item_expression, context=self.get_factory_context()) return None @memoized def get_columns(self): return self.indicators.get_columns() @property @memoized def columns_by_id(self): return {c.id: c for c in self.get_columns()} def get_column_by_id(self, column_id): return self.columns_by_id.get(column_id) def get_items(self, document, eval_context=None): if self.filter(document, eval_context): if not self.base_item_expression: return [document] else: result = self.parsed_expression(document, eval_context) if result is None: return [] elif isinstance(result, list): return result else: return [result] else: return [] def get_all_values(self, doc, eval_context=None): if not eval_context: eval_context = EvaluationContext(doc) if self.has_validations: try: self.validate_document(doc, eval_context) except ValidationError as e: for error in e.errors: InvalidUCRData.objects.get_or_create( doc_id=doc['_id'], indicator_config_id=self._id, validation_name=error[0], defaults={ 'doc_type': doc['doc_type'], 'domain': doc['domain'], 'validation_text': error[1], }) return [] rows = [] for item in self.get_items(doc, eval_context): values = self.indicators.get_values(item, eval_context) rows.append(values) eval_context.increment_iteration() return rows def get_report_count(self): """ Return the number of ReportConfigurations that reference this data source. """ return ReportConfiguration.count_by_data_source(self.domain, self._id) def validate_db_config(self): mirrored_engine_ids = self.mirrored_engine_ids if not mirrored_engine_ids: return if self.engine_id in mirrored_engine_ids: raise BadSpecError( "mirrored_engine_ids list should not contain engine_id") for engine_id in mirrored_engine_ids: if not connection_manager.engine_id_is_available(engine_id): raise BadSpecError( "DB for engine_id {} is not availble".format(engine_id)) if not connection_manager.resolves_to_unique_dbs(mirrored_engine_ids + [self.engine_id]): raise BadSpecError( "No two engine_ids should point to the same database") def validate(self, required=True): super(DataSourceConfiguration, self).validate(required) # these two properties implicitly call other validation self._get_main_filter() self._get_deleted_filter() # validate indicators and column uniqueness columns = [c.id for c in self.indicators.get_columns()] unique_columns = set(columns) if len(columns) != len(unique_columns): for column in set(columns): columns.remove(column) raise DuplicateColumnIdError(columns=columns) if self.referenced_doc_type not in VALID_REFERENCED_DOC_TYPES: raise BadSpecError( _('Report contains invalid referenced_doc_type: {}').format( self.referenced_doc_type)) self.parsed_expression self.pk_columns @classmethod def by_domain(cls, domain): return get_datasources_for_domain(domain) @classmethod def all_ids(cls): return [ res['id'] for res in cls.get_db().view( 'userreports/data_sources_by_build_info', reduce=False, include_docs=False) ] @classmethod def all(cls): for result in iter_docs(cls.get_db(), cls.all_ids()): yield cls.wrap(result) @property def is_static(self): return id_is_static(self._id) def deactivate(self, initiated_by=None): if not self.is_static: self.is_deactivated = True self.save() get_indicator_adapter(self).drop_table( initiated_by=initiated_by, source='deactivate-data-source') def get_case_type_or_xmlns_filter(self): """Returns a list of case types or xmlns from the filter of this data source. If this can't figure out the case types or xmlns's that filter, then returns [None] Currently always returns a list because it is called by a loop in _iteratively_build_table Could be reworked to return [] to be more pythonic """ if self.referenced_doc_type not in FILTER_INTERPOLATION_DOC_TYPES: return [None] property_name = FILTER_INTERPOLATION_DOC_TYPES[ self.referenced_doc_type] prop_value = self._filter_interploation_helper(self.configured_filter, property_name) return prop_value or [None] def _filter_interploation_helper(self, config_filter, property_name): filter_type = config_filter.get('type') if filter_type == 'and': sub_config_filters = [ self._filter_interploation_helper(f, property_name) for f in config_filter.get('filters') ] for filter_ in sub_config_filters: if filter_[0]: return filter_ if filter_type != 'boolean_expression': return [None] if config_filter['operator'] not in ('eq', 'in'): return [None] expression = config_filter['expression'] if not isinstance(expression, dict): return [None] if expression['type'] == 'property_name' and expression[ 'property_name'] == property_name: prop_value = config_filter['property_value'] if not isinstance(prop_value, list): prop_value = [prop_value] return prop_value return [None] @property def pk_columns(self): columns = [] for col in self.get_columns(): if col.is_primary_key: column_name = decode_column_name(col) columns.append(column_name) if self.sql_settings.primary_key: if set(columns) != set(self.sql_settings.primary_key): raise BadSpecError( "Primary key columns must have is_primary_key set to true", self.data_source_id) columns = self.sql_settings.primary_key return columns
class IndicatorDefinition(Document, AdminCRUDDocumentMixin): """ An Indicator Definition defines how to compute the indicator that lives in the namespaced computed_ property of a case or form. """ namespace = StringProperty() domain = StringProperty() slug = StringProperty() version = IntegerProperty() class_path = StringProperty() last_modified = DateTimeProperty() _admin_crud_class = IndicatorAdminCRUDManager _class_path = "corehq.apps.indicators.models" _returns_multiple = False def __init__(self, _d=None, **kwargs): super(IndicatorDefinition, self).__init__(_d, **kwargs) self.class_path = self._class_path def __str__(self): return u"\n\n%(class_name)s - Modified %(last_modified)s\n %(slug)s, domain: %(domain)s," \ u" version: %(version)s, namespace: %(namespace)s. ID: %(indicator_id)s." % { 'class_name': self.__class__.__name__, 'slug': self.slug, 'domain': self.domain, 'version': self.version, 'namespace': self.namespace, 'last_modified': (self.last_modified.strftime('%m %B %Y at %H:%M') if self.last_modified else "Ages Ago"), 'indicator_id': self._id, } @classmethod def key_properties(cls): """ The ordering of these property names should match the ordering of what's emitted in the first part of the couch views used for fetching these indicators. These views currently are: - indicators/dynamic_indicator_definitions (Couch View Indicator Defs) - indicators/indicator_definitions (Form and Case Indicator Defs) """ return ["namespace", "domain", "slug"] @classmethod def indicator_list_view(cls): return "indicators/indicator_definitions" @classmethod def _generate_couch_key(cls, version=None, reverse=False, **kwargs): key = list() key_prefix = list() for p in cls.key_properties(): k = kwargs.get(p) if k is not None: key_prefix.append(p) key.append(k) key = [" ".join(key_prefix)] + key couch_key = dict(startkey=key, endkey=key + [{}]) if version is None else dict(key=key + [version]) if reverse: return dict(startkey=couch_key.get('endkey'), endkey=couch_key.get('startkey')) return couch_key @classmethod def increment_or_create_unique(cls, namespace, domain, slug=None, version=None, **kwargs): """ If an indicator with the same namespace, domain, and version exists, create a new indicator with the version number incremented. # todo, this feels a bit buggy, so replace bulk copy indicators with # copy to domain at some point """ couch_key = cls._generate_couch_key(namespace=namespace, domain=domain, slug=slug, reverse=True, **kwargs) existing_indicator = cls.view(cls.indicator_list_view(), reduce=False, include_docs=True, descending=True, limit=1, **couch_key).first() if existing_indicator: version = existing_indicator.version + 1 elif version is None: version = 1 new_indicator = cls(version=version, namespace=namespace, domain=domain, slug=slug, **kwargs) new_indicator.last_modified = datetime.datetime.utcnow() new_indicator.save() return new_indicator @classmethod def copy_to_domain(cls, domain, doc, override=False): """ This copies an indicator doc to the current domain. Intended to be used by the export indicators feature. :param domain: the name of the domain the indicator should be copied to :param doc: the dictionary of kwargs to create the indicator :param override: Whether to override the existing indicator :return: True if indicator was copied, False if not """ for reserved in ['_id', '_rev', 'last_modified']: if reserved in doc: del doc[reserved] couch_key = cls._generate_couch_key(domain=domain, reverse=True, **doc) existing_indicator = cls.view(cls.indicator_list_view(), reduce=False, include_docs=False, descending=True, limit=1, **couch_key).first() if existing_indicator and not override: return False if existing_indicator: existing_indicator.delete() new_indicator = cls(domain=domain, **doc) new_indicator.last_modified = datetime.datetime.utcnow() new_indicator.save() return True @classmethod @memoized def get_current(cls, namespace, domain, slug, version=None, wrap=True, **kwargs): couch_key = cls._generate_couch_key(namespace=namespace, domain=domain, slug=slug, version=version, reverse=True, **kwargs) results = cache_core.cached_view(cls.get_db(), cls.indicator_list_view(), cache_expire=60 * 60 * 6, reduce=False, include_docs=False, descending=True, **couch_key) doc = results[0] if results else None if wrap and doc: try: doc_class = to_function( doc.get('value', "%s.%s" % (cls._class_path, cls.__name__))) doc_instance = doc_class.get(doc.get('id')) return doc_instance except Exception as e: logging.error( "No matching documents found for indicator %s: %s" % (slug, e)) return None return doc @classmethod def all_slugs(cls, namespace, domain, **kwargs): couch_key = cls._generate_couch_key(namespace=namespace, domain=domain, reverse=True, **kwargs) couch_key['startkey'][0] = couch_key.get('startkey', [])[0] + ' slug' couch_key['endkey'][0] = couch_key.get('endkey', [])[0] + ' slug' data = cls.view(cls.indicator_list_view(), group=True, group_level=cls.key_properties().index('slug') + 2, descending=True, **couch_key).all() return [item.get('key', [])[-1] for item in data] @classmethod @memoized def get_all(cls, namespace, domain, version=None, **kwargs): all_slugs = cls.all_slugs(namespace, domain, **kwargs) all_indicators = list() for slug in all_slugs: indicator = cls.get_current(namespace, domain, slug, version=version, **kwargs) if indicator and issubclass(indicator.__class__, cls): all_indicators.append(indicator) return all_indicators @classmethod def get_all_of_type(cls, namespace, domain, show_only_current=False): key = ["type", namespace, domain, cls.__name__] indicators = cls.view(cls.indicator_list_view(), reduce=False, include_docs=True, startkey=key, endkey=key + [{}]).all() unique = {} for ind in indicators: if ind.base_doc == "CaseIndicatorDefinition": specific_doc = ind.case_type elif ind.base_doc == "FormIndicatorDefinition": specific_doc = ind.xmlns else: specific_doc = "couch" unique["%s.%s.%s" % (ind.slug, ind.namespace, specific_doc)] = ind return unique.values() @classmethod def get_nice_name(cls): return "Indicator Definition"
class Product(Document): """ A product, e.g. "coartem" or "tylenol" """ domain = StringProperty() name = StringProperty() unit = StringProperty() code_ = StringProperty() description = StringProperty() category = StringProperty() program_id = StringProperty() cost = DecimalProperty() product_data = DictProperty() is_archived = BooleanProperty(default=False) last_modified = DateTimeProperty() @classmethod def wrap(cls, data): from corehq.apps.groups.models import dt_no_Z_re # If "Z" is missing because of the Aug 2014 migration, then add it. # cf. Group class last_modified = data.get('last_modified') if last_modified and dt_no_Z_re.match(last_modified): data['last_modified'] += 'Z' return super(Product, cls).wrap(data) @classmethod def save_docs(cls, docs, use_uuids=True, codes_by_domain=None): from corehq.apps.commtrack.util import generate_code codes_by_domain = codes_by_domain or {} def get_codes(domain): if domain not in codes_by_domain: codes_by_domain[domain] = SQLProduct.objects.filter(domain=domain)\ .values_list('code', flat=True).distinct() return codes_by_domain[domain] for doc in docs: doc.last_modified = datetime.utcnow() if not doc['code_']: doc['code_'] = generate_code(doc['name'], get_codes(doc['domain'])) super(Product, cls).save_docs(docs, use_uuids) domains = {doc['domain'] for doc in docs} for domain in domains: cls.clear_caches(domain) bulk_save = save_docs def sync_to_sql(self): properties_to_sync = [ ('product_id', '_id'), 'domain', 'name', 'is_archived', ('code', 'code_'), 'description', 'category', 'program_id', 'cost', ('units', 'unit'), 'product_data', ] # sync properties to SQL version sql_product, _ = SQLProduct.objects.get_or_create(product_id=self._id) for prop in properties_to_sync: if isinstance(prop, tuple): sql_prop, couch_prop = prop else: sql_prop = couch_prop = prop if hasattr(self, couch_prop): setattr(sql_product, sql_prop, getattr(self, couch_prop)) sql_product.save() def save(self, *args, **kwargs): """ Saving a couch version of Product will trigger one way syncing to the SQLProduct version of this product. """ # mark modified time stamp for selective syncing self.last_modified = datetime.utcnow() # generate code if user didn't specify one if not self.code: from corehq.apps.commtrack.util import generate_code self.code = generate_code( self.name, SQLProduct.objects.filter(domain=self.domain).values_list( 'code', flat=True).distinct()) result = super(Product, self).save(*args, **kwargs) self.clear_caches(self.domain) self.sync_to_sql() return result @property def code(self): return self.code_ @code.setter def code(self, val): self.code_ = val.lower() if val else None @classmethod def clear_caches(cls, domain): from casexml.apps.phone.utils import clear_fixture_cache from corehq.apps.products.fixtures import ALL_CACHE_PREFIXES for prefix in ALL_CACHE_PREFIXES: clear_fixture_cache(domain, prefix) @classmethod def by_domain(cls, domain, wrap=True, include_archived=False): queryset = SQLProduct.objects.filter(domain=domain) if not include_archived: queryset = queryset.filter(is_archived=False) return list(queryset.couch_products(wrapped=wrap)) @classmethod def _export_attrs(cls): return [ ('name', str), ('unit', str), 'description', 'category', ('program_id', str), ('cost', lambda a: Decimal(a) if a else None), ] def to_dict(self): from corehq.apps.commtrack.util import encode_if_needed product_dict = {} product_dict['id'] = self._id product_dict['product_id'] = self.code_ for attr in self._export_attrs(): real_attr = attr[0] if isinstance(attr, tuple) else attr product_dict[real_attr] = encode_if_needed(getattr( self, real_attr)) return product_dict def custom_property_dict(self): from corehq.apps.commtrack.util import encode_if_needed property_dict = {} for prop, val in self.product_data.items(): property_dict['data: ' + prop] = encode_if_needed(val) return property_dict def archive(self): """ Mark a product as archived. This will cause it (and its data) to not show up in default Couch and SQL views. """ self.is_archived = True self.save() def unarchive(self): """ Unarchive a product, causing it (and its data) to show up in Couch and SQL views again. """ if self.code: if SQLProduct.objects.filter(domain=self.domain, code=self.code, is_archived=False).exists(): raise DuplicateProductCodeException() self.is_archived = False self.save() @classmethod def from_excel(cls, row, custom_data_validator): if not row: return None id = row.get('id') if id: try: p = cls.get(id) except ResourceNotFound: raise InvalidProductException( _("Product with ID '{product_id}' could not be found!"). format(product_id=id)) else: p = cls() p.code = str(row.get('product_id') or '') for attr in cls._export_attrs(): key = attr[0] if isinstance(attr, tuple) else attr if key in row: val = row[key] if val is None: val = '' if isinstance(attr, tuple): val = attr[1](val) setattr(p, key, val) else: break if not p.code: raise InvalidProductException( _('Product ID is a required field and cannot be blank!')) if not p.name: raise InvalidProductException( _('Product name is a required field and cannot be blank!')) custom_data = row.get('data', {}) error = custom_data_validator(custom_data) if error: raise InvalidProductException(error) p.product_data = custom_data p.product_data.update(row.get('uncategorized_data', {})) return p
class AtomFeedStatus(DocumentSchema): last_polled_at = DateTimeProperty(default=None) # The first time the feed is polled, don't replay all the changes # since OpenMRS was installed. Start from the most recent changes. last_page = StringProperty(default='recent')