class DataSourceConfiguration(UnicodeMixIn, CachedCouchDocumentMixin, Document):
"""
A data source configuration. These map 1:1 with database tables that get created.
Each data source can back an arbitrary number of reports.
"""
domain = StringProperty(required=True)
engine_id = StringProperty(default=UCR_ENGINE_ID)
es_index_settings = SchemaProperty(ElasticSearchIndexSettings)
backend_id = StringProperty(default=UCR_SQL_BACKEND)
referenced_doc_type = StringProperty(required=True)
table_id = StringProperty(required=True)
display_name = StringProperty()
base_item_expression = DictProperty()
configured_filter = DictProperty()
configured_indicators = ListProperty()
named_expressions = DictProperty()
named_filters = DictProperty()
meta = SchemaProperty(DataSourceMeta)
is_deactivated = BooleanProperty(default=False)
last_modified = DateTimeProperty()
asynchronous = BooleanProperty(default=False)
class Meta(object):
# prevent JsonObject from auto-converting dates etc.
string_conversions = ()
def __unicode__(self):
return u'{} - {}'.format(self.domain, self.display_name)
def save(self, **params):
self.last_modified = datetime.utcnow()
super(DataSourceConfiguration, self).save(**params)
def filter(self, document):
filter_fn = self._get_main_filter()
return filter_fn(document, EvaluationContext(document, 0))
def deleted_filter(self, document):
filter_fn = self._get_deleted_filter()
return filter_fn and filter_fn(document, EvaluationContext(document, 0))
@memoized
def _get_main_filter(self):
return self._get_filter([self.referenced_doc_type])
@memoized
def _get_deleted_filter(self):
return self._get_filter(get_deleted_doc_types(self.referenced_doc_type), include_configured=False)
def _get_filter(self, doc_types, include_configured=True):
if not doc_types:
return None
extras = (
[self.configured_filter]
if include_configured and self.configured_filter else []
)
built_in_filters = [
self._get_domain_filter_spec(),
{
'type': 'or',
'filters': [
{
'type': 'property_match',
'property_name': 'doc_type',
'property_value': doc_type,
}
for doc_type in doc_types
],
},
]
return FilterFactory.from_spec(
{
'type': 'and',
'filters': built_in_filters + extras,
},
context=self.get_factory_context(),
)
def _get_domain_filter_spec(self):
return {
'type': 'property_match',
'property_name': 'domain',
'property_value': self.domain,
}
@property
@memoized
def named_expression_objects(self):
named_expression_specs = deepcopy(self.named_expressions)
named_expressions = {}
spec_error = None
while named_expression_specs:
number_generated = 0
for name, expression in named_expression_specs.items():
try:
named_expressions[name] = ExpressionFactory.from_spec(
expression,
FactoryContext(named_expressions=named_expressions, named_filters={})
)
number_generated += 1
del named_expression_specs[name]
except BadSpecError as spec_error:
# maybe a nested name resolution issue, try again on the next pass
pass
if number_generated == 0 and named_expression_specs:
# we unsuccessfully generated anything on this pass and there are still unresolved
# references. we have to fail.
assert spec_error is not None
raise spec_error
return named_expressions
@property
@memoized
def named_filter_objects(self):
return {name: FilterFactory.from_spec(filter, FactoryContext(self.named_expression_objects, {}))
for name, filter in self.named_filters.items()}
def get_factory_context(self):
return FactoryContext(self.named_expression_objects, self.named_filter_objects)
@property
@memoized
def default_indicators(self):
default_indicators = [IndicatorFactory.from_spec({
"column_id": "doc_id",
"type": "expression",
"display_name": "document id",
"datatype": "string",
"is_nullable": False,
"is_primary_key": True,
"expression": {
"type": "root_doc",
"expression": {
"type": "property_name",
"property_name": "_id"
}
}
}, self.get_factory_context())]
default_indicators.append(IndicatorFactory.from_spec({
"type": "inserted_at",
}, self.get_factory_context()))
if self.base_item_expression:
default_indicators.append(IndicatorFactory.from_spec({
"type": "repeat_iteration",
}, self.get_factory_context()))
return default_indicators
@property
@memoized
def indicators(self):
return CompoundIndicator(
self.display_name,
self.default_indicators + [
IndicatorFactory.from_spec(indicator, self.get_factory_context())
for indicator in self.configured_indicators
]
)
@property
@memoized
def parsed_expression(self):
if self.base_item_expression:
return ExpressionFactory.from_spec(self.base_item_expression, context=self.get_factory_context())
return None
def get_columns(self):
return self.indicators.get_columns()
def get_items(self, document, eval_context=None):
if self.filter(document):
if not self.base_item_expression:
return [document]
else:
result = self.parsed_expression(document, eval_context)
if result is None:
return []
elif isinstance(result, list):
return result
else:
return [result]
else:
return []
def get_all_values(self, doc, eval_context=None):
if not eval_context:
eval_context = EvaluationContext(doc)
rows = []
for item in self.get_items(doc, eval_context):
indicators = self.indicators.get_values(item, eval_context)
rows.append(indicators)
eval_context.increment_iteration()
return rows
def get_report_count(self):
"""
Return the number of ReportConfigurations that reference this data source.
"""
return ReportConfiguration.count_by_data_source(self.domain, self._id)
def validate(self, required=True):
super(DataSourceConfiguration, self).validate(required)
# these two properties implicitly call other validation
self._get_main_filter()
self._get_deleted_filter()
# validate indicators and column uniqueness
columns = [c.id for c in self.indicators.get_columns()]
unique_columns = set(columns)
if len(columns) != len(unique_columns):
for column in set(columns):
columns.remove(column)
raise BadSpecError(_('Report contains duplicate column ids: {}').format(', '.join(set(columns))))
if self.referenced_doc_type not in VALID_REFERENCED_DOC_TYPES:
raise BadSpecError(
_('Report contains invalid referenced_doc_type: {}').format(self.referenced_doc_type))
self.parsed_expression
@classmethod
def by_domain(cls, domain):
return get_datasources_for_domain(domain)
@classmethod
def all_ids(cls):
return [res['id'] for res in cls.get_db().view('userreports/data_sources_by_build_info',
reduce=False, include_docs=False)]
@classmethod
def all(cls):
for result in iter_docs(cls.get_db(), cls.all_ids()):
yield cls.wrap(result)
@property
def is_static(self):
return id_is_static(self._id)
def deactivate(self):
if not self.is_static:
self.is_deactivated = True
self.save()
get_indicator_adapter(self).drop_table()
def get_es_index_settings(self):
es_index_settings = self.es_index_settings.to_json()
es_index_settings.pop('doc_type')
return {"settings": es_index_settings}
class Repeater(QuickCachedDocumentMixin, Document):
"""
Represents the configuration of a repeater. Will specify the URL to forward to and
other properties of the configuration.
"""
base_doc = 'Repeater'
domain = StringProperty()
connection_settings_id = IntegerProperty(required=False, default=None)
# TODO: Delete the following properties once all Repeaters have been
# migrated to ConnectionSettings. (2020-05-16)
url = StringProperty()
auth_type = StringProperty(choices=(BASIC_AUTH, DIGEST_AUTH, OAUTH1,
BEARER_AUTH),
required=False)
username = StringProperty()
password = StringProperty() # See also plaintext_password()
skip_cert_verify = BooleanProperty(default=False) # See also verify()
notify_addresses_str = StringProperty(
default="") # See also notify_addresses()
format = StringProperty()
friendly_name = _("Data")
paused = BooleanProperty(default=False)
# TODO: Use to collect stats to determine whether remote endpoint is valid
started_at = DateTimeProperty(default=datetime.utcnow)
last_success_at = DateTimeProperty(required=False, default=None)
failure_streak = IntegerProperty(default=0)
payload_generator_classes = ()
_has_config = False
def __str__(self):
return f'{self.__class__.__name__}: {self.name}'
def __repr__(self):
return f"<{self.__class__.__name__} {self._id} {self.name!r}>"
@cached_property
def connection_settings(self):
if not self.connection_settings_id:
return self.create_connection_settings()
return ConnectionSettings.objects.get(pk=self.connection_settings_id)
@property
def name(self):
return self.connection_settings.name
@classmethod
def available_for_domain(cls, domain):
"""Returns whether this repeater can be used by a particular domain
"""
return True
def get_pending_record_count(self):
return get_pending_repeat_record_count(self.domain, self._id)
def get_failure_record_count(self):
return get_failure_repeat_record_count(self.domain, self._id)
def get_success_record_count(self):
return get_success_repeat_record_count(self.domain, self._id)
def get_cancelled_record_count(self):
return get_cancelled_repeat_record_count(self.domain, self._id)
def _format_or_default_format(self):
from corehq.motech.repeaters.repeater_generators import RegisterGenerator
return self.format or RegisterGenerator.default_format_by_repeater(
self.__class__)
def _get_payload_generator(self, payload_format):
from corehq.motech.repeaters.repeater_generators import RegisterGenerator
gen = RegisterGenerator.generator_class_by_repeater_format(
self.__class__, payload_format)
return gen(self)
@property
@memoized
def generator(self):
return self._get_payload_generator(self._format_or_default_format())
def payload_doc(self, repeat_record):
raise NotImplementedError
@memoized
def get_payload(self, repeat_record):
return self.generator.get_payload(repeat_record,
self.payload_doc(repeat_record))
def get_attempt_info(self, repeat_record):
return None
def register(self, payload, next_check=None):
if not self.allowed_to_forward(payload):
return
now = datetime.utcnow()
repeat_record = RepeatRecord(repeater_id=self.get_id,
repeater_type=self.doc_type,
domain=self.domain,
registered_on=now,
next_check=next_check or now,
payload_id=payload.get_id)
repeat_record.save()
if next_check is None:
repeat_record.attempt_forward_now()
return repeat_record
def allowed_to_forward(self, payload):
"""
Return True/False depending on whether the payload meets forawrding criteria or not
"""
return True
def clear_caches(self):
super(Repeater, self).clear_caches()
# Also expire for cases repeater is fetched using Repeater class.
# The quick cache called in clear_cache also check on relies of doc class
# so in case the class is set as Repeater it is not expired like in edit forms.
# So expire it explicitly here with Repeater class as well.
Repeater.get.clear(Repeater, self._id)
if self.__class__ == Repeater:
cls = self.get_class_from_doc_type(self.doc_type)
else:
cls = self.__class__
# clear cls.by_domain (i.e. filtered by doc type)
Repeater.by_domain.clear(cls, self.domain)
# clear Repeater.by_domain (i.e. not filtered by doc type)
Repeater.by_domain.clear(Repeater, self.domain)
@classmethod
@quickcache(['cls.__name__', 'domain'], timeout=5 * 60, memoize_timeout=10)
def by_domain(cls, domain):
key = [domain]
if cls.__name__ in get_all_repeater_types():
key.append(cls.__name__)
elif cls.__name__ == Repeater.__name__:
# In this case the wrap function delegates to the
# appropriate sub-repeater types.
pass
else:
# Any repeater type can be posted to the API, and the installed apps
# determine whether we actually know about it.
# But if we do not know about it, then may as well return nothing now
return []
raw_docs = cls.view('repeaters/repeaters',
startkey=key,
endkey=key + [{}],
include_docs=True,
reduce=False,
wrap_doc=False)
return [
cls.wrap(repeater_doc['doc']) for repeater_doc in raw_docs
if cls.get_class_from_doc_type(repeater_doc['doc']['doc_type'])
]
@classmethod
def wrap(cls, data):
data.pop('name', None)
if cls.__name__ == Repeater.__name__:
cls_ = cls.get_class_from_doc_type(data['doc_type'])
if cls_:
return cls_.wrap(data)
else:
raise ResourceNotFound('Unknown repeater type: %s' % data)
else:
return super(Repeater, cls).wrap(data)
@staticmethod
def get_class_from_doc_type(doc_type):
doc_type = doc_type.replace(DELETED_SUFFIX, '')
repeater_types = get_all_repeater_types()
if doc_type in repeater_types:
return repeater_types[doc_type]
else:
return None
def retire(self):
if DELETED_SUFFIX not in self['doc_type']:
self['doc_type'] += DELETED_SUFFIX
if DELETED_SUFFIX not in self['base_doc']:
self['base_doc'] += DELETED_SUFFIX
self.paused = False
self.save()
def pause(self):
self.paused = True
self.save()
def resume(self):
self.paused = False
self.save()
def get_url(self, repeat_record):
# to be overridden
return self.connection_settings.url
def allow_retries(self, response):
"""Whether to requeue the repeater when it fails
"""
return True
def get_headers(self, repeat_record):
# to be overridden
return self.generator.get_headers()
@property
def plaintext_password(self):
if self.password is None:
return ''
if self.password.startswith('${algo}$'.format(algo=ALGO_AES)):
ciphertext = self.password.split('$', 2)[2]
return b64_aes_decrypt(ciphertext)
return self.password
@property
def verify(self):
return not self.skip_cert_verify
def send_request(self, repeat_record, payload):
url = self.get_url(repeat_record)
return simple_post(
self.domain,
url,
payload,
headers=self.get_headers(repeat_record),
auth_manager=self.connection_settings.get_auth_manager(),
verify=self.verify,
notify_addresses=self.connection_settings.notify_addresses,
payload_id=repeat_record.payload_id,
)
def fire_for_record(self, repeat_record):
payload = self.get_payload(repeat_record)
try:
response = self.send_request(repeat_record, payload)
except (Timeout, ConnectionError) as error:
log_repeater_timeout_in_datadog(self.domain)
return self.handle_response(RequestConnectionError(error),
repeat_record)
except Exception as e:
return self.handle_response(e, repeat_record)
else:
return self.handle_response(response, repeat_record)
def handle_response(self, result, repeat_record):
"""
route the result to the success, failure, or exception handlers
result may be either a response object or an exception
"""
if isinstance(result, Exception):
attempt = repeat_record.handle_exception(result)
elif _is_response(
result) and 200 <= result.status_code < 300 or result is True:
attempt = repeat_record.handle_success(result)
else:
attempt = repeat_record.handle_failure(result)
return attempt
@property
def form_class_name(self):
"""
Return the name of the class whose edit form this class uses.
(Most classes that extend CaseRepeater, and all classes that
extend FormRepeater, use the same form.)
"""
return self.__class__.__name__
def create_connection_settings(self):
if self.connection_settings_id:
return # Nothing to do
conn = ConnectionSettings(
domain=self.domain,
name=self.url,
url=self.url,
auth_type=self.auth_type,
username=self.username,
skip_cert_verify=self.skip_cert_verify,
notify_addresses_str=self.notify_addresses_str,
)
# Allow ConnectionSettings to encrypt old Repeater passwords:
conn.plaintext_password = self.plaintext_password
conn.save()
self.connection_settings_id = conn.id
self.save()
return conn
class CommCareMultimedia(BlobMixin, SafeSaveDocument):
"""
The base object of all CommCare Multimedia
"""
file_hash = StringProperty() # use this to search for multimedia in couch
aux_media = SchemaListProperty(AuxMedia)
last_modified = DateTimeProperty()
valid_domains = StringListProperty(
) # A list of domains that uses this file
owners = StringListProperty(
default=[]) # list of domains that uploaded this file
licenses = SchemaListProperty(HQMediaLicense, default=[])
shared_by = StringListProperty(
default=[]) # list of domains that can share this file
tags = DictProperty(default={}) # dict of string lists
_blobdb_type_code = CODES.multimedia
@classmethod
def get(cls, docid, rev=None, db=None, dynamic_properties=True):
# copied and tweaked from the superclass's method
if not db:
db = cls.get_db()
cls._allow_dynamic_properties = dynamic_properties
# on cloudant don't get the doc back until all nodes agree
# on the copy, to avoid race conditions
extras = get_safe_read_kwargs()
return db.get(docid, rev=rev, wrapper=cls.wrap, **extras)
@property
def is_shared(self):
return len(self.shared_by) > 0
@property
def license(self):
return self.licenses[0] if self.licenses else None
@retry_resource(3)
def update_or_add_license(self,
domain,
type="",
author="",
attribution_notes="",
org="",
should_save=True):
for license in self.licenses:
if license.domain == domain:
license.type = type or license.type
license.author = author or license.author
license.organization = org or license.organization
license.attribution_notes = attribution_notes or license.attribution_notes
break
else:
license = HQMediaLicense(domain=domain,
type=type,
author=author,
attribution_notes=attribution_notes,
organization=org)
self.licenses.append(license)
if should_save:
self.save()
def url(self):
return reverse("hqmedia_download", args=[self.doc_type, self._id])
def attach_data(self,
data,
original_filename=None,
username=None,
attachment_id=None,
media_meta=None):
"""
This creates the auxmedia attachment with the downloaded data.
"""
self.last_modified = datetime.utcnow()
if not attachment_id:
attachment_id = self.file_hash
if not self.blobs or attachment_id not in self.blobs:
if not getattr(self, '_id'):
# put_attchment blows away existing data, so make sure an id has been assigned
# to this guy before we do it. This is the usual path; remote apps are the exception.
self.save()
self.put_attachment(
data,
attachment_id,
content_type=self.get_mime_type(data,
filename=original_filename),
domain=SHARED_DOMAIN,
)
new_media = AuxMedia()
new_media.uploaded_date = datetime.utcnow()
new_media.attachment_id = attachment_id
new_media.uploaded_filename = original_filename
new_media.uploaded_by = username
new_media.checksum = self.file_hash
if media_meta:
new_media.media_meta = media_meta
self.aux_media.append(new_media)
self.save()
return True
def add_domain(self, domain, owner=None, **kwargs):
if len(self.owners) == 0:
# this is intended to simulate migration--if it happens that a media file somehow gets no more owners
# (which should be impossible) it will transfer ownership to all copiers... not necessarily a bad thing,
# just something to be aware of
self.owners = self.valid_domains
if owner and domain not in self.owners:
self.owners.append(domain)
elif not owner and domain in self.owners:
self.owners.remove(domain)
if domain in self.owners:
shared = kwargs.get('shared', '')
if shared and domain not in self.shared_by:
self.shared_by.append(domain)
elif not shared and shared != '' and domain in self.shared_by:
self.shared_by.remove(domain)
if kwargs.get('tags', ''):
self.tags[domain] = kwargs['tags']
if domain not in self.valid_domains:
self.valid_domains.append(domain)
self.save()
def get_display_file(self, return_type=True):
if self.attachment_id:
data = self.fetch_attachment(self.attachment_id,
stream=True).read()
if return_type:
content_type = self.blobs[self.attachment_id].content_type
return data, content_type
else:
return data
return None
def get_file_extension(self):
extension = ''
if self.aux_media:
extension = '.{}'.format(
self.aux_media[-1]['uploaded_filename'].split(".")[-1])
return extension
def get_media_info(self, path, is_updated=False, original_path=None):
return {
"path":
path,
"uid":
self.file_hash,
"m_id":
self._id,
"url":
reverse("hqmedia_download",
args=[self.__class__.__name__, self._id]),
"updated":
is_updated,
"original_path":
original_path,
"icon_class":
self.get_icon_class(),
"media_type":
self.get_nice_name(),
"humanized_content_length":
filesizeformat(self.content_length),
}
@property
def attachment_id(self):
if not self.aux_media:
return None
ids = set([aux.attachment_id for aux in self.aux_media])
assert len(ids) == 1
return ids.pop()
@property
def content_length(self):
if self.attachment_id:
return self.blobs[self.attachment_id].content_length
@classmethod
def get_mime_type(cls, data, filename=None):
mime = magic.Magic(mime=True)
mime_type = mime.from_buffer(data)
if mime_type.startswith('application') and filename is not None:
guessed_type = mimetypes.guess_type(filename)
mime_type = guessed_type[0] if guessed_type[0] else mime_type
return mime_type
@classmethod
def get_base_mime_type(cls, data, filename=None):
mime_type = cls.get_mime_type(data, filename=filename)
return mime_type.split('/')[0] if mime_type else None
@classmethod
def generate_hash(cls, data):
return hashlib.md5(data).hexdigest()
@classmethod
def get_by_hash(cls, file_hash):
result = cls.view('hqmedia/by_hash', key=file_hash,
include_docs=True).first()
if not result:
result = cls()
result.file_hash = file_hash
return result
@classmethod
def get_by_data(cls, data):
file_hash = cls.generate_hash(data)
return cls.get_by_hash(file_hash)
@classmethod
def get_doc_types(cls):
return ('CommCareImage', 'CommCareAudio', 'CommCareVideo')
@classmethod
def get_doc_class(cls, doc_type):
return {
'CommCareImage': CommCareImage,
'CommCareAudio': CommCareAudio,
'CommCareVideo': CommCareVideo,
'CommCareMultimedia': CommCareMultimedia,
}[doc_type]
@classmethod
def get_class_by_data(cls, data, filename=None):
return {
'image': CommCareImage,
'audio': CommCareAudio,
'video': CommCareVideo,
}.get(cls.get_base_mime_type(data, filename=filename))
@classmethod
def get_form_path(cls, path, lowercase=False):
path = path.strip()
if lowercase:
path = path.lower()
if path.startswith(MULTIMEDIA_PREFIX):
return path
if path.startswith('/'):
path = path[1:]
return "%s%s" % (MULTIMEDIA_PREFIX, path)
@classmethod
def get_standard_path(cls, path):
return path.replace(MULTIMEDIA_PREFIX, "")
@classmethod
def wrap(cls, data):
should_save = False
if data.get('tags') == []:
data['tags'] = {}
if not data.get('owners'):
data['owners'] = data.get('valid_domains', [])
if isinstance(data.get('licenses', ''), dict):
# need to migrate licncses from old format to new format
# old: {"mydomain": "public", "yourdomain": "cc"}
migrated = [HQMediaLicense(domain=domain, type=type)._doc \
for domain, type in data["licenses"].items()]
data['licenses'] = migrated
# deprecating support for public domain license
if isinstance(data.get("licenses", ""),
list) and len(data["licenses"]) > 0:
if data["licenses"][0].get("type", "") == "public":
data["licenses"][0]["type"] = "cc"
should_save = True
self = super(CommCareMultimedia, cls).wrap(data)
if should_save:
self.save()
return self
@classmethod
def get_nice_name(cls):
return _("Generic Multimedia")
@classmethod
def get_icon_class(cls):
return "fa fa-desktop"
class Product(Document):
"""
A product, e.g. "coartem" or "tylenol"
"""
domain = StringProperty()
name = StringProperty()
unit = StringProperty()
code_ = StringProperty() # todo: why the hell is this code_ and not code
description = StringProperty()
category = StringProperty()
program_id = StringProperty()
cost = DecimalProperty()
product_data = DictProperty()
is_archived = BooleanProperty(default=False)
last_modified = DateTimeProperty()
@classmethod
def wrap(cls, data):
from corehq.apps.groups.models import dt_no_Z_re
# If "Z" is missing because of the Aug 2014 migration, then add it.
# cf. Group class
last_modified = data.get('last_modified')
if last_modified and dt_no_Z_re.match(last_modified):
data['last_modified'] += 'Z'
return super(Product, cls).wrap(data)
@classmethod
def save_docs(cls,
docs,
use_uuids=True,
all_or_nothing=False,
codes_by_domain=None):
from corehq.apps.commtrack.util import generate_code
codes_by_domain = codes_by_domain or {}
def get_codes(domain):
if domain not in codes_by_domain:
codes_by_domain[domain] = SQLProduct.objects.filter(domain=domain)\
.values_list('code', flat=True).distinct()
return codes_by_domain[domain]
for doc in docs:
if not doc['code_']:
doc['code_'] = generate_code(doc['name'],
get_codes(doc['domain']))
super(Product, cls).save_docs(docs, use_uuids, all_or_nothing)
bulk_save = save_docs
def sync_to_sql(self):
properties_to_sync = [
('product_id', '_id'),
'domain',
'name',
'is_archived',
('code', 'code_'),
'description',
'category',
'program_id',
'cost',
('units', 'unit'),
'product_data',
]
# sync properties to SQL version
sql_product, _ = SQLProduct.objects.get_or_create(product_id=self._id)
for prop in properties_to_sync:
if isinstance(prop, tuple):
sql_prop, couch_prop = prop
else:
sql_prop = couch_prop = prop
if hasattr(self, couch_prop):
setattr(sql_product, sql_prop, getattr(self, couch_prop))
sql_product.save()
def save(self, *args, **kwargs):
"""
Saving a couch version of Product will trigger
one way syncing to the SQLProduct version of this
product.
"""
# mark modified time stamp for selective syncing
self.last_modified = datetime.utcnow()
# generate code if user didn't specify one
if not self.code:
from corehq.apps.commtrack.util import generate_code
self.code = generate_code(
self.name,
SQLProduct.objects.filter(domain=self.domain).values_list(
'code', flat=True).distinct())
result = super(Product, self).save(*args, **kwargs)
self.sync_to_sql()
return result
@property
def code(self):
return self.code_
@code.setter
def code(self, val):
self.code_ = val.lower() if val else None
@classmethod
def get_by_code(cls, domain, code):
if not code:
return None
result = cls.view("commtrack/product_by_code",
key=[domain, code.lower()],
include_docs=True).first()
return result
@classmethod
def by_program_id(cls, domain, prog_id, wrap=True, **kwargs):
kwargs.update(
dict(view_name='commtrack/product_by_program_id',
key=[domain, prog_id],
include_docs=True))
if wrap:
return Product.view(**kwargs)
else:
return [
row["doc"] for row in Product.view(wrap_doc=False, **kwargs)
]
@classmethod
def by_domain(cls, domain, wrap=True, include_archived=False, **kwargs):
"""
Gets all products in a domain.
By default this filters out any archived products.
WARNING: this doesn't paginate correctly; it filters after the query
If you need pagination, use SQLProduct instead
"""
kwargs.update(
dict(view_name='commtrack/products',
startkey=[domain],
endkey=[domain, {}],
include_docs=True))
if wrap:
products = Product.view(**kwargs)
if not include_archived:
return filter(lambda p: not p.is_archived, products)
else:
return products
else:
if not include_archived:
return [
row["doc"]
for row in Product.view(wrap_doc=False, **kwargs)
if not row["doc"].get('is_archived', False)
]
else:
return [
row["doc"]
for row in Product.view(wrap_doc=False, **kwargs)
]
@classmethod
def archived_by_domain(cls, domain, wrap=True, **kwargs):
products = cls.by_domain(domain, wrap, kwargs)
if wrap:
return filter(lambda p: p.is_archived, products)
else:
return [p for p in products if p.get('is_archived', False)]
@classmethod
def ids_by_domain(cls, domain):
"""
Gets all product ids in a domain.
"""
view_results = Product.get_db().view(
'commtrack/products',
startkey=[domain],
endkey=[domain, {}],
include_docs=False,
)
return [row['id'] for row in view_results]
@classmethod
def count_by_domain(cls, domain):
"""
Gets count of products in a domain
"""
# todo: we should add a reduce so we can get this out of couch
return len(cls.ids_by_domain(domain))
@classmethod
def _export_attrs(cls):
return [
('name', unicode),
('unit', unicode),
'description',
'category',
('program_id', str),
('cost', lambda a: Decimal(a) if a else None),
]
def to_dict(self):
from corehq.apps.commtrack.util import encode_if_needed
product_dict = {}
product_dict['id'] = self._id
product_dict['product_id'] = self.code_
for attr in self._export_attrs():
real_attr = attr[0] if isinstance(attr, tuple) else attr
product_dict[real_attr] = encode_if_needed(getattr(
self, real_attr))
return product_dict
def custom_property_dict(self):
from corehq.apps.commtrack.util import encode_if_needed
property_dict = {}
for prop, val in self.product_data.iteritems():
property_dict['data: ' + prop] = encode_if_needed(val)
return property_dict
def archive(self):
"""
Mark a product as archived. This will cause it (and its data)
to not show up in default Couch and SQL views.
"""
self.is_archived = True
self.save()
def unarchive(self):
"""
Unarchive a product, causing it (and its data) to show
up in Couch and SQL views again.
"""
if self.code:
if SQLProduct.objects.filter(code=self.code,
is_archived=False).exists():
raise DuplicateProductCodeException()
self.is_archived = False
self.save()
@classmethod
def from_excel(cls, row, custom_data_validator):
if not row:
return None
id = row.get('id')
if id:
try:
p = cls.get(id)
except ResourceNotFound:
raise InvalidProductException(
_("Product with ID '{product_id}' could not be found!").
format(product_id=id))
else:
p = cls()
p.code = str(row.get('product_id') or '')
for attr in cls._export_attrs():
key = attr[0] if isinstance(attr, tuple) else attr
if key in row:
val = row[key]
if val is None:
val = ''
if isinstance(attr, tuple):
val = attr[1](val)
setattr(p, key, val)
else:
break
if not p.code:
raise InvalidProductException(
_('Product ID is a required field and cannot be blank!'))
if not p.name:
raise InvalidProductException(
_('Product name is a required field and cannot be blank!'))
custom_data = row.get('data', {})
error = custom_data_validator(custom_data)
if error:
raise InvalidProductException(error)
p.product_data = custom_data
p.product_data.update(row.get('uncategorized_data', {}))
return p
class RepeatRecord(Document):
"""
An record of a particular instance of something that needs to be forwarded
with a link to the proper repeater object
"""
domain = StringProperty()
repeater_id = StringProperty()
repeater_type = StringProperty()
payload_id = StringProperty()
overall_tries = IntegerProperty(default=0)
max_possible_tries = IntegerProperty(default=3)
attempts = ListProperty(RepeatRecordAttempt)
cancelled = BooleanProperty(default=False)
registered_on = DateTimeProperty()
last_checked = DateTimeProperty()
failure_reason = StringProperty()
next_check = DateTimeProperty()
succeeded = BooleanProperty(default=False)
@property
def record_id(self):
return self._id
@classmethod
def wrap(cls, data):
should_bootstrap_attempts = ('attempts' not in data)
self = super(RepeatRecord, cls).wrap(data)
if should_bootstrap_attempts and self.last_checked:
assert not self.attempts
self.attempts = [
RepeatRecordAttempt(
cancelled=self.cancelled,
datetime=self.last_checked,
failure_reason=self.failure_reason,
success_response=None,
next_check=self.next_check,
succeeded=self.succeeded,
)
]
return self
@property
@memoized
def repeater(self):
try:
return Repeater.get(self.repeater_id)
except ResourceNotFound:
return None
@property
def url(self):
warnings.warn(
"RepeatRecord.url is deprecated. Use Repeater.get_url instead",
DeprecationWarning)
if self.repeater:
return self.repeater.get_url(self)
@property
def state(self):
state = RECORD_PENDING_STATE
if self.succeeded:
state = RECORD_SUCCESS_STATE
elif self.cancelled:
state = RECORD_CANCELLED_STATE
elif self.failure_reason:
state = RECORD_FAILURE_STATE
return state
@classmethod
def all(cls, domain=None, due_before=None, limit=None):
json_now = json_format_datetime(due_before or datetime.utcnow())
repeat_records = RepeatRecord.view(
"repeaters/repeat_records_by_next_check",
startkey=[domain],
endkey=[domain, json_now, {}],
include_docs=True,
reduce=False,
limit=limit,
)
return repeat_records
@classmethod
def count(cls, domain=None):
results = RepeatRecord.view(
"repeaters/repeat_records_by_next_check",
startkey=[domain],
endkey=[domain, {}],
reduce=True,
).one()
return results['value'] if results else 0
def add_attempt(self, attempt):
self.attempts.append(attempt)
self.last_checked = attempt.datetime
self.next_check = attempt.next_check
self.succeeded = attempt.succeeded
self.cancelled = attempt.cancelled
self.failure_reason = attempt.failure_reason
def get_numbered_attempts(self):
for i, attempt in enumerate(self.attempts):
yield i + 1, attempt
def postpone_by(self, duration):
self.last_checked = datetime.utcnow()
self.next_check = self.last_checked + duration
self.save()
def make_set_next_try_attempt(self, failure_reason):
# we use an exponential back-off to avoid submitting to bad urls
# too frequently.
assert self.succeeded is False
assert self.next_check is not None
window = timedelta(minutes=0)
if self.last_checked:
window = self.next_check - self.last_checked
window += (window // 2) # window *= 1.5
if window < MIN_RETRY_WAIT:
window = MIN_RETRY_WAIT
elif window > MAX_RETRY_WAIT:
window = MAX_RETRY_WAIT
now = datetime.utcnow()
return RepeatRecordAttempt(
cancelled=False,
datetime=now,
failure_reason=failure_reason,
success_response=None,
next_check=now + window,
succeeded=False,
)
def try_now(self):
# try when we haven't succeeded and either we've
# never checked, or it's time to check again
return not self.succeeded
def get_payload(self):
return self.repeater.get_payload(self)
def get_attempt_info(self):
return self.repeater.get_attempt_info(self)
def handle_payload_exception(self, exception):
now = datetime.utcnow()
return RepeatRecordAttempt(
cancelled=True,
datetime=now,
failure_reason=six.text_type(exception),
success_response=None,
next_check=None,
succeeded=False,
)
def fire(self, force_send=False):
if self.try_now() or force_send:
self.overall_tries += 1
try:
attempt = self.repeater.fire_for_record(self)
except Exception as e:
log_repeater_error_in_datadog(self.domain,
status_code=None,
repeater_type=self.repeater_type)
attempt = self.handle_payload_exception(e)
raise
finally:
# pycharm warns attempt might not be defined.
# that'll only happen if fire_for_record raise a non-Exception exception (e.g. SIGINT)
# or handle_payload_exception raises an exception. I'm okay with that. -DMR
self.add_attempt(attempt)
self.save()
@staticmethod
def _format_response(response):
return '{}: {}.\n{}'.format(response.status_code, response.reason,
getattr(response, 'content', None))
def handle_success(self, response):
"""Do something with the response if the repeater succeeds
"""
now = datetime.utcnow()
log_repeater_success_in_datadog(
self.domain, response.status_code if response else None,
self.repeater_type)
return RepeatRecordAttempt(
cancelled=False,
datetime=now,
failure_reason=None,
success_response=self._format_response(response)
if response else None,
next_check=None,
succeeded=True,
info=self.get_attempt_info(),
)
def handle_failure(self, response):
"""Do something with the response if the repeater fails
"""
return self._make_failure_attempt(self._format_response(response),
response)
def handle_exception(self, exception):
"""handle internal exceptions
"""
return self._make_failure_attempt(six.text_type(exception), None)
def _make_failure_attempt(self, reason, response):
log_repeater_error_in_datadog(
self.domain, response.status_code if response else None,
self.repeater_type)
if self.repeater.allow_retries(
response) and self.overall_tries < self.max_possible_tries:
return self.make_set_next_try_attempt(reason)
else:
now = datetime.utcnow()
return RepeatRecordAttempt(
cancelled=True,
datetime=now,
failure_reason=reason,
success_response=None,
next_check=None,
succeeded=False,
info=self.get_attempt_info(),
)
def cancel(self):
self.next_check = None
self.cancelled = True
def requeue(self):
self.cancelled = False
self.succeeded = False
self.failure_reason = ''
self.overall_tries = 0
self.next_check = datetime.utcnow()
class Domain(QuickCachedDocumentMixin, BlobMixin, Document, SnapshotMixin):
"""
Domain is the highest level collection of people/stuff
in the system. Pretty much everything happens at the
domain-level, including user membership, permission to
see data, reports, charts, etc.
Exceptions: accounting has some models that combine multiple domains,
which make "enterprise" multi-domain features like the enterprise dashboard possible.
Naming conventions:
Most often, variables representing domain names are named `domain`, and
variables representing domain objects are named `domain_obj`. New code should
follow this convention, unless it's in an area that consistently uses `domain`
for the object and `domain_name` for the string.
There's a `project` attribute attached to requests that's a domain object.
In spite of this, don't use `project` in new code.
"""
_blobdb_type_code = BLOB_CODES.domain
name = StringProperty()
is_active = BooleanProperty()
date_created = DateTimeProperty()
default_timezone = StringProperty(
default=getattr(settings, "TIME_ZONE", "UTC"))
case_sharing = BooleanProperty(default=False)
secure_submissions = BooleanProperty(default=False)
cloudcare_releases = StringProperty(
choices=['stars', 'nostars', 'default'], default='default')
organization = StringProperty()
hr_name = StringProperty() # the human-readable name for this project
project_description = StringProperty() # Brief description of the project
creating_user = StringProperty(
) # username of the user who created this domain
# domain metadata
project_type = StringProperty() # e.g. MCH, HIV
customer_type = StringProperty() # plus, full, etc.
is_test = StringProperty(choices=["true", "false", "none"], default="none")
description = StringProperty()
short_description = StringProperty()
is_shared = BooleanProperty(default=False)
commtrack_enabled = BooleanProperty(default=False)
call_center_config = SchemaProperty(CallCenterProperties)
restrict_superusers = BooleanProperty(default=False)
allow_domain_requests = BooleanProperty(default=False)
location_restriction_for_users = BooleanProperty(default=False)
usercase_enabled = BooleanProperty(default=False)
hipaa_compliant = BooleanProperty(default=False)
use_sql_backend = BooleanProperty(default=False)
first_domain_for_user = BooleanProperty(default=False)
case_display = SchemaProperty(CaseDisplaySettings)
# CommConnect settings
commconnect_enabled = BooleanProperty(default=False)
survey_management_enabled = BooleanProperty(default=False)
# Whether or not a case can register via sms
sms_case_registration_enabled = BooleanProperty(default=False)
# Case type to apply to cases registered via sms
sms_case_registration_type = StringProperty()
# Owner to apply to cases registered via sms
sms_case_registration_owner_id = StringProperty()
# Submitting user to apply to cases registered via sms
sms_case_registration_user_id = StringProperty()
# Whether or not a mobile worker can register via sms
sms_mobile_worker_registration_enabled = BooleanProperty(default=False)
use_default_sms_response = BooleanProperty(default=False)
default_sms_response = StringProperty()
chat_message_count_threshold = IntegerProperty()
custom_chat_template = StringProperty(
) # See settings.CUSTOM_CHAT_TEMPLATES
custom_case_username = StringProperty(
) # Case property to use when showing the case's name in a chat window
# If empty, sms can be sent at any time. Otherwise, only send during
# these windows of time. SMS_QUEUE_ENABLED must be True in localsettings
# for this be considered.
restricted_sms_times = SchemaListProperty(DayTimeWindow)
# If empty, this is ignored. Otherwise, the framework will make sure
# that during these days/times, no automated outbound sms will be sent
# to someone if they have sent in an sms within sms_conversation_length
# minutes. Outbound sms sent from a user in a chat window, however, will
# still be sent. This is meant to prevent chat conversations from being
# interrupted by automated sms reminders.
# SMS_QUEUE_ENABLED must be True in localsettings for this to be
# considered.
sms_conversation_times = SchemaListProperty(DayTimeWindow)
# In minutes, see above.
sms_conversation_length = IntegerProperty(default=10)
# Set to True to prevent survey questions and answers form being seen in
# SMS chat windows.
filter_surveys_from_chat = BooleanProperty(default=False)
# The below option only matters if filter_surveys_from_chat = True.
# If set to True, invalid survey responses will still be shown in the chat
# window, while questions and valid responses will be filtered out.
show_invalid_survey_responses_in_chat = BooleanProperty(default=False)
# If set to True, if a message is read by anyone it counts as being read by
# everyone. Set to False so that a message is only counted as being read
# for a user if only that user has read it.
count_messages_as_read_by_anyone = BooleanProperty(default=False)
enable_registration_welcome_sms_for_case = BooleanProperty(default=False)
enable_registration_welcome_sms_for_mobile_worker = BooleanProperty(
default=False)
sms_survey_date_format = StringProperty()
granted_messaging_access = BooleanProperty(default=False)
# Allowed outbound SMS per day
# If this is None, then the default is applied. See get_daily_outbound_sms_limit()
custom_daily_outbound_sms_limit = IntegerProperty()
# Allowed number of case updates or closes from automatic update rules in the daily rule run.
# If this value is None, the value in settings.MAX_RULE_UPDATES_IN_ONE_RUN is used.
auto_case_update_limit = IntegerProperty()
# Allowed number of max OData feeds that this domain can create.
# If this value is None, the value in settings.DEFAULT_ODATA_FEED_LIMIT is used
odata_feed_limit = IntegerProperty()
# exchange/domain copying stuff
is_snapshot = BooleanProperty(default=False)
is_approved = BooleanProperty(default=False)
snapshot_time = DateTimeProperty()
published = BooleanProperty(default=False)
license = StringProperty(choices=LICENSES, default='cc')
title = StringProperty()
cda = SchemaProperty(LicenseAgreement)
multimedia_included = BooleanProperty(default=True)
downloads = IntegerProperty(
default=0) # number of downloads for this specific snapshot
full_downloads = IntegerProperty(
default=0) # number of downloads for all snapshots from this domain
author = StringProperty()
phone_model = StringProperty()
attribution_notes = StringProperty()
publisher = StringProperty(choices=["organization", "user"],
default="user")
yt_id = StringProperty()
snapshot_head = BooleanProperty(default=False)
deployment = SchemaProperty(Deployment)
image_path = StringProperty()
image_type = StringProperty()
cached_properties = DictProperty()
internal = SchemaProperty(InternalProperties)
dynamic_reports = SchemaListProperty(DynamicReportSet)
# extra user specified properties
tags = StringListProperty()
area = StringProperty(choices=AREA_CHOICES)
sub_area = StringProperty(choices=SUB_AREA_CHOICES)
launch_date = DateTimeProperty
last_modified = DateTimeProperty(default=datetime(2015, 1, 1))
# when turned on, use SECURE_TIMEOUT for sessions of users who are members of this domain
secure_sessions = BooleanProperty(default=False)
two_factor_auth = BooleanProperty(default=False)
strong_mobile_passwords = BooleanProperty(default=False)
# There is no longer a way to request a report builder trial, so this property should be removed in the near
# future. (Keeping it for now in case a user has requested a trial and but has not yet been granted it)
requested_report_builder_trial = StringListProperty()
requested_report_builder_subscription = StringListProperty()
report_whitelist = StringListProperty()
# seconds between sending mobile UCRs to users. Can be overridden per user
default_mobile_ucr_sync_interval = IntegerProperty()
@classmethod
def wrap(cls, data):
# for domains that still use original_doc
should_save = False
if 'original_doc' in data:
original_doc = data['original_doc']
del data['original_doc']
should_save = True
if original_doc:
original_doc = Domain.get_by_name(original_doc)
data['copy_history'] = [original_doc._id]
# for domains that have a public domain license
if 'license' in data:
if data.get("license", None) == "public":
data["license"] = "cc"
should_save = True
if 'slug' in data and data["slug"]:
data["hr_name"] = data["slug"]
del data["slug"]
if 'is_test' in data and isinstance(data["is_test"], bool):
data["is_test"] = "true" if data["is_test"] else "false"
should_save = True
if 'cloudcare_releases' not in data:
data['cloudcare_releases'] = 'nostars' # legacy default setting
# Don't actually remove location_types yet. We can migrate fully and
# remove this after everything's hunky-dory in production. 2015-03-06
if 'location_types' in data:
data['obsolete_location_types'] = data.pop('location_types')
if 'granted_messaging_access' not in data:
# enable messaging for domains created before this flag was added
data['granted_messaging_access'] = True
self = super(Domain, cls).wrap(data)
if self.deployment is None:
self.deployment = Deployment()
if should_save:
self.save()
return self
def get_default_timezone(self):
"""return a timezone object from self.default_timezone"""
import pytz
return pytz.timezone(self.default_timezone)
@staticmethod
@quickcache(['name'], timeout=24 * 60 * 60)
def is_secure_session_required(name):
domain_obj = Domain.get_by_name(name)
return domain_obj and domain_obj.secure_sessions
@staticmethod
@quickcache(['couch_user._id', 'is_active'],
timeout=5 * 60,
memoize_timeout=10)
def active_for_couch_user(couch_user, is_active=True):
domain_names = couch_user.get_domains()
return Domain.view(
"domain/by_status",
keys=[[is_active, d] for d in domain_names],
reduce=False,
include_docs=True,
).all()
@staticmethod
def active_for_user(user, is_active=True):
if isinstance(user, AnonymousUser):
return []
from corehq.apps.users.models import CouchUser
if isinstance(user, CouchUser):
couch_user = user
else:
couch_user = CouchUser.from_django_user(user)
if couch_user:
return Domain.active_for_couch_user(couch_user,
is_active=is_active)
else:
return []
def add(self, model_instance, is_active=True):
"""
Add something to this domain, through the generic relation.
Returns the created membership object
"""
# Add membership info to Couch
couch_user = model_instance.get_profile().get_couch_user()
couch_user.add_domain_membership(self.name)
couch_user.save()
def applications(self):
return get_brief_apps_in_domain(self.name)
def full_applications(self, include_builds=True):
from corehq.apps.app_manager.util import get_correct_app_class
from corehq.apps.app_manager.models import Application
def wrap_application(a):
return get_correct_app_class(a['doc']).wrap(a['doc'])
if include_builds:
startkey = [self.name]
endkey = [self.name, {}]
else:
startkey = [self.name, None]
endkey = [self.name, None, {}]
return Application.get_db().view('app_manager/applications',
startkey=startkey,
endkey=endkey,
include_docs=True,
wrapper=wrap_application).all()
@cached_property
def versions(self):
apps = self.applications()
return list(set(a.application_version for a in apps))
@cached_property
def has_media(self):
from corehq.apps.app_manager.util import is_remote_app
for app in self.full_applications():
if not is_remote_app(app) and app.has_media():
return True
return False
@property
def use_cloudcare_releases(self):
return self.cloudcare_releases != 'nostars'
def all_users(self):
from corehq.apps.users.models import CouchUser
return CouchUser.by_domain(self.name)
def recent_submissions(self):
return domain_has_submission_in_last_30_days(self.name)
@cached_property
def languages(self):
apps = self.applications()
return set(chain.from_iterable([a.langs for a in apps]))
@classmethod
@quickcache(['name'],
skip_arg='strict',
timeout=30 * 60,
session_function=icds_conditional_session_key())
def get_by_name(cls, name, strict=False):
if not name:
# get_by_name should never be called with name as None (or '', etc)
# I fixed the code in such a way that if I raise a ValueError
# all tests pass and basic pages load,
# but in order not to break anything in the wild,
# I'm opting to notify by email if/when this happens
# but fall back to the previous behavior of returning None
if settings.DEBUG:
raise ValueError('%r is not a valid domain name' % name)
else:
_assert = soft_assert(notify_admins=True,
exponential_backoff=False)
_assert(False, '%r is not a valid domain name' % name)
return None
def _get_by_name(stale=False):
extra_args = {'stale': settings.COUCH_STALE_QUERY} if stale else {}
result = cls.view("domain/domains",
key=name,
reduce=False,
include_docs=True,
**extra_args).first()
if not isinstance(result, Domain):
# A stale view may return a result with no doc if the doc has just been deleted.
# In this case couchdbkit just returns the raw view result as a dict
return None
else:
return result
domain = _get_by_name(stale=(not strict))
if domain is None and not strict:
# on the off chance this is a brand new domain, try with strict
domain = _get_by_name(stale=False)
return domain
@classmethod
def get_or_create_with_name(cls,
name,
is_active=False,
secure_submissions=True,
use_sql_backend=False):
result = cls.view("domain/domains",
key=name,
reduce=False,
include_docs=True).first()
if result:
return result
else:
new_domain = Domain(
name=name,
is_active=is_active,
date_created=datetime.utcnow(),
secure_submissions=secure_submissions,
use_sql_backend=use_sql_backend,
)
new_domain.save(**get_safe_write_kwargs())
return new_domain
@classmethod
def generate_name(cls, hr_name, max_length=25):
'''
Generate a URL-friendly name based on a given human-readable name.
Normalizes given name, then looks for conflicting domains, addressing
conflicts by adding "-1", "-2", etc. May return None if it fails to
generate a new, unique name. Throws exception if it can't figure out
a name, which shouldn't happen unless max_length is absurdly short.
'''
from corehq.apps.domain.utils import get_domain_url_slug
from corehq.apps.domain.dbaccessors import domain_or_deleted_domain_exists
name = get_domain_url_slug(hr_name, max_length=max_length)
if not name:
raise NameUnavailableException
if domain_or_deleted_domain_exists(name):
prefix = name
while len(prefix):
name = next_available_name(
prefix, Domain.get_names_by_prefix(prefix + '-'))
if domain_or_deleted_domain_exists(name):
# should never happen
raise NameUnavailableException
if len(name) <= max_length:
return name
prefix = prefix[:-1]
raise NameUnavailableException
return name
@classmethod
def get_all(cls, include_docs=True):
domains = Domain.view("domain/not_snapshots", include_docs=False).all()
if not include_docs:
return domains
else:
return map(cls.wrap,
iter_docs(cls.get_db(), [d['id'] for d in domains]))
@classmethod
def get_all_names(cls):
return sorted({d['key'] for d in cls.get_all(include_docs=False)})
@classmethod
def get_all_ids(cls):
return [d['id'] for d in cls.get_all(include_docs=False)]
@classmethod
def get_names_by_prefix(cls, prefix):
return [
d['key'] for d in Domain.view("domain/domains",
startkey=prefix,
endkey=prefix + "zzz",
reduce=False,
include_docs=False).all()
] + [
d['key'] for d in Domain.view("domain/deleted_domains",
startkey=prefix,
endkey=prefix + "zzz",
reduce=False,
include_docs=False).all()
]
def case_sharing_included(self):
return self.case_sharing or reduce(lambda x, y: x or y, [
getattr(app, 'case_sharing', False) for app in self.applications()
], False)
def save(self, **params):
from corehq.apps.domain.dbaccessors import domain_or_deleted_domain_exists
self.last_modified = datetime.utcnow()
if not self._rev:
if domain_or_deleted_domain_exists(self.name):
raise NameUnavailableException(self.name)
# mark any new domain as timezone migration complete
set_tz_migration_complete(self.name)
super(Domain, self).save(**params)
from corehq.apps.domain.signals import commcare_domain_post_save
results = commcare_domain_post_save.send_robust(sender='domain',
domain=self)
log_signal_errors(results,
"Error occurred during domain post_save (%s)",
{'domain': self.name})
def snapshots(self, **view_kwargs):
return Domain.view('domain/snapshots',
startkey=[self._id, {}],
endkey=[self._id],
include_docs=True,
reduce=False,
descending=True,
**view_kwargs)
def update_deployment(self, **kwargs):
self.deployment.update(kwargs)
self.save()
def update_internal(self, **kwargs):
self.internal.update(kwargs)
self.save()
def display_name(self):
if self.is_snapshot:
return "Snapshot of %s" % self.copied_from.display_name()
return self.hr_name or self.name
def long_display_name(self):
if self.is_snapshot:
return format_html("Snapshot of {}",
self.copied_from.display_name())
return self.hr_name or self.name
__str__ = long_display_name
def get_license_display(self):
return LICENSES.get(self.license)
def get_license_url(self):
return LICENSE_LINKS.get(self.license)
def copies(self):
return Domain.view('domain/copied_from_snapshot',
key=self._id,
include_docs=True)
def copies_of_parent(self):
return Domain.view('domain/copied_from_snapshot',
keys=[s._id for s in self.copied_from.snapshots()],
include_docs=True)
def delete(self, leave_tombstone=False):
if not leave_tombstone and not settings.UNIT_TESTING:
raise ValueError(
'Cannot delete domain without leaving a tombstone except during testing'
)
self._pre_delete()
if leave_tombstone:
domain = self.get(self._id)
if not domain.doc_type.endswith('-Deleted'):
domain.doc_type = '{}-Deleted'.format(domain.doc_type)
domain.save()
else:
super().delete()
# The save signals can undo effect of clearing the cache within the save
# because they query the stale view (but attaches the up to date doc).
# This is only a problem on delete/soft-delete,
# because these change the presence in the index, not just the doc content.
# Since this is rare, I'm opting to just re-clear the cache here
# rather than making the signals use a strict lookup or something like that.
self.clear_caches()
def _pre_delete(self):
from corehq.apps.domain.deletion import apply_deletion_operations
# delete SQL models first because UCR tables are indexed by configs in couch
apply_deletion_operations(self.name)
# delete couch docs
for db, related_doc_ids in get_all_doc_ids_for_domain_grouped_by_db(
self.name):
iter_bulk_delete(db, related_doc_ids, chunksize=500)
@classmethod
def get_module_by_name(cls, domain_name):
"""
import and return the python module corresponding to domain_name, or
None if it doesn't exist.
"""
module_name = settings.DOMAIN_MODULE_MAP.get(domain_name, domain_name)
try:
return import_module(module_name) if module_name else None
except ImportError:
return None
@property
@memoized
def commtrack_settings(self):
# this import causes some dependency issues so lives in here
from corehq.apps.commtrack.models import CommtrackConfig
if self.commtrack_enabled:
return CommtrackConfig.for_domain(self.name)
else:
return None
@property
def has_custom_logo(self):
return self.has_attachment(LOGO_ATTACHMENT)
def get_custom_logo(self):
if not self.has_custom_logo:
return None
return (self.fetch_attachment(LOGO_ATTACHMENT),
self.blobs[LOGO_ATTACHMENT].content_type)
def put_attachment(self, *args, **kw):
return super(Domain, self).put_attachment(domain=self.name,
*args,
**kw)
def get_case_display(self, case):
"""Get the properties display definition for a given case"""
return self.case_display.case_details.get(case.type)
def get_form_display(self, form):
"""Get the properties display definition for a given XFormInstance"""
return self.case_display.form_details.get(form.xmlns)
@property
def location_types(self):
from corehq.apps.locations.models import LocationType
return LocationType.objects.filter(domain=self.name).all()
@memoized
def has_privilege(self, privilege):
from corehq.apps.accounting.utils import domain_has_privilege
return domain_has_privilege(self, privilege)
@property
@memoized
def uses_locations(self):
from corehq import privileges
from corehq.apps.locations.models import LocationType
return (self.has_privilege(privileges.LOCATIONS) and
(self.commtrack_enabled
or LocationType.objects.filter(domain=self.name).exists()))
def convert_to_commtrack(self):
"""
One-stop-shop to make a domain CommTrack
"""
from corehq.apps.commtrack.util import make_domain_commtrack
make_domain_commtrack(self)
def clear_caches(self):
from .utils import domain_restricts_superusers
super(Domain, self).clear_caches()
self.get_by_name.clear(self.__class__, self.name)
self.is_secure_session_required.clear(self.name)
domain_restricts_superusers.clear(self.name)
def get_daily_outbound_sms_limit(self):
if self.custom_daily_outbound_sms_limit:
return self.custom_daily_outbound_sms_limit
# https://manage.dimagi.com/default.asp?274299
return 50000
class RegistrationRequest(Document):
tos_confirmed = BooleanProperty(default=False)
request_time = DateTimeProperty()
request_ip = StringProperty()
activation_guid = StringProperty()
confirm_time = DateTimeProperty()
confirm_ip = StringProperty()
domain = StringProperty()
new_user_username = StringProperty()
requesting_user_username = StringProperty()
@property
@memoized
def project(self):
return Domain.get_by_name(self.domain)
@classmethod
def get_by_guid(cls, guid):
result = cls.view("registration/requests_by_guid",
key=guid,
reduce=False,
include_docs=True).first()
return result
@classmethod
def get_requests_today(cls):
today = datetime.datetime.utcnow()
yesterday = today - datetime.timedelta(1)
result = cls.view("registration/requests_by_time",
startkey=yesterday.isoformat(),
endkey=today.isoformat(),
reduce=True).all()
if not result:
return 0
return result[0]['value']
@classmethod
def get_requests_24hrs_ago(cls):
today = datetime.datetime.utcnow()
yesterday = today - datetime.timedelta(1)
join_on_start = datetime.datetime(yesterday.year, yesterday.month,
yesterday.day, yesterday.hour, 0, 0,
0)
join_on_end = datetime.datetime(yesterday.year, yesterday.month,
yesterday.day, yesterday.hour, 59, 59,
59)
result = cls.view("registration/requests_by_time",
startkey=join_on_start.isoformat(),
endkey=join_on_end.isoformat(),
include_docs=True,
reduce=False).all()
return [
doc for doc in result
if (doc.new_user_username == doc.requesting_user_username
and doc.confirm_time is None)
]
@classmethod
def get_request_for_username(cls, username):
result = cls.view("registration/requests_by_username",
key=username,
reduce=False,
include_docs=True).first()
return result
class AtomFeedStatus(DocumentSchema):
last_polled_at = DateTimeProperty(default=None)
last_page = StringProperty(default=None)
class Program(Document):
"""
A program, e.g. "hiv" or "tb"
"""
domain = StringProperty()
name = StringProperty()
code = StringProperty()
last_modified = DateTimeProperty()
default = BooleanProperty(default=False)
is_archived = BooleanProperty(default=False)
@classmethod
def wrap(cls, data):
# If "Z" is missing because of the Aug 2014 migration, then add it.
# cf. Group class
last_modified = data.get('last_modified')
if last_modified and dt_no_Z_re.match(last_modified):
data['last_modified'] += 'Z'
return super(Program, cls).wrap(data)
def save(self, *args, **kwargs):
self.last_modified = datetime.utcnow()
super(Program, self).save(*args, **kwargs)
self.clear_caches(self.domain)
@classmethod
def by_domain(cls, domain, wrap=True):
"""
Gets all programs in a domain.
"""
kwargs = dict(
view_name='program_by_code/view',
startkey=[domain],
endkey=[domain, {}],
include_docs=True
)
if wrap:
return Program.view(**kwargs)
else:
return [row["doc"] for row in Program.view(wrap_doc=False, **kwargs)]
@classmethod
def default_for_domain(cls, domain):
programs = cls.by_domain(domain)
for p in programs:
if p.default:
return p
def delete(self):
# you cannot delete the default program
if self.default:
raise Exception(_('You cannot delete the default program'))
default = Program.default_for_domain(self.domain)
sql_products = SQLProduct.objects.filter(domain=self.domain,
program_id=self.get_id)
to_save = []
for product in sql_products.couch_products():
product['program_id'] = default._id
to_save.append(product)
# break up saving in case there are many products
if len(to_save) > 500:
Product.bulk_save(to_save)
to_save = []
Product.bulk_save(to_save)
# bulk update sqlproducts
sql_products.update(program_id=default._id)
super(Program, self).delete()
self.clear_caches(self.domain)
def unarchive(self):
"""
Unarchive a program, causing it (and its data) to show
up in Couch and SQL views again.
"""
self.is_archived = False
self.save()
@classmethod
def get_by_code(cls, domain, code):
result = cls.view("program_by_code/view",
key=[domain, code],
include_docs=True,
limit=1).first()
return result
def get_products_count(self):
return (SQLProduct.objects
.filter(domain=self.domain, program_id=self.get_id)
.count())
@classmethod
def clear_caches(cls, domain):
from casexml.apps.phone.utils import clear_fixture_cache
from corehq.apps.programs.fixtures import PROGRAM_FIXTURE_BUCKET
clear_fixture_cache(domain, PROGRAM_FIXTURE_BUCKET)
class RegistrationRequest(SyncCouchToSQLMixin, Document,
RegistrationRequestMixin):
tos_confirmed = BooleanProperty(default=False)
request_time = DateTimeProperty()
request_ip = StringProperty()
activation_guid = StringProperty()
confirm_time = DateTimeProperty()
confirm_ip = StringProperty()
domain = StringProperty()
new_user_username = StringProperty()
requesting_user_username = StringProperty()
@classmethod
def get_by_guid(cls, guid):
return SQLRegistrationRequest.objects.filter(
activation_guid=guid).first()
@classmethod
def get_requests_today(cls):
today = datetime.datetime.utcnow()
yesterday = today - datetime.timedelta(1)
return SQLRegistrationRequest.objects.filter(
request_time__gte=yesterday.isoformat(),
request_time__lte=today.isoformat(),
).count()
@classmethod
def get_requests_24hrs_ago(cls):
today = datetime.datetime.utcnow()
yesterday = today - datetime.timedelta(1)
join_on_start = datetime.datetime(yesterday.year, yesterday.month,
yesterday.day, yesterday.hour, 0, 0,
0)
join_on_end = datetime.datetime(yesterday.year, yesterday.month,
yesterday.day, yesterday.hour, 59, 59,
59)
requests = SQLRegistrationRequest.objects.filter(
request_time__gte=join_on_start,
request_time__lte=join_on_end,
confirm_time__isnull=True)
return [
req for req in requests
if req.new_user_username == req.requesting_user_username
]
@classmethod
def get_request_for_username(cls, username):
return SQLRegistrationRequest.objects.filter(
new_user_username=username).first()
@classmethod
def _migration_get_fields(cls):
return [
"tos_confirmed",
"request_time",
"request_ip",
"activation_guid",
"confirm_time",
"confirm_ip",
"domain",
"new_user_username",
"requesting_user_username",
]
@classmethod
def _migration_get_sql_model_class(cls):
return SQLRegistrationRequest
class Domain(QuickCachedDocumentMixin, Document, SnapshotMixin):
"""Domain is the highest level collection of people/stuff
in the system. Pretty much everything happens at the
domain-level, including user membership, permission to
see data, reports, charts, etc."""
name = StringProperty()
is_active = BooleanProperty()
date_created = DateTimeProperty()
default_timezone = StringProperty(default=getattr(settings, "TIME_ZONE", "UTC"))
case_sharing = BooleanProperty(default=False)
secure_submissions = BooleanProperty(default=False)
cloudcare_releases = StringProperty(choices=['stars', 'nostars', 'default'], default='default')
organization = StringProperty()
hr_name = StringProperty() # the human-readable name for this project
creating_user = StringProperty() # username of the user who created this domain
# domain metadata
project_type = StringProperty() # e.g. MCH, HIV
customer_type = StringProperty() # plus, full, etc.
is_test = StringProperty(choices=["true", "false", "none"], default="none")
description = StringProperty()
short_description = StringProperty()
is_shared = BooleanProperty(default=False)
commtrack_enabled = BooleanProperty(default=False)
call_center_config = SchemaProperty(CallCenterProperties)
has_careplan = BooleanProperty(default=False)
restrict_superusers = BooleanProperty(default=False)
allow_domain_requests = BooleanProperty(default=False)
location_restriction_for_users = BooleanProperty(default=False)
usercase_enabled = BooleanProperty(default=False)
hipaa_compliant = BooleanProperty(default=False)
use_sql_backend = BooleanProperty(default=False)
case_display = SchemaProperty(CaseDisplaySettings)
# CommConnect settings
commconnect_enabled = BooleanProperty(default=False)
survey_management_enabled = BooleanProperty(default=False)
# Whether or not a case can register via sms
sms_case_registration_enabled = BooleanProperty(default=False)
# Case type to apply to cases registered via sms
sms_case_registration_type = StringProperty()
# Owner to apply to cases registered via sms
sms_case_registration_owner_id = StringProperty()
# Submitting user to apply to cases registered via sms
sms_case_registration_user_id = StringProperty()
# Whether or not a mobile worker can register via sms
sms_mobile_worker_registration_enabled = BooleanProperty(default=False)
use_default_sms_response = BooleanProperty(default=False)
default_sms_response = StringProperty()
chat_message_count_threshold = IntegerProperty()
custom_chat_template = StringProperty() # See settings.CUSTOM_CHAT_TEMPLATES
custom_case_username = StringProperty() # Case property to use when showing the case's name in a chat window
# If empty, sms can be sent at any time. Otherwise, only send during
# these windows of time. SMS_QUEUE_ENABLED must be True in localsettings
# for this be considered.
restricted_sms_times = SchemaListProperty(DayTimeWindow)
# If empty, this is ignored. Otherwise, the framework will make sure
# that during these days/times, no automated outbound sms will be sent
# to someone if they have sent in an sms within sms_conversation_length
# minutes. Outbound sms sent from a user in a chat window, however, will
# still be sent. This is meant to prevent chat conversations from being
# interrupted by automated sms reminders.
# SMS_QUEUE_ENABLED must be True in localsettings for this to be
# considered.
sms_conversation_times = SchemaListProperty(DayTimeWindow)
# In minutes, see above.
sms_conversation_length = IntegerProperty(default=10)
# Set to True to prevent survey questions and answers form being seen in
# SMS chat windows.
filter_surveys_from_chat = BooleanProperty(default=False)
# The below option only matters if filter_surveys_from_chat = True.
# If set to True, invalid survey responses will still be shown in the chat
# window, while questions and valid responses will be filtered out.
show_invalid_survey_responses_in_chat = BooleanProperty(default=False)
# If set to True, if a message is read by anyone it counts as being read by
# everyone. Set to False so that a message is only counted as being read
# for a user if only that user has read it.
count_messages_as_read_by_anyone = BooleanProperty(default=False)
# Set to True to allow sending sms and all-label surveys to cases whose
# phone number is duplicated with another contact
send_to_duplicated_case_numbers = BooleanProperty(default=True)
enable_registration_welcome_sms_for_case = BooleanProperty(default=False)
enable_registration_welcome_sms_for_mobile_worker = BooleanProperty(default=False)
sms_survey_date_format = StringProperty()
# exchange/domain copying stuff
is_snapshot = BooleanProperty(default=False)
is_approved = BooleanProperty(default=False)
snapshot_time = DateTimeProperty()
published = BooleanProperty(default=False)
license = StringProperty(choices=LICENSES, default='cc')
title = StringProperty()
cda = SchemaProperty(LicenseAgreement)
multimedia_included = BooleanProperty(default=True)
downloads = IntegerProperty(default=0) # number of downloads for this specific snapshot
full_downloads = IntegerProperty(default=0) # number of downloads for all snapshots from this domain
author = StringProperty()
phone_model = StringProperty()
attribution_notes = StringProperty()
publisher = StringProperty(choices=["organization", "user"], default="user")
yt_id = StringProperty()
snapshot_head = BooleanProperty(default=False)
deployment = SchemaProperty(Deployment)
image_path = StringProperty()
image_type = StringProperty()
cached_properties = DictProperty()
internal = SchemaProperty(InternalProperties)
dynamic_reports = SchemaListProperty(DynamicReportSet)
# extra user specified properties
tags = StringListProperty()
area = StringProperty(choices=AREA_CHOICES)
sub_area = StringProperty(choices=SUB_AREA_CHOICES)
launch_date = DateTimeProperty
# to be eliminated from projects and related documents when they are copied for the exchange
_dirty_fields = ('admin_password', 'admin_password_charset', 'city', 'countries', 'region', 'customer_type')
last_modified = DateTimeProperty(default=datetime(2015, 1, 1))
# when turned on, use SECURE_TIMEOUT for sessions of users who are members of this domain
secure_sessions = BooleanProperty(default=False)
two_factor_auth = BooleanProperty(default=False)
strong_mobile_passwords = BooleanProperty(default=False)
# There is no longer a way to request a report builder trial, so this property should be removed in the near
# future. (Keeping it for now in case a user has requested a trial and but has not yet been granted it)
requested_report_builder_trial = StringListProperty()
requested_report_builder_subscription = StringListProperty()
@classmethod
def wrap(cls, data):
# for domains that still use original_doc
should_save = False
if 'original_doc' in data:
original_doc = data['original_doc']
del data['original_doc']
should_save = True
if original_doc:
original_doc = Domain.get_by_name(original_doc)
data['copy_history'] = [original_doc._id]
# for domains that have a public domain license
if 'license' in data:
if data.get("license", None) == "public":
data["license"] = "cc"
should_save = True
if 'slug' in data and data["slug"]:
data["hr_name"] = data["slug"]
del data["slug"]
if 'is_test' in data and isinstance(data["is_test"], bool):
data["is_test"] = "true" if data["is_test"] else "false"
should_save = True
if 'cloudcare_releases' not in data:
data['cloudcare_releases'] = 'nostars' # legacy default setting
# Don't actually remove location_types yet. We can migrate fully and
# remove this after everything's hunky-dory in production. 2015-03-06
if 'location_types' in data:
data['obsolete_location_types'] = data.pop('location_types')
self = super(Domain, cls).wrap(data)
if self.deployment is None:
self.deployment = Deployment()
if should_save:
self.save()
return self
def get_default_timezone(self):
"""return a timezone object from self.default_timezone"""
import pytz
return pytz.timezone(self.default_timezone)
@staticmethod
@quickcache(['name'], timeout=24 * 60 * 60)
def is_secure_session_required(name):
domain = Domain.get_by_name(name)
return domain and domain.secure_sessions
@staticmethod
@skippable_quickcache(['couch_user._id', 'is_active'],
skip_arg='strict', timeout=5*60, memoize_timeout=10)
def active_for_couch_user(couch_user, is_active=True, strict=False):
domain_names = couch_user.get_domains()
return Domain.view(
"domain/by_status",
keys=[[is_active, d] for d in domain_names],
reduce=False,
include_docs=True,
stale=settings.COUCH_STALE_QUERY if not strict else None,
).all()
@staticmethod
def active_for_user(user, is_active=True, strict=False):
if isinstance(user, AnonymousUser):
return []
from corehq.apps.users.models import CouchUser
if isinstance(user, CouchUser):
couch_user = user
else:
couch_user = CouchUser.from_django_user(user)
if couch_user:
return Domain.active_for_couch_user(
couch_user, is_active=is_active, strict=strict)
else:
return []
@classmethod
def field_by_prefix(cls, field, prefix=''):
# unichr(0xfff8) is something close to the highest character available
res = cls.view("domain/fields_by_prefix",
group=True,
startkey=[field, True, prefix],
endkey=[field, True, "%s%c" % (prefix, unichr(0xfff8)), {}])
vals = [(d['value'], d['key'][2]) for d in res]
vals.sort(reverse=True)
return [(v[1], v[0]) for v in vals]
def add(self, model_instance, is_active=True):
"""
Add something to this domain, through the generic relation.
Returns the created membership object
"""
# Add membership info to Couch
couch_user = model_instance.get_profile().get_couch_user()
couch_user.add_domain_membership(self.name)
couch_user.save()
def applications(self):
return get_brief_apps_in_domain(self.name)
def full_applications(self, include_builds=True):
from corehq.apps.app_manager.models import Application, RemoteApp
WRAPPERS = {'Application': Application, 'RemoteApp': RemoteApp}
def wrap_application(a):
return WRAPPERS[a['doc']['doc_type']].wrap(a['doc'])
if include_builds:
startkey = [self.name]
endkey = [self.name, {}]
else:
startkey = [self.name, None]
endkey = [self.name, None, {}]
return Application.get_db().view('app_manager/applications',
startkey=startkey,
endkey=endkey,
include_docs=True,
wrapper=wrap_application).all()
@cached_property
def versions(self):
apps = self.applications()
return list(set(a.application_version for a in apps))
@cached_property
def has_case_management(self):
for app in self.full_applications():
if app.doc_type == 'Application':
if app.has_case_management():
return True
return False
@cached_property
def has_media(self):
for app in self.full_applications():
if app.doc_type == 'Application' and app.has_media():
return True
return False
@property
def use_cloudcare_releases(self):
return self.cloudcare_releases != 'nostars'
def all_users(self):
from corehq.apps.users.models import CouchUser
return CouchUser.by_domain(self.name)
def recent_submissions(self):
return domain_has_submission_in_last_30_days(self.name)
@cached_property
def languages(self):
apps = self.applications()
return set(chain.from_iterable([a.langs for a in apps]))
def readable_languages(self):
return ', '.join(lang_lookup[lang] or lang for lang in self.languages())
def __unicode__(self):
return self.name
@classmethod
@skippable_quickcache(['name'], skip_arg='strict', timeout=30*60)
def get_by_name(cls, name, strict=False):
if not name:
# get_by_name should never be called with name as None (or '', etc)
# I fixed the code in such a way that if I raise a ValueError
# all tests pass and basic pages load,
# but in order not to break anything in the wild,
# I'm opting to notify by email if/when this happens
# but fall back to the previous behavior of returning None
if settings.DEBUG:
raise ValueError('%r is not a valid domain name' % name)
else:
_assert = soft_assert(notify_admins=True, exponential_backoff=False)
_assert(False, '%r is not a valid domain name' % name)
return None
def _get_by_name(stale=False):
extra_args = {'stale': settings.COUCH_STALE_QUERY} if stale else {}
result = cls.view("domain/domains", key=name, reduce=False, include_docs=True, **extra_args).first()
if not isinstance(result, Domain):
# A stale view may return a result with no doc if the doc has just been deleted.
# In this case couchdbkit just returns the raw view result as a dict
return None
else:
return result
domain = _get_by_name(stale=(not strict))
if domain is None and not strict:
# on the off chance this is a brand new domain, try with strict
domain = _get_by_name(stale=False)
return domain
@classmethod
def get_or_create_with_name(cls, name, is_active=False, secure_submissions=True, use_sql_backend=False):
result = cls.view("domain/domains", key=name, reduce=False, include_docs=True).first()
if result:
return result
else:
new_domain = Domain(
name=name,
is_active=is_active,
date_created=datetime.utcnow(),
secure_submissions=secure_submissions,
use_sql_backend=use_sql_backend,
)
new_domain.save(**get_safe_write_kwargs())
return new_domain
@classmethod
def generate_name(cls, hr_name, max_length=25):
'''
Generate a URL-friendly name based on a given human-readable name.
Normalizes given name, then looks for conflicting domains, addressing
conflicts by adding "-1", "-2", etc. May return None if it fails to
generate a new, unique name. Throws exception if it can't figure out
a name, which shouldn't happen unless max_length is absurdly short.
'''
from corehq.apps.domain.utils import get_domain_url_slug
name = get_domain_url_slug(hr_name, max_length=max_length)
if not name:
raise NameUnavailableException
if Domain.get_by_name(name):
prefix = name
while len(prefix):
name = next_available_name(prefix, Domain.get_names_by_prefix(prefix + '-'))
if Domain.get_by_name(name):
# should never happen
raise NameUnavailableException
if len(name) <= max_length:
return name
prefix = prefix[:-1]
raise NameUnavailableException
return name
@classmethod
def get_all(cls, include_docs=True):
domains = Domain.view("domain/not_snapshots", include_docs=False).all()
if not include_docs:
return domains
else:
return imap(cls.wrap, iter_docs(cls.get_db(), [d['id'] for d in domains]))
@classmethod
def get_all_names(cls):
return [d['key'] for d in cls.get_all(include_docs=False)]
@classmethod
def get_all_ids(cls):
return [d['id'] for d in cls.get_all(include_docs=False)]
@classmethod
def get_names_by_prefix(cls, prefix):
return [d['key'] for d in Domain.view(
"domain/domains",
startkey=prefix,
endkey=prefix + u"zzz",
reduce=False,
include_docs=False
).all()]
def case_sharing_included(self):
return self.case_sharing or reduce(lambda x, y: x or y, [getattr(app, 'case_sharing', False) for app in self.applications()], False)
def save(self, **params):
self.last_modified = datetime.utcnow()
if not self._rev:
# mark any new domain as timezone migration complete
set_migration_complete(self.name)
super(Domain, self).save(**params)
from corehq.apps.domain.signals import commcare_domain_post_save
results = commcare_domain_post_save.send_robust(sender='domain', domain=self)
for result in results:
# Second argument is None if there was no error
if result[1]:
notify_exception(
None,
message="Error occured during domain post_save %s: %s" %
(self.name, str(result[1]))
)
def save_copy(self, new_domain_name=None, new_hr_name=None, user=None,
copy_by_id=None, share_reminders=True,
share_user_roles=True):
from corehq.apps.app_manager.dbaccessors import get_app
from corehq.apps.reminders.models import CaseReminderHandler
from corehq.apps.fixtures.models import FixtureDataItem
from corehq.apps.app_manager.dbaccessors import get_brief_apps_in_domain
from corehq.apps.domain.dbaccessors import get_doc_ids_in_domain_by_class
from corehq.apps.fixtures.models import FixtureDataType
from corehq.apps.users.models import UserRole
db = Domain.get_db()
new_id = db.copy_doc(self.get_id)['id']
if new_domain_name is None:
new_domain_name = new_id
with CriticalSection(['request_domain_name_{}'.format(new_domain_name)]):
new_domain_name = Domain.generate_name(new_domain_name)
new_domain = Domain.get(new_id)
new_domain.name = new_domain_name
new_domain.hr_name = new_hr_name
new_domain.copy_history = self.get_updated_history()
new_domain.is_snapshot = False
new_domain.snapshot_time = None
new_domain.organization = None # TODO: use current user's organization (?)
# reset stuff
new_domain.cda.signed = False
new_domain.cda.date = None
new_domain.cda.type = None
new_domain.cda.user_id = None
new_domain.cda.user_ip = None
new_domain.is_test = "none"
new_domain.internal = InternalProperties()
new_domain.creating_user = user.username if user else None
new_domain.date_created = datetime.utcnow()
for field in self._dirty_fields:
if hasattr(new_domain, field):
delattr(new_domain, field)
# Saving the domain should happen before we import any apps since
# importing apps can update the domain object (for example, if user
# as a case needs to be enabled)
new_domain.save()
new_app_components = {} # a mapping of component's id to its copy
def copy_data_items(old_type_id, new_type_id):
for item in FixtureDataItem.by_data_type(self.name, old_type_id):
comp = self.copy_component(
item.doc_type, item._id, new_domain_name, user=user)
comp.data_type_id = new_type_id
comp.save()
def get_latest_app_id(doc_id):
app = get_app(self.name, doc_id).get_latest_saved()
if app:
return app._id, app.doc_type
for app in get_brief_apps_in_domain(self.name):
doc_id, doc_type = app.get_id, app.doc_type
original_doc_id = doc_id
if copy_by_id and doc_id not in copy_by_id:
continue
if not self.is_snapshot:
doc_id, doc_type = get_latest_app_id(doc_id) or (doc_id, doc_type)
component = self.copy_component(doc_type, doc_id, new_domain_name, user=user)
if component:
new_app_components[original_doc_id] = component
for doc_id in get_doc_ids_in_domain_by_class(self.name, FixtureDataType):
if copy_by_id and doc_id not in copy_by_id:
continue
component = self.copy_component(
'FixtureDataType', doc_id, new_domain_name, user=user)
copy_data_items(doc_id, component._id)
if share_reminders:
for doc_id in get_doc_ids_in_domain_by_class(self.name, CaseReminderHandler):
self.copy_component(
'CaseReminderHandler', doc_id, new_domain_name, user=user)
if share_user_roles:
for doc_id in get_doc_ids_in_domain_by_class(self.name, UserRole):
self.copy_component('UserRole', doc_id, new_domain_name, user=user)
if user:
def add_dom_to_user(user):
user.add_domain_membership(new_domain_name, is_admin=True)
apply_update(user, add_dom_to_user)
def update_events(handler):
"""
Change the form_unique_id to the proper form for each event in a newly copied CaseReminderHandler
"""
from corehq.apps.app_manager.models import FormBase
for event in handler.events:
if not event.form_unique_id:
continue
form = FormBase.get_form(event.form_unique_id)
form_app = form.get_app()
m_index, f_index = form_app.get_form_location(form.unique_id)
form_copy = new_app_components[form_app._id].get_module(m_index).get_form(f_index)
event.form_unique_id = form_copy.unique_id
def update_for_copy(handler):
handler.active = False
update_events(handler)
if share_reminders:
for handler in CaseReminderHandler.get_handlers(new_domain_name):
apply_update(handler, update_for_copy)
return new_domain
def reminder_should_be_copied(self, handler):
from corehq.apps.reminders.models import ON_DATETIME
return (handler.start_condition_type != ON_DATETIME and
handler.user_group_id is None)
def copy_component(self, doc_type, id, new_domain_name, user=None):
from corehq.apps.app_manager.models import import_app
from corehq.apps.users.models import UserRole
from corehq.apps.reminders.models import CaseReminderHandler
from corehq.apps.fixtures.models import FixtureDataType, FixtureDataItem
str_to_cls = {
'UserRole': UserRole,
'CaseReminderHandler': CaseReminderHandler,
'FixtureDataType': FixtureDataType,
'FixtureDataItem': FixtureDataItem,
}
if doc_type in ('Application', 'RemoteApp'):
new_doc = import_app(id, new_domain_name)
new_doc.copy_history.append(id)
new_doc.case_sharing = False
# when copying from app-docs that don't have
# unique_id attribute on Modules
new_doc.ensure_module_unique_ids(should_save=False)
else:
cls = str_to_cls[doc_type]
db = cls.get_db()
if doc_type == 'CaseReminderHandler':
cur_doc = cls.get(id)
if not self.reminder_should_be_copied(cur_doc):
return None
new_id = db.copy_doc(id)['id']
new_doc = cls.get(new_id)
for field in self._dirty_fields:
if hasattr(new_doc, field):
delattr(new_doc, field)
if hasattr(cls, '_meta_fields'):
for field in cls._meta_fields:
if not field.startswith('_') and hasattr(new_doc, field):
delattr(new_doc, field)
new_doc.domain = new_domain_name
if doc_type == 'FixtureDataType':
new_doc.copy_from = id
new_doc.is_global = True
if self.is_snapshot and doc_type == 'Application':
new_doc.prepare_multimedia_for_exchange()
new_doc.save()
return new_doc
def save_snapshot(self, share_reminders, copy_by_id=None):
if self.is_snapshot:
return self
else:
try:
copy = self.save_copy(
copy_by_id=copy_by_id, share_reminders=share_reminders,
share_user_roles=False)
except NameUnavailableException:
return None
copy.is_snapshot = True
head = self.snapshots(limit=1).first()
if head and head.snapshot_head:
head.snapshot_head = False
head.save()
copy.snapshot_head = True
copy.snapshot_time = datetime.utcnow()
del copy.deployment
copy.save()
return copy
def snapshots(self, **view_kwargs):
return Domain.view('domain/snapshots',
startkey=[self._id, {}],
endkey=[self._id],
include_docs=True,
reduce=False,
descending=True,
**view_kwargs
)
@memoized
def published_snapshot(self):
snapshots = self.snapshots().all()
for snapshot in snapshots:
if snapshot.published:
return snapshot
return None
def update_deployment(self, **kwargs):
self.deployment.update(kwargs)
self.save()
def update_internal(self, **kwargs):
self.internal.update(kwargs)
self.save()
def display_name(self):
if self.is_snapshot:
return "Snapshot of %s" % self.copied_from.display_name()
return self.hr_name or self.name
def long_display_name(self):
if self.is_snapshot:
return format_html("Snapshot of {}", self.copied_from.display_name())
return self.hr_name or self.name
__str__ = long_display_name
def get_license_display(self):
return LICENSES.get(self.license)
def get_license_url(self):
return LICENSE_LINKS.get(self.license)
def copies(self):
return Domain.view('domain/copied_from_snapshot', key=self._id, include_docs=True)
def copies_of_parent(self):
return Domain.view('domain/copied_from_snapshot', keys=[s._id for s in self.copied_from.snapshots()], include_docs=True)
def delete(self):
self._pre_delete()
super(Domain, self).delete()
def _pre_delete(self):
from corehq.apps.domain.signals import commcare_domain_pre_delete
from corehq.apps.domain.deletion import apply_deletion_operations
dynamic_deletion_operations = []
results = commcare_domain_pre_delete.send_robust(sender='domain', domain=self)
for result in results:
response = result[1]
if isinstance(response, Exception):
raise DomainDeleteException(u"Error occurred during domain pre_delete {}: {}".format(self.name, str(response)))
elif response:
assert isinstance(response, list)
dynamic_deletion_operations.extend(response)
# delete all associated objects
for db, related_doc_ids in get_all_doc_ids_for_domain_grouped_by_db(self.name):
iter_bulk_delete(db, related_doc_ids, chunksize=500)
apply_deletion_operations(self.name, dynamic_deletion_operations)
def all_media(self, from_apps=None): # todo add documentation or refactor
from corehq.apps.hqmedia.models import CommCareMultimedia
dom_with_media = self if not self.is_snapshot else self.copied_from
if self.is_snapshot:
app_ids = [app.copied_from.get_id for app in self.full_applications()]
if from_apps:
from_apps = set([a_id for a_id in app_ids if a_id in from_apps])
else:
from_apps = app_ids
if from_apps:
media = []
media_ids = set()
apps = [app for app in dom_with_media.full_applications() if app.get_id in from_apps]
for app in apps:
if app.doc_type != 'Application':
continue
for _, m in app.get_media_objects():
if m.get_id not in media_ids:
media.append(m)
media_ids.add(m.get_id)
return media
return CommCareMultimedia.view('hqmedia/by_domain', key=dom_with_media.name, include_docs=True).all()
def most_restrictive_licenses(self, apps_to_check=None):
from corehq.apps.hqmedia.utils import most_restrictive
licenses = [m.license['type'] for m in self.all_media(from_apps=apps_to_check) if m.license]
return most_restrictive(licenses)
@classmethod
def get_module_by_name(cls, domain_name):
"""
import and return the python module corresponding to domain_name, or
None if it doesn't exist.
"""
from corehq.apps.domain.utils import get_domain_module_map
module_name = get_domain_module_map().get(domain_name, domain_name)
try:
return import_module(module_name) if module_name else None
except ImportError:
return None
@property
@memoized
def commtrack_settings(self):
# this import causes some dependency issues so lives in here
from corehq.apps.commtrack.models import CommtrackConfig
if self.commtrack_enabled:
return CommtrackConfig.for_domain(self.name)
else:
return None
@property
def has_custom_logo(self):
return (self['_attachments'] and
LOGO_ATTACHMENT in self['_attachments'])
def get_custom_logo(self):
if not self.has_custom_logo:
return None
return (
self.fetch_attachment(LOGO_ATTACHMENT),
self['_attachments'][LOGO_ATTACHMENT]['content_type']
)
def get_case_display(self, case):
"""Get the properties display definition for a given case"""
return self.case_display.case_details.get(case.type)
def get_form_display(self, form):
"""Get the properties display definition for a given XFormInstance"""
return self.case_display.form_details.get(form.xmlns)
@property
def total_downloads(self):
"""
Returns the total number of downloads from every snapshot created from this domain
"""
from corehq.apps.domain.dbaccessors import count_downloads_for_all_snapshots
return count_downloads_for_all_snapshots(self.get_id)
@property
@memoized
def download_count(self):
"""
Updates and returns the total number of downloads from every sister snapshot.
"""
if self.is_snapshot:
self.full_downloads = self.copied_from.total_downloads
return self.full_downloads
@property
@memoized
def published_by(self):
from corehq.apps.users.models import CouchUser
pb_id = self.cda.user_id
return CouchUser.get_by_user_id(pb_id) if pb_id else None
@property
def name_of_publisher(self):
return self.published_by.human_friendly_name if self.published_by else ""
@property
def location_types(self):
from corehq.apps.locations.models import LocationType
return LocationType.objects.filter(domain=self.name).all()
@memoized
def has_privilege(self, privilege):
from corehq.apps.accounting.utils import domain_has_privilege
return domain_has_privilege(self, privilege)
@property
@memoized
def uses_locations(self):
from corehq import privileges
from corehq.apps.locations.models import LocationType
return (self.has_privilege(privileges.LOCATIONS)
and (self.commtrack_enabled
or LocationType.objects.filter(domain=self.name).exists()))
@property
def supports_multiple_locations_per_user(self):
"""
This method is a wrapper around the toggle that
enables multiple location functionality. Callers of this
method should know that this is special functionality
left around for special applications, and not a feature
flag that should be set normally.
"""
return toggles.MULTIPLE_LOCATIONS_PER_USER.enabled(self.name)
def convert_to_commtrack(self):
"""
One-stop-shop to make a domain CommTrack
"""
from corehq.apps.commtrack.util import make_domain_commtrack
make_domain_commtrack(self)
def clear_caches(self):
from .utils import domain_restricts_superusers
super(Domain, self).clear_caches()
self.get_by_name.clear(self.__class__, self.name)
self.is_secure_session_required.clear(self.name)
domain_restricts_superusers.clear(self.name)
class RequisitionCase(CommCareCase):
"""
A wrapper around CommCareCases to get more built in functionality
specific to requisitions.
"""
class Meta:
# This is necessary otherwise syncdb will confuse this app with casexml
app_label = "commtrack"
requisition_status = StringProperty()
# TODO none of these properties are supported on mobile currently
# we need to discuss what will be eventually so we know what we need
# to support here
requested_on = DateTimeProperty()
approved_on = DateTimeProperty()
fulfilled_on = DateTimeProperty()
received_on = DateTimeProperty()
requested_by = StringProperty()
approved_by = StringProperty()
fulfilled_by = StringProperty()
received_by = StringProperty()
@memoized
def get_location(self):
try:
return SupplyPointCase.get(self.indices[0].referenced_id).location
except ResourceNotFound:
return None
@memoized
def get_requester(self):
# TODO this doesn't get set by mobile yet
# if self.requested_by:
# return CommCareUser.get(self.requested_by)
return None
def sms_format(self):
if self.requisition_status == RequisitionStatus.REQUESTED:
section = 'ct-requested'
elif self.requisition_status == RequisitionStatus.APPROVED:
section = 'ct-approved'
else:
section = 'stock'
formatted_strings = []
states = StockState.objects.filter(case_id=self._id,
section_id=section)
for state in states:
product = Product.get(state.product_id)
formatted_strings.append('%s:%d' %
(product.code, state.stock_on_hand))
return ' '.join(sorted(formatted_strings))
def get_next_action(self):
req_config = CommtrackConfig.for_domain(self.domain).requisition_config
return req_config.get_next_action(
RequisitionStatus.to_action_type(self.requisition_status))
@classmethod
def get_by_external_id(cls, domain, external_id):
# only used by openlmis
raise NotImplementedError()
# return cls.view('hqcase/by_domain_external_id',
# key=[domain, external_id],
# include_docs=True, reduce=False,
# classes={'CommCareCase': RequisitionCase}
# ).all()
@classmethod
def get_display_config(cls):
return [{
"layout": [
[{
"name": _("Status"),
"expr": "requisition_status"
}],
]
}, {
"layout": [[{
"name": _("Requested On"),
"expr": "requested_on",
"parse_date": True
}, {
"name": _("requested_by"),
"expr": "requested_by"
}],
[{
"name": _("Approved On"),
"expr": "approved_on",
"parse_date": True
}, {
"name": _("approved_by"),
"expr": "approved_by"
}],
[{
"name": _("Received On"),
"expr": "received_on",
"parse_date": True
}, {
"name": _("received_by"),
"expr": "received_by"
}]]
}]
class AbstractSyncLog(SafeSaveDocument):
date = DateTimeProperty()
domain = StringProperty()
user_id = StringProperty()
request_user_id = StringProperty() # ID of user making request
build_id = StringProperty() # only works with app-aware sync
app_id = StringProperty() # only works with app-aware sync
previous_log_id = StringProperty() # previous sync log, forming a chain
duration = IntegerProperty() # in seconds
log_format = StringProperty()
# owner_ids_on_phone stores the ids the phone thinks it's the owner of.
# This typically includes the user id,
# as well as all groups that that user is a member of.
owner_ids_on_phone = StringListProperty()
# for debugging / logging
previous_log_rev = StringProperty(
) # rev of the previous log at the time of creation
last_submitted = DateTimeProperty(
) # last time a submission caused this to be modified
rev_before_last_submitted = StringProperty(
) # rev when the last submission was saved
last_cached = DateTimeProperty(
) # last time this generated a cached response
hash_at_last_cached = StringProperty(
) # the state hash of this when it was last cached
# save state errors and hashes here
had_state_error = BooleanProperty(default=False)
error_date = DateTimeProperty()
error_hash = StringProperty()
cache_payload_paths = DictProperty()
last_ucr_sync_times = SchemaListProperty(UCRSyncLog)
strict = True # for asserts
@classmethod
def wrap(cls, data):
ret = super(AbstractSyncLog, cls).wrap(data)
if hasattr(ret, 'has_assert_errors'):
ret.strict = False
return ret
def save(self):
self._synclog_sql = save_synclog_to_sql(self)
def delete(self):
if getattr(self, '_synclog_sql', None):
self._synclog_sql.delete()
def case_count(self):
"""
How many cases are associated with this. Used in reports.
"""
raise NotImplementedError()
def phone_is_holding_case(self, case_id):
raise NotImplementedError()
def get_footprint_of_cases_on_phone(self):
"""
Gets the phone's flat list of all case ids on the phone,
owned or not owned but relevant.
"""
raise NotImplementedError()
def get_state_hash(self):
return CaseStateHash(
Checksum(self.get_footprint_of_cases_on_phone()).hexdigest())
def update_phone_lists(self, xform, case_list):
"""
Given a form an list of touched cases, update this sync log to reflect the updated
state on the phone.
"""
raise NotImplementedError()
@classmethod
def from_other_format(cls, other_sync_log):
"""
Convert to an instance of a subclass from another subclass. Subclasses can
override this to provide conversion functions.
"""
raise IncompatibleSyncLogType('Unable to convert from {} to {}'.format(
type(other_sync_log),
cls,
))
class UCRSyncLog(Document):
report_uuid = StringProperty()
datetime = DateTimeProperty()
class Group(QuickCachedDocumentMixin, UndoableDocument):
"""
The main use case for these 'groups' of users is currently
so that we can break down reports by arbitrary regions.
(Things like who sees what reports are determined by permissions.)
"""
domain = StringProperty()
name = StringProperty()
# a list of user ids for users
users = ListProperty()
# a list of user ids that have been removed from the Group.
# This is recorded so that we can update the user at a later point
removed_users = SetProperty()
case_sharing = BooleanProperty()
reporting = BooleanProperty(default=True)
last_modified = DateTimeProperty()
# custom data can live here
metadata = DictProperty()
@classmethod
def wrap(cls, data):
last_modified = data.get('last_modified')
# if it's missing a Z because of the Aug. 2014 migration
# that added this in iso_format() without Z, then add a Z
if last_modified and dt_no_Z_re.match(last_modified):
data['last_modified'] += 'Z'
return super(Group, cls).wrap(data)
def save(self, *args, **kwargs):
self.last_modified = datetime.utcnow()
super(Group, self).save(*args, **kwargs)
refresh_group_views()
@classmethod
def save_docs(cls, docs, use_uuids=True):
utcnow = datetime.utcnow()
for doc in docs:
doc['last_modified'] = utcnow
super(Group, cls).save_docs(docs, use_uuids)
refresh_group_views()
bulk_save = save_docs
def delete(self):
super(Group, self).delete()
refresh_group_views()
@classmethod
def delete_docs(cls, docs, **params):
super(Group, cls).delete_docs(docs, **params)
refresh_group_views()
bulk_delete = delete_docs
def clear_caches(self):
super(Group, self).clear_caches()
self.by_domain.clear(self.__class__, self.domain)
self.ids_by_domain.clear(self.__class__, self.domain)
def add_user(self, couch_user_id, save=True):
if not isinstance(couch_user_id, str):
couch_user_id = couch_user_id.user_id
if couch_user_id not in self.users:
self.users.append(couch_user_id)
if couch_user_id in self.removed_users:
self.removed_users.remove(couch_user_id)
if save:
self.save()
def remove_user(self, couch_user_id):
'''
Returns True if it removed a user, False otherwise
'''
if not isinstance(couch_user_id, str):
couch_user_id = couch_user_id.user_id
if couch_user_id in self.users:
for i in range(0, len(self.users)):
if self.users[i] == couch_user_id:
del self.users[i]
self.removed_users.add(couch_user_id)
return True
return False
def get_user_ids(self, is_active=True):
return [user.user_id for user in self.get_users(is_active=is_active)]
@memoized
def get_users(self, is_active=True, only_commcare=False):
def is_relevant_user(user):
if user.is_deleted():
return False
if only_commcare and user.__class__ != CommCareUser().__class__:
return False
if is_active and not user.is_active:
return False
return True
users = map(CouchUser.wrap_correctly,
iter_docs(self.get_db(), self.users))
return list(filter(is_relevant_user, users))
@memoized
def get_static_user_ids(self, is_active=True):
return [user.user_id for user in self.get_static_users(is_active)]
@classmethod
def get_static_user_ids_for_groups(cls, group_ids):
static_user_ids = []
for group_id in group_ids:
group = cls.get(group_id)
static_user_ids.append(group.get_static_user_ids())
return static_user_ids
@memoized
def get_static_users(self, is_active=True):
return self.get_users(is_active)
@classmethod
@quickcache(['cls.__name__', 'domain'])
def by_domain(cls, domain):
return group_by_domain(domain)
@classmethod
def choices_by_domain(cls, domain):
group_ids = cls.ids_by_domain(domain)
group_choices = []
for group_doc in iter_docs(cls.get_db(), group_ids):
group_choices.append((group_doc['_id'], group_doc['name']))
return group_choices
@classmethod
@quickcache(['cls.__name__', 'domain'])
def ids_by_domain(cls, domain):
return get_group_ids_by_domain(domain)
@classmethod
def by_name(cls, domain, name, one=True):
result = stale_group_by_name(domain, name)
if one and result:
return result[0]
else:
return result
@classmethod
def by_user_id(cls, user_id, wrap=True):
results = cls.view('groups/by_user', key=user_id, include_docs=wrap)
if wrap:
return results
else:
return [r['id'] for r in results]
@classmethod
def get_case_sharing_accessible_locations(cls, domain, user):
return [
location.case_sharing_group_object()
for location in SQLLocation.objects.accessible_to_user(
domain, user).filter(location_type__shares_cases=True)
]
@classmethod
def get_case_sharing_groups(cls, domain, wrap=True):
all_groups = cls.by_domain(domain)
if wrap:
groups = [group for group in all_groups if group.case_sharing]
groups.extend([
location.case_sharing_group_object()
for location in SQLLocation.objects.filter(
domain=domain, location_type__shares_cases=True)
])
return groups
else:
return [group._id for group in all_groups if group.case_sharing]
@classmethod
def get_reporting_groups(cls, domain):
key = ['^Reporting', domain]
return cls.view(
'groups/by_name',
startkey=key,
endkey=key + [{}],
include_docs=True,
stale=settings.COUCH_STALE_QUERY,
).all()
def create_delete_record(self, *args, **kwargs):
return DeleteGroupRecord(*args, **kwargs)
@property
def display_name(self):
if self.name:
return self.name
else:
return "[No Name]"
@classmethod
def user_in_group(cls, user_id, group_id):
if not user_id or not group_id:
return False
c = cls.get_db().view('groups/by_user',
key=user_id,
startkey_docid=group_id,
endkey_docid=group_id).count()
if c == 0:
return False
elif c == 1:
return True
else:
raise Exception(
"This should just logically not be possible unless the group "
"has the user in there twice")
def is_member_of(self, domain):
return self.domain == domain
@property
def is_deleted(self):
return self.doc_type.endswith(DELETED_SUFFIX)
def __repr__(self):
return ("Group(domain={self.domain!r}, name={self.name!r}, "
"case_sharing={self.case_sharing!r})").format(self=self)
class ComputedDocumentMixin(DocumentSchema):
"""
Use this mixin for things like CommCareCase or XFormInstance documents that take advantage
of indicator definitions.
computed_ is namespaced and may look like the following for indicators:
computed_: {
mvp_indicators: {
indicator_slug: {
version: 1,
value: "foo"
}
}
}
"""
computed_ = DictProperty()
computed_modified_on_ = DateTimeProperty()
# a flag for the indicator pillows so that there aren't any Document Update Conflicts
initial_processing_complete = BooleanProperty()
def update_indicator(self,
indicator_def,
save_on_update=True,
logger=None):
existing_indicators = self.computed_.get(indicator_def.namespace, {})
updated_indicators, is_update = indicator_def.update_computed_namespace(
existing_indicators, self)
if is_update:
self.computed_[indicator_def.namespace] = updated_indicators
self.computed_modified_on_ = datetime.datetime.utcnow()
if logger:
logger.info(
"[INDICATOR %(namespace)s %(domain)s] Updating %(indicator_type)s:%(indicator_slug)s "
"in %(document_type)s [%(document_id)s]." % {
'namespace': indicator_def.namespace,
'domain': indicator_def.domain,
'indicator_type': indicator_def.__class__.__name__,
'indicator_slug': indicator_def.slug,
'document_type': self.__class__.__name__,
'document_id': self._id,
})
if save_on_update:
self.save(**get_safe_write_kwargs())
if logger:
logger.debug("Saved %s." % self._id)
return is_update
def update_indicators_in_bulk(self,
indicators,
save_on_update=True,
logger=None):
is_update = False
for indicator in indicators:
try:
if self.update_indicator(indicator,
save_on_update=False,
logger=logger):
is_update = True
except Exception:
logger.exception(
"[INDICATOR %(namespace)s %(domain)s] Failed to update %(indicator_type)s: "
"%(indicator_slug)s in %(document_type)s [%(document_id)s]."
% {
'namespace': indicator.namespace,
'domain': indicator.domain,
'indicator_type': indicator.__class__.__name__,
'indicator_slug': indicator.slug,
'document_type': self.__class__.__name__,
'document_id': self._id,
})
if is_update and save_on_update:
try:
self.save(**get_safe_write_kwargs())
if logger:
logger.info("Saved %s." % self._id)
except ResourceConflict:
logger.error(
"[INDICATOR %(domain)s] Resource conflict failed to save document indicators for "
"%(document_type)s [%(document_id)s]." % {
'domain': self.domain,
'document_type': self.__class__.__name__,
'document_id': self._id,
})
return is_update
class RepeatRecord(Document):
"""
An record of a particular instance of something that needs to be forwarded
with a link to the proper repeater object
"""
domain = StringProperty()
repeater_id = StringProperty()
repeater_type = StringProperty()
payload_id = StringProperty()
overall_tries = IntegerProperty(default=0)
max_possible_tries = IntegerProperty(default=6)
attempts = ListProperty(RepeatRecordAttempt)
cancelled = BooleanProperty(default=False)
registered_on = DateTimeProperty()
last_checked = DateTimeProperty()
failure_reason = StringProperty()
next_check = DateTimeProperty()
succeeded = BooleanProperty(default=False)
@property
def record_id(self):
return self._id
@classmethod
def wrap(cls, data):
should_bootstrap_attempts = ('attempts' not in data)
self = super(RepeatRecord, cls).wrap(data)
if should_bootstrap_attempts and self.last_checked:
assert not self.attempts
self.attempts = [
RepeatRecordAttempt(
cancelled=self.cancelled,
datetime=self.last_checked,
failure_reason=self.failure_reason,
success_response=None,
next_check=self.next_check,
succeeded=self.succeeded,
)
]
return self
@property
@memoized
def repeater(self):
try:
return Repeater.get(self.repeater_id)
except ResourceNotFound:
return None
@property
def url(self):
warnings.warn(
"RepeatRecord.url is deprecated. Use Repeater.get_url instead",
DeprecationWarning)
if self.repeater:
return self.repeater.get_url(self)
@property
def state(self):
state = RECORD_PENDING_STATE
if self.succeeded:
state = RECORD_SUCCESS_STATE
elif self.cancelled:
state = RECORD_CANCELLED_STATE
elif self.failure_reason:
state = RECORD_FAILURE_STATE
return state
@classmethod
def all(cls, domain=None, due_before=None, limit=None):
json_now = json_format_datetime(due_before or datetime.utcnow())
repeat_records = RepeatRecord.view(
"repeaters/repeat_records_by_next_check",
startkey=[domain],
endkey=[domain, json_now, {}],
include_docs=True,
reduce=False,
limit=limit,
)
return repeat_records
@classmethod
def count(cls, domain=None):
results = RepeatRecord.view(
"repeaters/repeat_records_by_next_check",
startkey=[domain],
endkey=[domain, {}],
reduce=True,
).one()
return results['value'] if results else 0
def add_attempt(self, attempt):
self.attempts.append(attempt)
self.last_checked = attempt.datetime
self.next_check = attempt.next_check
self.succeeded = attempt.succeeded
self.cancelled = attempt.cancelled
self.failure_reason = attempt.failure_reason
def get_numbered_attempts(self):
for i, attempt in enumerate(self.attempts):
yield i + 1, attempt
def postpone_by(self, duration):
self.last_checked = datetime.utcnow()
self.next_check = self.last_checked + duration
self.save()
def make_set_next_try_attempt(self, failure_reason):
assert self.succeeded is False
assert self.next_check is not None
now = datetime.utcnow()
return RepeatRecordAttempt(
cancelled=False,
datetime=now,
failure_reason=failure_reason,
success_response=None,
next_check=now + _get_retry_interval(self.last_checked, now),
succeeded=False,
)
def try_now(self):
# try when we haven't succeeded and either we've
# never checked, or it's time to check again
return not self.succeeded
def get_payload(self):
return self.repeater.get_payload(self)
def get_attempt_info(self):
return self.repeater.get_attempt_info(self)
def handle_payload_exception(self, exception):
now = datetime.utcnow()
return RepeatRecordAttempt(
cancelled=True,
datetime=now,
failure_reason=str(exception),
success_response=None,
next_check=None,
succeeded=False,
)
def fire(self, force_send=False):
if self.try_now() or force_send:
self.overall_tries += 1
try:
attempt = self.repeater.fire_for_record(self)
except Exception as e:
log_repeater_error_in_datadog(self.domain,
status_code=None,
repeater_type=self.repeater_type)
attempt = self.handle_payload_exception(e)
raise
finally:
# pycharm warns attempt might not be defined.
# that'll only happen if fire_for_record raise a non-Exception exception (e.g. SIGINT)
# or handle_payload_exception raises an exception. I'm okay with that. -DMR
self.add_attempt(attempt)
self.save()
@staticmethod
def _format_response(response):
if not _is_response(response):
return None
response_body = getattr(response, "text", "")
return '{}: {}.\n{}'.format(response.status_code, response.reason,
response_body)
def handle_success(self, response):
"""
Log success in Datadog and return a success RepeatRecordAttempt.
``response`` can be a Requests response instance, or True if the
payload did not result in an API call.
"""
now = datetime.utcnow()
if _is_response(response):
# ^^^ Don't bother logging success in Datadog if the payload
# did not need to be sent. (This can happen with DHIS2 if
# the form that triggered the forwarder doesn't contain data
# for a DHIS2 Event.)
log_repeater_success_in_datadog(self.domain, response.status_code,
self.repeater_type)
return RepeatRecordAttempt(
cancelled=False,
datetime=now,
failure_reason=None,
success_response=self._format_response(response),
next_check=None,
succeeded=True,
info=self.get_attempt_info(),
)
def handle_failure(self, response):
"""Do something with the response if the repeater fails
"""
return self._make_failure_attempt(self._format_response(response),
response)
def handle_exception(self, exception):
"""handle internal exceptions
"""
return self._make_failure_attempt(str(exception), None)
def _make_failure_attempt(self, reason, response):
log_repeater_error_in_datadog(
self.domain, response.status_code if response else None,
self.repeater_type)
if self.repeater.allow_retries(
response) and self.overall_tries < self.max_possible_tries:
return self.make_set_next_try_attempt(reason)
else:
now = datetime.utcnow()
return RepeatRecordAttempt(
cancelled=True,
datetime=now,
failure_reason=reason,
success_response=None,
next_check=None,
succeeded=False,
info=self.get_attempt_info(),
)
def cancel(self):
self.next_check = None
self.cancelled = True
def attempt_forward_now(self):
from corehq.motech.repeaters.tasks import process_repeat_record
def is_ready():
return self.next_check < datetime.utcnow()
def already_processed():
return self.succeeded or self.cancelled or self.next_check is None
if already_processed() or not is_ready():
return
# Set the next check to happen an arbitrarily long time from now so
# if something goes horribly wrong with the delayed task it will not
# be lost forever. A check at this time is expected to occur rarely,
# if ever, because `process_repeat_record` will usually succeed or
# reset the next check to sometime sooner.
self.next_check = datetime.utcnow() + timedelta(hours=48)
try:
self.save()
except ResourceConflict:
# Another process beat us to the punch. This takes advantage
# of Couch DB's optimistic locking, which prevents a process
# with stale data from overwriting the work of another.
return
process_repeat_record.delay(self)
def requeue(self):
self.cancelled = False
self.succeeded = False
self.failure_reason = ''
self.overall_tries = 0
self.next_check = datetime.utcnow()
class XFormInstance(DeferredBlobMixin, SafeSaveDocument, ComputedDocumentMixin,
CouchDocLockableMixIn, AbstractXFormInstance):
"""An XForms instance."""
domain = StringProperty()
app_id = StringProperty()
xmlns = StringProperty()
form = DictProperty()
received_on = DateTimeProperty()
server_modified_on = DateTimeProperty()
# Used to tag forms that were forcefully submitted
# without a touchforms session completing normally
partial_submission = BooleanProperty(default=False)
history = SchemaListProperty(XFormOperation)
auth_context = DictProperty()
submit_ip = StringProperty()
path = StringProperty()
openrosa_headers = DictProperty()
last_sync_token = StringProperty()
# almost always a datetime, but if it's not parseable it'll be a string
date_header = DefaultProperty()
build_id = StringProperty()
export_tag = DefaultProperty(name='#export_tag')
_blobdb_type_code = CODES.form_xml
class Meta(object):
app_label = 'couchforms'
@classmethod
def get(cls, docid, rev=None, db=None, dynamic_properties=True):
# copied and tweaked from the superclass's method
if not db:
db = cls.get_db()
cls._allow_dynamic_properties = dynamic_properties
# on cloudant don't get the doc back until all nodes agree
# on the copy, to avoid race conditions
extras = get_safe_read_kwargs()
try:
if cls == XFormInstance:
doc = db.get(docid, rev=rev, **extras)
if doc['doc_type'] in doc_types():
return doc_types()[doc['doc_type']].wrap(doc)
try:
return XFormInstance.wrap(doc)
except WrappingAttributeError:
raise ResourceNotFound(
"The doc with _id {} and doc_type {} can't be wrapped "
"as an XFormInstance".format(docid, doc['doc_type']))
return db.get(docid, rev=rev, wrapper=cls.wrap, **extras)
except ResourceNotFound:
raise XFormNotFound(docid)
@property
def form_id(self):
return self._id
@form_id.setter
def form_id(self, value):
self._id = value
@property
def form_data(self):
return DictProperty().unwrap(self.form)[1]
@property
def user_id(self):
return getattr(self.metadata, 'userID', None)
@property
def is_error(self):
return self.doc_type != 'XFormInstance'
@property
def is_duplicate(self):
return self.doc_type == 'XFormDuplicate'
@property
def is_archived(self):
return self.doc_type == 'XFormArchived'
@property
def is_deprecated(self):
return self.doc_type == 'XFormDeprecated'
@property
def is_submission_error_log(self):
return self.doc_type == 'SubmissionErrorLog'
@property
def is_deleted(self):
return self.doc_type.endswith(DELETED_SUFFIX)
@property
def is_normal(self):
return self.doc_type == 'XFormInstance'
@property
def deletion_id(self):
return getattr(self, '-deletion_id', None)
@property
def deletion_date(self):
return getattr(self, '-deletion_date', None)
@property
def metadata(self):
if const.TAG_META in self.form:
return Metadata.wrap(
clean_metadata(self.to_json()[const.TAG_FORM][const.TAG_META]))
return None
@property
def time_start(self):
# Will be addressed in https://github.com/dimagi/commcare-hq/pull/19391/
return None
@property
def time_end(self):
return None
@property
def commcare_version(self):
return str(self.metadata.commcare_version)
@property
def app_version(self):
return None
def __str__(self):
return "%s (%s)" % (self.type, self.xmlns)
def save(self, **kwargs):
# HACK: cloudant has a race condition when saving newly created forms
# which throws errors here. use a try/retry loop here to get around
# it until we find something more stable.
RETRIES = 10
SLEEP = 0.5 # seconds
tries = 0
self.server_modified_on = datetime.datetime.utcnow()
while True:
try:
return super(XFormInstance, self).save(**kwargs)
except PreconditionFailed:
if tries == 0:
logging.error('doc %s got a precondition failed', self._id)
if tries < RETRIES:
tries += 1
time.sleep(SLEEP)
else:
raise
def get_data(self, path):
"""
Evaluates an xpath expression like: path/to/node and returns the value
of that element, or None if there is no value.
"""
return safe_index(self, path.split("/"))
def soft_delete(self):
self.doc_type += DELETED_SUFFIX
self.save()
def get_xml(self):
try:
return self.fetch_attachment(ATTACHMENT_NAME)
except ResourceNotFound:
logging.warn("no xml found for %s, trying old attachment scheme." %
self.get_id)
try:
return self[const.TAG_XML]
except AttributeError:
return None
def get_attachment(self, attachment_name):
return self.fetch_attachment(attachment_name)
def get_xml_element(self):
xml_string = self.get_xml()
if not xml_string:
return None
return self._xml_string_to_element(xml_string)
def _xml_string_to_element(self, xml_string):
def _to_xml_element(payload):
if isinstance(payload, six.text_type):
payload = payload.encode('utf-8', errors='replace')
return etree.fromstring(payload)
try:
return _to_xml_element(xml_string)
except XMLSyntaxError:
# there is a bug at least in pact code that double
# saves a submission in a way that the attachments get saved in a base64-encoded format
decoded_payload = base64.b64decode(xml_string)
element = _to_xml_element(decoded_payload)
# in this scenario resave the attachment properly in case future calls circumvent this method
self.save()
self.put_attachment(decoded_payload, ATTACHMENT_NAME)
return element
def put_attachment(self, content, name, **kw):
if kw.get("type_code") is None:
kw["type_code"] = (CODES.form_xml if name == ATTACHMENT_NAME else
CODES.form_attachment)
return super(XFormInstance, self).put_attachment(content, name, **kw)
@property
def attachments(self):
"""
Get the extra attachments for this form. This will not include
the form itself
"""
def _meta_to_json(meta):
is_image = False
if meta.content_type is not None:
is_image = True if meta.content_type.startswith(
'image/') else False
meta_json = meta.to_json()
meta_json['is_image'] = is_image
return meta_json
return {
name: _meta_to_json(meta)
for name, meta in six.iteritems(self.blobs)
if name != ATTACHMENT_NAME
}
def xml_md5(self):
return hashlib.md5(self.get_xml()).hexdigest()
def archive(self, user_id=None):
if self.is_archived:
return
# If this archive was initiated by a user, delete all other stubs for this action so that this action
# isn't overridden
from couchforms.models import UnfinishedArchiveStub
UnfinishedArchiveStub.objects.filter(
xform_id=self.form_id).all().delete()
from corehq.form_processor.submission_process_tracker import unfinished_archive
with unfinished_archive(instance=self, user_id=user_id,
archive=True) as archive_stub:
self.doc_type = "XFormArchived"
self.history.append(
XFormOperation(
user=user_id,
operation='archive',
))
self.save()
archive_stub.archive_history_updated()
xform_archived.send(sender="couchforms", xform=self)
def unarchive(self, user_id=None):
if not self.is_archived:
return
# If this unarchive was initiated by a user, delete all other stubs for this action so that this action
# isn't overridden
from couchforms.models import UnfinishedArchiveStub
UnfinishedArchiveStub.objects.filter(
xform_id=self.form_id).all().delete()
from corehq.form_processor.submission_process_tracker import unfinished_archive
with unfinished_archive(instance=self, user_id=user_id,
archive=False) as archive_stub:
self.doc_type = "XFormInstance"
self.history.append(
XFormOperation(
user=user_id,
operation='unarchive',
))
XFormInstance.save(
self
) # subclasses explicitly set the doc type so force regular save
archive_stub.archive_history_updated()
xform_unarchived.send(sender="couchforms", xform=self)
def publish_archive_action_to_kafka(self, user_id, archive):
from couchforms.models import UnfinishedArchiveStub
from corehq.form_processor.submission_process_tracker import unfinished_archive
# Delete the original stub
UnfinishedArchiveStub.objects.filter(
xform_id=self.form_id).all().delete()
with unfinished_archive(instance=self,
user_id=user_id,
archive=archive):
if archive:
xform_archived.send(sender="couchforms", xform=self)
else:
xform_unarchived.send(sender="couchforms", xform=self)
class DataSourceConfiguration(CachedCouchDocumentMixin, Document,
AbstractUCRDataSource):
"""
A data source configuration. These map 1:1 with database tables that get created.
Each data source can back an arbitrary number of reports.
"""
domain = StringProperty(required=True)
engine_id = StringProperty(default=UCR_ENGINE_ID)
backend_id = StringProperty(default=UCR_SQL_BACKEND) # no longer used
referenced_doc_type = StringProperty(required=True)
table_id = StringProperty(required=True)
display_name = StringProperty()
base_item_expression = DictProperty()
configured_filter = DictProperty()
configured_indicators = ListProperty()
named_expressions = DictProperty()
named_filters = DictProperty()
meta = SchemaProperty(DataSourceMeta)
is_deactivated = BooleanProperty(default=False)
last_modified = DateTimeProperty()
asynchronous = BooleanProperty(default=False)
sql_column_indexes = SchemaListProperty(SQLColumnIndexes)
disable_destructive_rebuild = BooleanProperty(default=False)
sql_settings = SchemaProperty(SQLSettings)
validations = SchemaListProperty(Validation)
mirrored_engine_ids = ListProperty(default=[])
class Meta(object):
# prevent JsonObject from auto-converting dates etc.
string_conversions = ()
def __str__(self):
return '{} - {}'.format(self.domain, self.display_name)
def save(self, **params):
self.last_modified = datetime.utcnow()
super(DataSourceConfiguration, self).save(**params)
@property
def data_source_id(self):
return self._id
def filter(self, document, eval_context=None):
if eval_context is None:
eval_context = EvaluationContext(document)
filter_fn = self._get_main_filter()
return filter_fn(document, eval_context)
def deleted_filter(self, document):
filter_fn = self._get_deleted_filter()
return filter_fn and filter_fn(document, EvaluationContext(
document, 0))
@property
def has_validations(self):
return len(self.validations) > 0
def validate_document(self, document, eval_context=None):
if eval_context is None:
eval_context = EvaluationContext(document)
errors = []
for validation in self._validations():
if validation.validation_function(document, eval_context) is False:
errors.append((validation.name, validation.error_message))
if errors:
raise ValidationError(errors)
@memoized
def _validations(self):
return [
_Validation(
validation.name, validation.error_message,
FilterFactory.from_spec(validation.expression,
context=self.get_factory_context()))
for validation in self.validations
]
@memoized
def _get_main_filter(self):
return self._get_filter([self.referenced_doc_type])
@memoized
def _get_deleted_filter(self):
return self._get_filter(get_deleted_doc_types(
self.referenced_doc_type),
include_configured=False)
def _get_filter(self, doc_types, include_configured=True):
if not doc_types:
return None
extras = ([self.configured_filter]
if include_configured and self.configured_filter else [])
built_in_filters = [
self._get_domain_filter_spec(),
{
'type':
'or',
'filters': [{
"type": "boolean_expression",
"expression": {
"type": "property_name",
"property_name": "doc_type",
},
"operator": "eq",
"property_value": doc_type,
} for doc_type in doc_types],
},
]
return FilterFactory.from_spec(
{
'type': 'and',
'filters': built_in_filters + extras,
},
context=self.get_factory_context(),
)
def _get_domain_filter_spec(self):
return {
"type": "boolean_expression",
"expression": {
"type": "property_name",
"property_name": "domain",
},
"operator": "eq",
"property_value": self.domain,
}
@property
@memoized
def named_expression_objects(self):
named_expression_specs = deepcopy(self.named_expressions)
named_expressions = {}
spec_error = None
while named_expression_specs:
number_generated = 0
for name, expression in list(named_expression_specs.items()):
try:
named_expressions[name] = ExpressionFactory.from_spec(
expression,
FactoryContext(named_expressions=named_expressions,
named_filters={}))
number_generated += 1
del named_expression_specs[name]
except BadSpecError as bad_spec_error:
# maybe a nested name resolution issue, try again on the next pass
spec_error = bad_spec_error
if number_generated == 0 and named_expression_specs:
# we unsuccessfully generated anything on this pass and there are still unresolved
# references. we have to fail.
assert spec_error is not None
raise spec_error
return named_expressions
@property
@memoized
def named_filter_objects(self):
return {
name: FilterFactory.from_spec(
filter, FactoryContext(self.named_expression_objects, {}))
for name, filter in self.named_filters.items()
}
def get_factory_context(self):
return FactoryContext(self.named_expression_objects,
self.named_filter_objects)
@property
@memoized
def default_indicators(self):
default_indicators = [
IndicatorFactory.from_spec(
{
"column_id": "doc_id",
"type": "expression",
"display_name": "document id",
"datatype": "string",
"is_nullable": False,
"is_primary_key": True,
"expression": {
"type": "root_doc",
"expression": {
"type": "property_name",
"property_name": "_id"
}
}
}, self.get_factory_context())
]
default_indicators.append(
IndicatorFactory.from_spec({
"type": "inserted_at",
}, self.get_factory_context()))
if self.base_item_expression:
default_indicators.append(
IndicatorFactory.from_spec({
"type": "repeat_iteration",
}, self.get_factory_context()))
return default_indicators
@property
@memoized
def indicators(self):
return CompoundIndicator(
self.display_name,
self.default_indicators + [
IndicatorFactory.from_spec(indicator,
self.get_factory_context())
for indicator in self.configured_indicators
],
None,
)
@property
@memoized
def parsed_expression(self):
if self.base_item_expression:
return ExpressionFactory.from_spec(
self.base_item_expression, context=self.get_factory_context())
return None
@memoized
def get_columns(self):
return self.indicators.get_columns()
@property
@memoized
def columns_by_id(self):
return {c.id: c for c in self.get_columns()}
def get_column_by_id(self, column_id):
return self.columns_by_id.get(column_id)
def get_items(self, document, eval_context=None):
if self.filter(document, eval_context):
if not self.base_item_expression:
return [document]
else:
result = self.parsed_expression(document, eval_context)
if result is None:
return []
elif isinstance(result, list):
return result
else:
return [result]
else:
return []
def get_all_values(self, doc, eval_context=None):
if not eval_context:
eval_context = EvaluationContext(doc)
if self.has_validations:
try:
self.validate_document(doc, eval_context)
except ValidationError as e:
for error in e.errors:
InvalidUCRData.objects.get_or_create(
doc_id=doc['_id'],
indicator_config_id=self._id,
validation_name=error[0],
defaults={
'doc_type': doc['doc_type'],
'domain': doc['domain'],
'validation_text': error[1],
})
return []
rows = []
for item in self.get_items(doc, eval_context):
values = self.indicators.get_values(item, eval_context)
rows.append(values)
eval_context.increment_iteration()
return rows
def get_report_count(self):
"""
Return the number of ReportConfigurations that reference this data source.
"""
return ReportConfiguration.count_by_data_source(self.domain, self._id)
def validate_db_config(self):
mirrored_engine_ids = self.mirrored_engine_ids
if not mirrored_engine_ids:
return
if self.engine_id in mirrored_engine_ids:
raise BadSpecError(
"mirrored_engine_ids list should not contain engine_id")
for engine_id in mirrored_engine_ids:
if not connection_manager.engine_id_is_available(engine_id):
raise BadSpecError(
"DB for engine_id {} is not availble".format(engine_id))
if not connection_manager.resolves_to_unique_dbs(mirrored_engine_ids +
[self.engine_id]):
raise BadSpecError(
"No two engine_ids should point to the same database")
def validate(self, required=True):
super(DataSourceConfiguration, self).validate(required)
# these two properties implicitly call other validation
self._get_main_filter()
self._get_deleted_filter()
# validate indicators and column uniqueness
columns = [c.id for c in self.indicators.get_columns()]
unique_columns = set(columns)
if len(columns) != len(unique_columns):
for column in set(columns):
columns.remove(column)
raise DuplicateColumnIdError(columns=columns)
if self.referenced_doc_type not in VALID_REFERENCED_DOC_TYPES:
raise BadSpecError(
_('Report contains invalid referenced_doc_type: {}').format(
self.referenced_doc_type))
self.parsed_expression
self.pk_columns
@classmethod
def by_domain(cls, domain):
return get_datasources_for_domain(domain)
@classmethod
def all_ids(cls):
return [
res['id'] for res in cls.get_db().view(
'userreports/data_sources_by_build_info',
reduce=False,
include_docs=False)
]
@classmethod
def all(cls):
for result in iter_docs(cls.get_db(), cls.all_ids()):
yield cls.wrap(result)
@property
def is_static(self):
return id_is_static(self._id)
def deactivate(self, initiated_by=None):
if not self.is_static:
self.is_deactivated = True
self.save()
get_indicator_adapter(self).drop_table(
initiated_by=initiated_by, source='deactivate-data-source')
def get_case_type_or_xmlns_filter(self):
"""Returns a list of case types or xmlns from the filter of this data source.
If this can't figure out the case types or xmlns's that filter, then returns [None]
Currently always returns a list because it is called by a loop in _iteratively_build_table
Could be reworked to return [] to be more pythonic
"""
if self.referenced_doc_type not in FILTER_INTERPOLATION_DOC_TYPES:
return [None]
property_name = FILTER_INTERPOLATION_DOC_TYPES[
self.referenced_doc_type]
prop_value = self._filter_interploation_helper(self.configured_filter,
property_name)
return prop_value or [None]
def _filter_interploation_helper(self, config_filter, property_name):
filter_type = config_filter.get('type')
if filter_type == 'and':
sub_config_filters = [
self._filter_interploation_helper(f, property_name)
for f in config_filter.get('filters')
]
for filter_ in sub_config_filters:
if filter_[0]:
return filter_
if filter_type != 'boolean_expression':
return [None]
if config_filter['operator'] not in ('eq', 'in'):
return [None]
expression = config_filter['expression']
if not isinstance(expression, dict):
return [None]
if expression['type'] == 'property_name' and expression[
'property_name'] == property_name:
prop_value = config_filter['property_value']
if not isinstance(prop_value, list):
prop_value = [prop_value]
return prop_value
return [None]
@property
def pk_columns(self):
columns = []
for col in self.get_columns():
if col.is_primary_key:
column_name = decode_column_name(col)
columns.append(column_name)
if self.sql_settings.primary_key:
if set(columns) != set(self.sql_settings.primary_key):
raise BadSpecError(
"Primary key columns must have is_primary_key set to true",
self.data_source_id)
columns = self.sql_settings.primary_key
return columns
class IndicatorDefinition(Document, AdminCRUDDocumentMixin):
"""
An Indicator Definition defines how to compute the indicator that lives
in the namespaced computed_ property of a case or form.
"""
namespace = StringProperty()
domain = StringProperty()
slug = StringProperty()
version = IntegerProperty()
class_path = StringProperty()
last_modified = DateTimeProperty()
_admin_crud_class = IndicatorAdminCRUDManager
_class_path = "corehq.apps.indicators.models"
_returns_multiple = False
def __init__(self, _d=None, **kwargs):
super(IndicatorDefinition, self).__init__(_d, **kwargs)
self.class_path = self._class_path
def __str__(self):
return u"\n\n%(class_name)s - Modified %(last_modified)s\n %(slug)s, domain: %(domain)s," \
u" version: %(version)s, namespace: %(namespace)s. ID: %(indicator_id)s." % {
'class_name': self.__class__.__name__,
'slug': self.slug,
'domain': self.domain,
'version': self.version,
'namespace': self.namespace,
'last_modified': (self.last_modified.strftime('%m %B %Y at %H:%M')
if self.last_modified else "Ages Ago"),
'indicator_id': self._id,
}
@classmethod
def key_properties(cls):
"""
The ordering of these property names should match the ordering of what's emitted in the first part of
the couch views used for fetching these indicators. These views currently are:
- indicators/dynamic_indicator_definitions (Couch View Indicator Defs)
- indicators/indicator_definitions (Form and Case Indicator Defs)
"""
return ["namespace", "domain", "slug"]
@classmethod
def indicator_list_view(cls):
return "indicators/indicator_definitions"
@classmethod
def _generate_couch_key(cls, version=None, reverse=False, **kwargs):
key = list()
key_prefix = list()
for p in cls.key_properties():
k = kwargs.get(p)
if k is not None:
key_prefix.append(p)
key.append(k)
key = [" ".join(key_prefix)] + key
couch_key = dict(startkey=key, endkey=key +
[{}]) if version is None else dict(key=key +
[version])
if reverse:
return dict(startkey=couch_key.get('endkey'),
endkey=couch_key.get('startkey'))
return couch_key
@classmethod
def increment_or_create_unique(cls,
namespace,
domain,
slug=None,
version=None,
**kwargs):
"""
If an indicator with the same namespace, domain, and version exists, create a new indicator with the
version number incremented.
# todo, this feels a bit buggy, so replace bulk copy indicators with
# copy to domain at some point
"""
couch_key = cls._generate_couch_key(namespace=namespace,
domain=domain,
slug=slug,
reverse=True,
**kwargs)
existing_indicator = cls.view(cls.indicator_list_view(),
reduce=False,
include_docs=True,
descending=True,
limit=1,
**couch_key).first()
if existing_indicator:
version = existing_indicator.version + 1
elif version is None:
version = 1
new_indicator = cls(version=version,
namespace=namespace,
domain=domain,
slug=slug,
**kwargs)
new_indicator.last_modified = datetime.datetime.utcnow()
new_indicator.save()
return new_indicator
@classmethod
def copy_to_domain(cls, domain, doc, override=False):
"""
This copies an indicator doc to the current domain. Intended to be used
by the export indicators feature.
:param domain: the name of the domain the indicator should be copied to
:param doc: the dictionary of kwargs to create the indicator
:param override: Whether to override the existing indicator
:return: True if indicator was copied, False if not
"""
for reserved in ['_id', '_rev', 'last_modified']:
if reserved in doc:
del doc[reserved]
couch_key = cls._generate_couch_key(domain=domain, reverse=True, **doc)
existing_indicator = cls.view(cls.indicator_list_view(),
reduce=False,
include_docs=False,
descending=True,
limit=1,
**couch_key).first()
if existing_indicator and not override:
return False
if existing_indicator:
existing_indicator.delete()
new_indicator = cls(domain=domain, **doc)
new_indicator.last_modified = datetime.datetime.utcnow()
new_indicator.save()
return True
@classmethod
@memoized
def get_current(cls,
namespace,
domain,
slug,
version=None,
wrap=True,
**kwargs):
couch_key = cls._generate_couch_key(namespace=namespace,
domain=domain,
slug=slug,
version=version,
reverse=True,
**kwargs)
results = cache_core.cached_view(cls.get_db(),
cls.indicator_list_view(),
cache_expire=60 * 60 * 6,
reduce=False,
include_docs=False,
descending=True,
**couch_key)
doc = results[0] if results else None
if wrap and doc:
try:
doc_class = to_function(
doc.get('value',
"%s.%s" % (cls._class_path, cls.__name__)))
doc_instance = doc_class.get(doc.get('id'))
return doc_instance
except Exception as e:
logging.error(
"No matching documents found for indicator %s: %s" %
(slug, e))
return None
return doc
@classmethod
def all_slugs(cls, namespace, domain, **kwargs):
couch_key = cls._generate_couch_key(namespace=namespace,
domain=domain,
reverse=True,
**kwargs)
couch_key['startkey'][0] = couch_key.get('startkey', [])[0] + ' slug'
couch_key['endkey'][0] = couch_key.get('endkey', [])[0] + ' slug'
data = cls.view(cls.indicator_list_view(),
group=True,
group_level=cls.key_properties().index('slug') + 2,
descending=True,
**couch_key).all()
return [item.get('key', [])[-1] for item in data]
@classmethod
@memoized
def get_all(cls, namespace, domain, version=None, **kwargs):
all_slugs = cls.all_slugs(namespace, domain, **kwargs)
all_indicators = list()
for slug in all_slugs:
indicator = cls.get_current(namespace,
domain,
slug,
version=version,
**kwargs)
if indicator and issubclass(indicator.__class__, cls):
all_indicators.append(indicator)
return all_indicators
@classmethod
def get_all_of_type(cls, namespace, domain, show_only_current=False):
key = ["type", namespace, domain, cls.__name__]
indicators = cls.view(cls.indicator_list_view(),
reduce=False,
include_docs=True,
startkey=key,
endkey=key + [{}]).all()
unique = {}
for ind in indicators:
if ind.base_doc == "CaseIndicatorDefinition":
specific_doc = ind.case_type
elif ind.base_doc == "FormIndicatorDefinition":
specific_doc = ind.xmlns
else:
specific_doc = "couch"
unique["%s.%s.%s" % (ind.slug, ind.namespace, specific_doc)] = ind
return unique.values()
@classmethod
def get_nice_name(cls):
return "Indicator Definition"
class Product(Document):
"""
A product, e.g. "coartem" or "tylenol"
"""
domain = StringProperty()
name = StringProperty()
unit = StringProperty()
code_ = StringProperty()
description = StringProperty()
category = StringProperty()
program_id = StringProperty()
cost = DecimalProperty()
product_data = DictProperty()
is_archived = BooleanProperty(default=False)
last_modified = DateTimeProperty()
@classmethod
def wrap(cls, data):
from corehq.apps.groups.models import dt_no_Z_re
# If "Z" is missing because of the Aug 2014 migration, then add it.
# cf. Group class
last_modified = data.get('last_modified')
if last_modified and dt_no_Z_re.match(last_modified):
data['last_modified'] += 'Z'
return super(Product, cls).wrap(data)
@classmethod
def save_docs(cls, docs, use_uuids=True, codes_by_domain=None):
from corehq.apps.commtrack.util import generate_code
codes_by_domain = codes_by_domain or {}
def get_codes(domain):
if domain not in codes_by_domain:
codes_by_domain[domain] = SQLProduct.objects.filter(domain=domain)\
.values_list('code', flat=True).distinct()
return codes_by_domain[domain]
for doc in docs:
doc.last_modified = datetime.utcnow()
if not doc['code_']:
doc['code_'] = generate_code(doc['name'],
get_codes(doc['domain']))
super(Product, cls).save_docs(docs, use_uuids)
domains = {doc['domain'] for doc in docs}
for domain in domains:
cls.clear_caches(domain)
bulk_save = save_docs
def sync_to_sql(self):
properties_to_sync = [
('product_id', '_id'),
'domain',
'name',
'is_archived',
('code', 'code_'),
'description',
'category',
'program_id',
'cost',
('units', 'unit'),
'product_data',
]
# sync properties to SQL version
sql_product, _ = SQLProduct.objects.get_or_create(product_id=self._id)
for prop in properties_to_sync:
if isinstance(prop, tuple):
sql_prop, couch_prop = prop
else:
sql_prop = couch_prop = prop
if hasattr(self, couch_prop):
setattr(sql_product, sql_prop, getattr(self, couch_prop))
sql_product.save()
def save(self, *args, **kwargs):
"""
Saving a couch version of Product will trigger
one way syncing to the SQLProduct version of this
product.
"""
# mark modified time stamp for selective syncing
self.last_modified = datetime.utcnow()
# generate code if user didn't specify one
if not self.code:
from corehq.apps.commtrack.util import generate_code
self.code = generate_code(
self.name,
SQLProduct.objects.filter(domain=self.domain).values_list(
'code', flat=True).distinct())
result = super(Product, self).save(*args, **kwargs)
self.clear_caches(self.domain)
self.sync_to_sql()
return result
@property
def code(self):
return self.code_
@code.setter
def code(self, val):
self.code_ = val.lower() if val else None
@classmethod
def clear_caches(cls, domain):
from casexml.apps.phone.utils import clear_fixture_cache
from corehq.apps.products.fixtures import ALL_CACHE_PREFIXES
for prefix in ALL_CACHE_PREFIXES:
clear_fixture_cache(domain, prefix)
@classmethod
def by_domain(cls, domain, wrap=True, include_archived=False):
queryset = SQLProduct.objects.filter(domain=domain)
if not include_archived:
queryset = queryset.filter(is_archived=False)
return list(queryset.couch_products(wrapped=wrap))
@classmethod
def _export_attrs(cls):
return [
('name', str),
('unit', str),
'description',
'category',
('program_id', str),
('cost', lambda a: Decimal(a) if a else None),
]
def to_dict(self):
from corehq.apps.commtrack.util import encode_if_needed
product_dict = {}
product_dict['id'] = self._id
product_dict['product_id'] = self.code_
for attr in self._export_attrs():
real_attr = attr[0] if isinstance(attr, tuple) else attr
product_dict[real_attr] = encode_if_needed(getattr(
self, real_attr))
return product_dict
def custom_property_dict(self):
from corehq.apps.commtrack.util import encode_if_needed
property_dict = {}
for prop, val in self.product_data.items():
property_dict['data: ' + prop] = encode_if_needed(val)
return property_dict
def archive(self):
"""
Mark a product as archived. This will cause it (and its data)
to not show up in default Couch and SQL views.
"""
self.is_archived = True
self.save()
def unarchive(self):
"""
Unarchive a product, causing it (and its data) to show
up in Couch and SQL views again.
"""
if self.code:
if SQLProduct.objects.filter(domain=self.domain,
code=self.code,
is_archived=False).exists():
raise DuplicateProductCodeException()
self.is_archived = False
self.save()
@classmethod
def from_excel(cls, row, custom_data_validator):
if not row:
return None
id = row.get('id')
if id:
try:
p = cls.get(id)
except ResourceNotFound:
raise InvalidProductException(
_("Product with ID '{product_id}' could not be found!").
format(product_id=id))
else:
p = cls()
p.code = str(row.get('product_id') or '')
for attr in cls._export_attrs():
key = attr[0] if isinstance(attr, tuple) else attr
if key in row:
val = row[key]
if val is None:
val = ''
if isinstance(attr, tuple):
val = attr[1](val)
setattr(p, key, val)
else:
break
if not p.code:
raise InvalidProductException(
_('Product ID is a required field and cannot be blank!'))
if not p.name:
raise InvalidProductException(
_('Product name is a required field and cannot be blank!'))
custom_data = row.get('data', {})
error = custom_data_validator(custom_data)
if error:
raise InvalidProductException(error)
p.product_data = custom_data
p.product_data.update(row.get('uncategorized_data', {}))
return p
class AtomFeedStatus(DocumentSchema):
last_polled_at = DateTimeProperty(default=None)
# The first time the feed is polled, don't replay all the changes
# since OpenMRS was installed. Start from the most recent changes.
last_page = StringProperty(default='recent')
