class FormQuestionSchema(Document):
"""
Contains information about the questions for a specific form
specifically the options that are available (or have ever been available) for
any multi-select questions.
Calling `update_schema` will load the app and any saved versions of the app
that have not already been processed and update the question schema with
any new options.
"""
domain = StringProperty(required=True)
app_id = StringProperty(required=True)
xmlns = StringProperty(required=True)
last_processed_version = IntegerProperty(default=0)
processed_apps = SetProperty(str)
apps_with_errors = SetProperty(str)
question_schema = SchemaDictProperty(QuestionMeta)
class Meta(object):
app_label = 'export'
@classmethod
def _get_id(cls, domain, app_id, xmlns):
def _none_to_empty_string(str):
return str if str is not None else ''
key = list(map(_none_to_empty_string, [domain, app_id, xmlns]))
return hashlib.sha1(':'.join(key).encode('utf-8')).hexdigest()
@classmethod
def get_by_key(cls, domain, app_id, xmlns):
_id = cls._get_id(domain, app_id, xmlns)
return cls.get(_id)
@classmethod
def get_or_create(cls, domain, app_id, xmlns):
try:
schema = cls.get_by_key(domain, app_id, xmlns)
except ResourceNotFound:
old_schemas = FormQuestionSchema.view(
'form_question_schema/by_xmlns',
key=[domain, app_id, xmlns],
include_docs=True).all()
if old_schemas:
doc = old_schemas[0].to_json()
del doc['_id']
del doc['_rev']
schema = FormQuestionSchema.wrap(doc)
schema.save()
for old in old_schemas:
old.delete()
else:
schema = FormQuestionSchema(domain=domain,
app_id=app_id,
xmlns=xmlns)
schema.save()
return schema
def validate(self, required=True):
# this isn't always set, so set to empty strings if not found
if self.app_id is None:
self.app_id = ''
super(FormQuestionSchema, self).validate(required=required)
if not self.get_id:
self._id = self._get_id(self.domain, self.app_id, self.xmlns)
def update_schema(self):
all_app_ids = get_build_ids_after_version(self.domain, self.app_id,
self.last_processed_version)
all_seen_apps = self.apps_with_errors | self.processed_apps
to_process = [
app_id for app_id in all_app_ids if app_id not in all_seen_apps
]
if self.app_id not in all_seen_apps:
to_process.append(self.app_id)
for app_doc in iter_docs(Application.get_db(), to_process):
if is_remote_app(app_doc):
continue
app = Application.wrap(app_doc)
try:
self.update_for_app(app)
except AppManagerException:
self.apps_with_errors.add(app.get_id)
self.last_processed_version = app.version
if to_process:
self.save()
def update_for_app(self, app):
xform = app.get_xform_by_xmlns(self.xmlns, log_missing=False)
if xform:
prefix = '/{}/'.format(xform.data_node.tag_name)
def to_json_path(xml_path):
if not xml_path:
return
if xml_path.startswith(prefix):
xml_path = xml_path[len(prefix):]
return 'form.{}'.format(xml_path.replace('/', '.'))
for question in xform.get_questions(app.langs):
question_path = to_json_path(question['value'])
if question['tag'] == 'select':
meta = self.question_schema.get(
question_path,
QuestionMeta(
repeat_context=to_json_path(question['repeat'])))
for opt in question['options']:
if opt['value'] not in meta.options:
meta.options.append(opt['value'])
self.question_schema[question_path] = meta
else:
# In the event that a question was previously a multi-select and not one any longer,
# we need to clear the question schema
self.question_schema.pop(question_path, None)
self.processed_apps.add(app.get_id)
self.last_processed_version = app.version
class SimplifiedSyncLog(AbstractSyncLog):
"""
New, simplified sync log class that is used by ownership cleanliness restore.
Just maintains a flat list of case IDs on the phone rather than the case/dependent state
lists from the SyncLog class.
"""
log_format = StringProperty(default=LOG_FORMAT_SIMPLIFIED)
case_ids_on_phone = SetProperty(six.text_type)
# this is a subset of case_ids_on_phone used to flag that a case is only around because it has dependencies
# this allows us to purge it if possible from other actions
dependent_case_ids_on_phone = SetProperty(six.text_type)
owner_ids_on_phone = SetProperty(six.text_type)
index_tree = SchemaProperty(IndexTree) # index tree of subcases / children
extension_index_tree = SchemaProperty(
IndexTree) # index tree of extensions
closed_cases = SetProperty(six.text_type)
extensions_checked = BooleanProperty(default=False)
device_id = StringProperty()
_purged_cases = None
@property
def purged_cases(self):
if self._purged_cases is None:
self._purged_cases = set()
return self._purged_cases
def case_count(self):
return len(self.case_ids_on_phone)
def phone_is_holding_case(self, case_id):
"""
Whether the phone currently has a case, according to this sync log
"""
return case_id in self.case_ids_on_phone
def get_footprint_of_cases_on_phone(self):
return list(self.case_ids_on_phone)
@property
def primary_case_ids(self):
return self.case_ids_on_phone - self.dependent_case_ids_on_phone
def purge(self, case_id, xform_id=None):
"""
This happens in 3 phases, and recursively tries to purge outgoing indices of purged cases.
Definitions:
-----------
A case is *relevant* if:
- it is open and owned or,
- it has a relevant child or,
- it has a relevant extension or,
- it is the extension of a relevant case.
A case is *available* if:
- it is open and not an extension case or,
- it is open and is the extension of an available case.
A case is *live* if:
- it is owned and available or,
- it has a live child or,
- it has a live extension or,
- it is the exension of a live case.
Algorithm:
----------
1. Mark *relevant* cases
Mark all open cases owned by the user relevant. Traversing all outgoing child
and extension indexes, as well as all incoming extension indexes, mark all
touched cases relevant.
2. Mark *available* cases
Mark all relevant cases that are open and have no outgoing extension indexes
as available. Traverse incoming extension indexes which don't lead to closed
cases, mark all touched cases as available.
3. Mark *live* cases
Mark all relevant, owned, available cases as live. Traverse incoming
extension indexes which don't lead to closed cases, mark all touched
cases as live.
"""
_get_logger().debug("purging: {}".format(case_id))
self.dependent_case_ids_on_phone.add(case_id)
relevant = self._get_relevant_cases(case_id)
available = self._get_available_cases(relevant)
live = self._get_live_cases(available)
to_remove = (relevant - self.purged_cases) - live
self._remove_cases_purge_indices(to_remove, case_id, xform_id)
def _get_relevant_cases(self, case_id):
"""
Mark all open cases owned by the user relevant. Traversing all outgoing child
and extension indexes, as well as all incoming extension indexes,
mark all touched cases relevant.
"""
relevant = IndexTree.get_all_dependencies(
case_id,
child_index_tree=self.index_tree,
extension_index_tree=self.extension_index_tree,
)
_get_logger().debug("Relevant cases of {}: {}".format(
case_id, relevant))
return relevant
def _get_available_cases(self, relevant):
"""
Mark all relevant cases that are open and have no outgoing extension indexes
as available. Traverse incoming extension indexes which don't lead to closed
cases, mark all touched cases as available
"""
incoming_extensions = self.extension_index_tree.reverse_indices
available = {
case
for case in relevant if case not in self.closed_cases and (
not self.extension_index_tree.indices.get(case)
or self.index_tree.indices.get(case))
}
new_available = set() | available
while new_available:
case_to_check = new_available.pop()
for incoming_extension in incoming_extensions.get(
case_to_check, []):
closed = incoming_extension in self.closed_cases
purged = incoming_extension in self.purged_cases
if not closed and not purged:
new_available.add(incoming_extension)
available = available | new_available
_get_logger().debug("Available cases: {}".format(available))
return available
def _get_live_cases(self, available):
"""
Mark all relevant, owned, available cases as live. Traverse incoming
extension indexes which don't lead to closed cases, mark all touched
cases as available.
"""
primary_case_ids = self.primary_case_ids
live = available & primary_case_ids
new_live = set() | live
checked = set()
while new_live:
case_to_check = new_live.pop()
checked.add(case_to_check)
new_live = new_live | IndexTree.get_all_outgoing_cases(
case_to_check, self.index_tree,
self.extension_index_tree) - self.purged_cases
new_live = new_live | IndexTree.traverse_incoming_extensions(
case_to_check,
self.extension_index_tree,
frozenset(self.closed_cases),
) - self.purged_cases
new_live = new_live - checked
live = live | new_live
_get_logger().debug("live cases: {}".format(live))
return live
def _remove_cases_purge_indices(self, all_to_remove, checked_case_id,
xform_id):
"""Remove all cases marked for removal. Traverse child cases and try to purge those too."""
_get_logger().debug("cases to to_remove: {}".format(all_to_remove))
for to_remove in all_to_remove:
indices = self.index_tree.indices.get(to_remove, {})
self._remove_case(to_remove, all_to_remove, checked_case_id,
xform_id)
for referenced_case in indices.values():
is_dependent_case = referenced_case in self.dependent_case_ids_on_phone
already_primed_for_removal = referenced_case in all_to_remove
if is_dependent_case and not already_primed_for_removal and referenced_case != checked_case_id:
self.purge(referenced_case, xform_id)
def _remove_case(self, to_remove, all_to_remove, checked_case_id,
xform_id):
"""Removes case from index trees, case_ids_on_phone and dependent_case_ids_on_phone if pertinent"""
_get_logger().debug('removing: {}'.format(to_remove))
deleted_indices = self.index_tree.indices.pop(to_remove, {})
deleted_indices.update(
self.extension_index_tree.indices.pop(to_remove, {}))
self._validate_case_removal(to_remove, all_to_remove, deleted_indices,
checked_case_id, xform_id)
try:
self.case_ids_on_phone.remove(to_remove)
except KeyError:
should_fail_softly = not xform_id or _domain_has_legacy_toggle_set(
)
if should_fail_softly:
pass
else:
# this is only a soft assert for now because of http://manage.dimagi.com/default.asp?181443
# we should convert back to a real Exception when we stop getting any of these
_assert = soft_assert(notify_admins=True,
exponential_backoff=False)
_assert(
False, 'case already remove from synclog', {
'case_id': to_remove,
'synclog_id': self._id,
'form_id': xform_id
})
else:
self.purged_cases.add(to_remove)
if to_remove in self.dependent_case_ids_on_phone:
self.dependent_case_ids_on_phone.remove(to_remove)
def _validate_case_removal(self, case_to_remove, all_to_remove,
deleted_indices, checked_case_id, xform_id):
"""Traverse immediate outgoing indices. Validate that these are also candidates for removal."""
if case_to_remove == checked_case_id:
return
# Logging removed temporarily: https://github.com/dimagi/commcare-hq/pull/16259#issuecomment-303176217
# for index in deleted_indices.values():
# if xform_id and not _domain_has_legacy_toggle_set():
# # unblocking http://manage.dimagi.com/default.asp?185850
# _assert = soft_assert(send_to_ops=False, log_to_file=True, exponential_backoff=True,
# fail_if_debug=True)
# _assert(index in (all_to_remove | set([checked_case_id])),
# "expected {} in {} but wasn't".format(index, all_to_remove))
def _add_primary_case(self, case_id):
self.case_ids_on_phone.add(case_id)
if case_id in self.dependent_case_ids_on_phone:
self.dependent_case_ids_on_phone.remove(case_id)
def _add_index(self, index, case_update):
_get_logger().debug('adding index {} --<{}>--> {} ({}).'.format(
index.case_id, index.relationship, index.referenced_id,
index.identifier))
if index.relationship == const.CASE_INDEX_EXTENSION:
self._add_extension_index(index, case_update)
else:
self._add_child_index(index)
def _add_extension_index(self, index, case_update):
assert index.relationship == const.CASE_INDEX_EXTENSION
self.extension_index_tree.set_index(index.case_id, index.identifier,
index.referenced_id)
if index.referenced_id not in self.case_ids_on_phone:
self.case_ids_on_phone.add(index.referenced_id)
self.dependent_case_ids_on_phone.add(index.referenced_id)
case_child_indices = [
idx for idx in case_update.indices_to_add
if idx.relationship == const.CASE_INDEX_CHILD
and idx.referenced_id == index.referenced_id
]
if not case_child_indices and not case_update.is_live:
# this case doesn't also have child indices, and it is not owned, so it is dependent
self.dependent_case_ids_on_phone.add(index.case_id)
def _add_child_index(self, index):
assert index.relationship == const.CASE_INDEX_CHILD
self.index_tree.set_index(index.case_id, index.identifier,
index.referenced_id)
if index.referenced_id not in self.case_ids_on_phone:
self.case_ids_on_phone.add(index.referenced_id)
self.dependent_case_ids_on_phone.add(index.referenced_id)
def _delete_index(self, index):
self.index_tree.delete_index(index.case_id, index.identifier)
self.extension_index_tree.delete_index(index.case_id, index.identifier)
def update_phone_lists(self, xform, case_list):
made_changes = False
_get_logger().debug('updating sync log for {}'.format(self.user_id))
_get_logger().debug('case ids before update: {}'.format(', '.join(
self.case_ids_on_phone)))
_get_logger().debug('dependent case ids before update: {}'.format(
', '.join(self.dependent_case_ids_on_phone)))
_get_logger().debug('index tree before update: {}'.format(
self.index_tree))
_get_logger().debug('extension index tree before update: {}'.format(
self.extension_index_tree))
class CaseUpdate(object):
def __init__(self, case_id, owner_ids_on_phone):
self.case_id = case_id
self.owner_ids_on_phone = owner_ids_on_phone
self.was_live_previously = True
self.final_owner_id = None
self.is_closed = None
self.indices_to_add = []
self.indices_to_delete = []
@property
def extension_indices_to_add(self):
return [
index for index in self.indices_to_add
if index.relationship == const.CASE_INDEX_EXTENSION
]
def has_extension_indices_to_add(self):
return len(self.extension_indices_to_add) > 0
@property
def is_live(self):
"""returns whether an update is live for a specifc set of owner_ids"""
if self.is_closed:
return False
elif self.final_owner_id is None:
# we likely didn't touch owner_id so just default to whatever it was previously
return self.was_live_previously
else:
return self.final_owner_id in self.owner_ids_on_phone
ShortIndex = namedtuple(
'ShortIndex',
['case_id', 'identifier', 'referenced_id', 'relationship'])
# this is a variable used via closures in the function below
owner_id_map = {}
def get_latest_owner_id(case_id, action=None):
# "latest" just means as this forms actions are played through
if action is not None:
owner_id_from_action = action.updated_known_properties.get(
"owner_id")
if owner_id_from_action is not None:
owner_id_map[case_id] = owner_id_from_action
return owner_id_map.get(case_id, None)
all_updates = {}
for case in case_list:
if case.case_id not in all_updates:
_get_logger().debug('initializing update for case {}'.format(
case.case_id))
all_updates[case.case_id] = CaseUpdate(
case_id=case.case_id,
owner_ids_on_phone=self.owner_ids_on_phone)
case_update = all_updates[case.case_id]
case_update.was_live_previously = case.case_id in self.primary_case_ids
actions = case.get_actions_for_form(xform)
for action in actions:
_get_logger().debug('{}: {}'.format(case.case_id,
action.action_type))
owner_id = get_latest_owner_id(case.case_id, action)
if owner_id is not None:
case_update.final_owner_id = owner_id
if action.action_type == const.CASE_ACTION_INDEX:
for index in action.indices:
if index.referenced_id:
case_update.indices_to_add.append(
ShortIndex(case.case_id, index.identifier,
index.referenced_id,
index.relationship))
else:
case_update.indices_to_delete.append(
ShortIndex(case.case_id, index.identifier,
None, None))
elif action.action_type == const.CASE_ACTION_CLOSE:
case_update.is_closed = True
non_live_updates = []
for case in case_list:
case_update = all_updates[case.case_id]
if case_update.is_live:
_get_logger().debug('case {} is live.'.format(
case_update.case_id))
if case.case_id not in self.case_ids_on_phone:
self._add_primary_case(case.case_id)
made_changes = True
elif case.case_id in self.dependent_case_ids_on_phone:
self.dependent_case_ids_on_phone.remove(case.case_id)
made_changes = True
for index in case_update.indices_to_add:
self._add_index(index, case_update)
made_changes = True
for index in case_update.indices_to_delete:
self._delete_index(index)
made_changes = True
class Group(QuickCachedDocumentMixin, UndoableDocument):
"""
The main use case for these 'groups' of users is currently
so that we can break down reports by arbitrary regions.
(Things like who sees what reports are determined by permissions.)
"""
domain = StringProperty()
name = StringProperty()
# a list of user ids for users
users = ListProperty()
# a list of user ids that have been removed from the Group.
# This is recorded so that we can update the user at a later point
removed_users = SetProperty()
path = ListProperty()
case_sharing = BooleanProperty()
reporting = BooleanProperty(default=True)
last_modified = DateTimeProperty()
# custom data can live here
metadata = DictProperty()
@classmethod
def wrap(cls, data):
last_modified = data.get('last_modified')
# if it's missing a Z because of the Aug. 2014 migration
# that added this in iso_format() without Z, then add a Z
if last_modified and dt_no_Z_re.match(last_modified):
data['last_modified'] += 'Z'
return super(Group, cls).wrap(data)
def save(self, *args, **kwargs):
self.last_modified = datetime.utcnow()
super(Group, self).save(*args, **kwargs)
refresh_group_views()
@classmethod
def save_docs(cls, docs, use_uuids=True):
utcnow = datetime.utcnow()
for doc in docs:
doc['last_modified'] = utcnow
super(Group, cls).save_docs(docs, use_uuids)
refresh_group_views()
bulk_save = save_docs
def delete(self):
super(Group, self).delete()
refresh_group_views()
@classmethod
def delete_docs(cls, docs, **params):
super(Group, cls).delete_docs(docs, **params)
refresh_group_views()
bulk_delete = delete_docs
def clear_caches(self):
super(Group, self).clear_caches()
self.by_domain.clear(self.__class__, self.domain)
self.ids_by_domain.clear(self.__class__, self.domain)
def add_user(self, couch_user_id, save=True):
if not isinstance(couch_user_id, str):
couch_user_id = couch_user_id.user_id
if couch_user_id not in self.users:
self.users.append(couch_user_id)
if couch_user_id in self.removed_users:
self.removed_users.remove(couch_user_id)
if save:
self.save()
def remove_user(self, couch_user_id):
'''
Returns True if it removed a user, False otherwise
'''
if not isinstance(couch_user_id, str):
couch_user_id = couch_user_id.user_id
if couch_user_id in self.users:
for i in range(0, len(self.users)):
if self.users[i] == couch_user_id:
del self.users[i]
self.removed_users.add(couch_user_id)
return True
return False
def add_group(self, group):
group.add_to_group(self)
def add_to_group(self, group):
"""
food = Food(path=[food_id])
fruit = Fruit(path=[fruit_id])
If fruit.add_to_group(food._id):
then update fruit.path to be [food_id, fruit_id]
"""
group_id = group._id
if group_id in self.path:
raise Exception("Group %s is already a member of %s" % (
self.get_id,
group_id,
))
new_path = [group_id]
new_path.extend(self.path)
self.path = new_path
self.save()
def remove_group(self, group):
group.remove_from_group(self)
def remove_from_group(self, group):
"""
food = Food(path=[food_id])
fruit = Fruit(path=[food_id, fruit_id])
If fruit.remove_from_group(food._id):
then update fruit.path to be [fruit_id]
"""
group_id = group._id
if group_id not in self.path:
raise Exception("Group %s is not a member of %s" % (
self.get_id,
group_id
))
index = 0
for i in range(0, len(self.path)):
if self.path[i] == group_id:
index = i
break
self.path = self.path[index:]
self.save()
def get_user_ids(self, is_active=True):
return [user.user_id for user in self.get_users(is_active=is_active)]
@memoized
def get_users(self, is_active=True, only_commcare=False):
def is_relevant_user(user):
if user.is_deleted():
return False
if only_commcare and user.__class__ != CommCareUser().__class__:
return False
if is_active and not user.is_active:
return False
return True
users = map(CouchUser.wrap_correctly, iter_docs(self.get_db(), self.users))
return list(filter(is_relevant_user, users))
@memoized
def get_static_user_ids(self, is_active=True):
return [user.user_id for user in self.get_static_users(is_active)]
@classmethod
def get_static_user_ids_for_groups(cls, group_ids):
static_user_ids = []
for group_id in group_ids:
group = cls.get(group_id)
static_user_ids.append(group.get_static_user_ids())
return static_user_ids
@memoized
def get_static_users(self, is_active=True):
return self.get_users(is_active)
@classmethod
@quickcache(['cls.__name__', 'domain'])
def by_domain(cls, domain):
return group_by_domain(domain)
@classmethod
def choices_by_domain(cls, domain):
group_ids = cls.ids_by_domain(domain)
group_choices = []
for group_doc in iter_docs(cls.get_db(), group_ids):
group_choices.append((group_doc['_id'], group_doc['name']))
return group_choices
@classmethod
@quickcache(['cls.__name__', 'domain'])
def ids_by_domain(cls, domain):
return get_group_ids_by_domain(domain)
@classmethod
def by_name(cls, domain, name, one=True):
result = stale_group_by_name(domain, name)
if one and result:
return result[0]
else:
return result
@classmethod
def by_user_id(cls, user_id, wrap=True):
results = cls.view('groups/by_user', key=user_id, include_docs=wrap)
if wrap:
return results
else:
return [r['id'] for r in results]
@classmethod
def get_case_sharing_accessible_locations(cls, domain, user):
return [
location.case_sharing_group_object() for location in
SQLLocation.objects.accessible_to_user(domain, user).filter(location_type__shares_cases=True)
]
@classmethod
def get_case_sharing_groups(cls, domain, wrap=True):
all_groups = cls.by_domain(domain)
if wrap:
groups = [group for group in all_groups if group.case_sharing]
groups.extend([
location.case_sharing_group_object() for location in
SQLLocation.objects.filter(domain=domain,
location_type__shares_cases=True)
])
return groups
else:
return [group._id for group in all_groups if group.case_sharing]
@classmethod
def get_reporting_groups(cls, domain):
key = ['^Reporting', domain]
return cls.view(
'groups/by_name',
startkey=key,
endkey=key + [{}],
include_docs=True,
stale=settings.COUCH_STALE_QUERY,
).all()
def create_delete_record(self, *args, **kwargs):
return DeleteGroupRecord(*args, **kwargs)
@property
def display_name(self):
if self.name:
return self.name
else:
return "[No Name]"
@classmethod
def user_in_group(cls, user_id, group_id):
if not user_id or not group_id:
return False
c = cls.get_db().view(
'groups/by_user',
key=user_id,
startkey_docid=group_id,
endkey_docid=group_id
).count()
if c == 0:
return False
elif c == 1:
return True
else:
raise Exception(
"This should just logically not be possible unless the group "
"has the user in there twice"
)
def is_member_of(self, domain):
return self.domain == domain
@property
def is_deleted(self):
return self.doc_type.endswith(DELETED_SUFFIX)
def __repr__(self):
return ("Group(domain={self.domain!r}, name={self.name!r}, "
"case_sharing={self.case_sharing!r})").format(self=self)