def hard_rebuild_case(domain, case_id, detail, lock=True):
if lock:
# only record metric if locking since otherwise it has been
# (most likley) recorded elsewhere
case_load_counter("rebuild_case", domain)()
case, lock_obj = FormProcessorSQL.get_case_with_lock(case_id, lock=lock)
found = bool(case)
if not found:
case = CommCareCaseSQL(case_id=case_id, domain=domain)
if lock:
lock_obj = CommCareCaseSQL.get_obj_lock_by_id(case_id)
acquire_lock(lock_obj, degrade_gracefully=False)
try:
assert case.domain == domain, (case.domain, domain)
case, rebuild_transaction = FormProcessorSQL._rebuild_case_from_transactions(case, detail)
if case.is_deleted and not case.is_saved():
return None
case.server_modified_on = rebuild_transaction.server_date
CaseAccessorSQL.save_case(case)
publish_case_saved(case)
return case
finally:
release_lock(lock_obj, degrade_gracefully=True)
def _sync_case_for_messaging_rule(domain, case_id, rule_id):
case_load_counter("messaging_rule_sync", domain)()
case = CaseAccessors(domain).get_case(case_id)
rule = _get_cached_rule(domain, rule_id)
if rule:
rule.run_rule(case, utcnow())
MessagingRuleProgressHelper(rule_id).increment_current_case_count()
def hard_rebuild_case(domain, case_id, detail, lock=True):
if lock:
# only record metric if locking since otherwise it has been
# (most likley) recorded elsewhere
case_load_counter("rebuild_case", domain)()
case, lock_obj = FormProcessorSQL.get_case_with_lock(case_id, lock=lock)
found = bool(case)
if not found:
case = CommCareCaseSQL(case_id=case_id, domain=domain)
if lock:
lock_obj = CommCareCaseSQL.get_obj_lock_by_id(case_id)
acquire_lock(lock_obj, degrade_gracefully=False)
def __init__(self,
domain=None,
deleted_ok=False,
lock=False,
wrap=True,
initial=None,
xforms=None,
load_src="unknown"):
self._track_load = case_load_counter(load_src, domain)
self._populate_from_initial(initial)
self.domain = domain
self.cached_xforms = xforms if xforms is not None else []
self.deleted_ok = deleted_ok
self.lock = lock
self.wrap = wrap
if self.lock and not self.wrap:
raise ValueError(
'Currently locking only supports explicitly wrapping cases!')
self.locks = []
self._changed = set()
# this is used to allow casedb to be re-entrant. Each new context pushes the parent context locks
# onto this stack and restores them when the context exits
self.lock_stack = []
self.processor_interface = FormProcessorInterface(self.domain)
def recalculate_stagnant_cases():
domain = 'icds-cas'
config_ids = [
'static-icds-cas-static-ccs_record_cases_monthly_v2',
'static-icds-cas-static-child_cases_monthly_v2',
]
track_case_load = case_load_counter("find_stagnant_cases", domain)
stagnant_cases = set()
for config_id in config_ids:
config, is_static = get_datasource_config(config_id, domain)
adapter = get_indicator_adapter(config,
load_source='find_stagnant_cases')
case_ids = _find_stagnant_cases(adapter)
num_cases = len(case_ids)
adapter.track_load(num_cases)
celery_task_logger.info("Found {} stagnant cases in config {}".format(
num_cases, config_id))
stagnant_cases = stagnant_cases.union(set(case_ids))
celery_task_logger.info(
"Total number of stagant cases is now {}".format(
len(stagnant_cases)))
case_accessor = CaseAccessors(domain)
num_stagnant_cases = len(stagnant_cases)
current_case_num = 0
for case_ids in chunked(stagnant_cases, 1000):
current_case_num += len(case_ids)
cases = case_accessor.get_cases(list(case_ids))
for case in cases:
track_case_load()
publish_case_saved(case, send_post_save_signal=False)
celery_task_logger.info("Resaved {} / {} cases".format(
current_case_num, num_stagnant_cases))
def hard_rebuild_case(domain, case_id, detail, save=True, lock=True):
if lock:
# only record metric if locking since otherwise it has been
# (most likley) recorded elsewhere
case_load_counter("rebuild_case", domain)()
case, lock_obj = FormProcessorCouch.get_case_with_lock(case_id, lock=lock, wrap=True)
found = bool(case)
if not found:
case = CommCareCase()
case.case_id = case_id
case.domain = domain
if lock:
lock_obj = CommCareCase.get_obj_lock_by_id(case_id)
acquire_lock(lock_obj, degrade_gracefully=False)
try:
assert case.domain == domain, (case.domain, domain)
forms = FormProcessorCouch.get_case_forms(case_id)
form_load_counter("rebuild_case", domain)(len(forms))
filtered_forms = [f for f in forms if f.is_normal]
sorted_forms = sorted(filtered_forms, key=lambda f: f.received_on)
actions = _get_actions_from_forms(domain, sorted_forms, case_id)
if not found and case.domain is None:
case.domain = domain
rebuild_case_from_actions(case, actions)
# todo: should this move to case.rebuild?
if not case.xform_ids:
if not found:
return None
# there were no more forms. 'delete' the case
case.doc_type = 'CommCareCase-Deleted'
# add a "rebuild" action
case.actions.append(_rebuild_action())
if save:
case.save()
return case
finally:
release_lock(lock_obj, degrade_gracefully=True)
def _sync_case_for_messaging(domain, case_id):
try:
case = CaseAccessors(domain).get_case(case_id)
sms_tasks.clear_case_caches(case)
except CaseNotFound:
case = None
case_load_counter("messaging_sync", domain)()
if case is None or case.is_deleted:
sms_tasks.delete_phone_numbers_for_owners([case_id])
delete_schedule_instances_for_cases(domain, [case_id])
return
if use_phone_entries():
sms_tasks._sync_case_phone_number(case)
rules = AutomaticUpdateRule.by_domain_cached(case.domain, AutomaticUpdateRule.WORKFLOW_SCHEDULING)
rules_by_case_type = AutomaticUpdateRule.organize_rules_by_case_type(rules)
for rule in rules_by_case_type.get(case.type, []):
rule.run_rule(case, utcnow())
def batch_cases(accessor, case_ids):
def take(n, iterable):
# https://docs.python.org/2/library/itertools.html#recipes
return list(islice(iterable, n))
track_load = case_load_counter("livequery_restore", accessor.domain)
ids = iter(case_ids)
while True:
next_ids = take(1000, ids)
if not next_ids:
break
track_load(len(next_ids))
yield accessor.get_cases(next_ids)
def batch_cases(accessor, case_ids):
def take(n, iterable):
# https://docs.python.org/2/library/itertools.html#recipes
return list(islice(iterable, n))
track_load = case_load_counter("livequery_restore", accessor.domain)
ids = iter(case_ids)
while True:
next_ids = take(1000, ids)
if not next_ids:
break
track_load(len(next_ids))
yield accessor.get_cases(next_ids)
def __init__(self, timing_context, case_ids_to_sync, restore_state):
self.restore_state = restore_state
self.case_accessor = CaseAccessors(self.restore_state.domain)
self.case_ids_to_sync = case_ids_to_sync
self.all_maybe_syncing = copy(case_ids_to_sync)
self.checked_cases = set()
self.child_indices = defaultdict(set)
self.extension_indices = defaultdict(set)
self.all_dependencies_syncing = set()
self.closed_cases = set()
self.potential_elements_to_sync = {}
self.timing_context = timing_context
self._track_load = case_load_counter("cleanowner_restore", restore_state.domain)
def __init__(self, timing_context, case_ids_to_sync, restore_state):
self.restore_state = restore_state
self.case_accessor = CaseAccessors(self.restore_state.domain)
self.case_ids_to_sync = case_ids_to_sync
self.all_maybe_syncing = copy(case_ids_to_sync)
self.checked_cases = set()
self.child_indices = defaultdict(set)
self.extension_indices = defaultdict(set)
self.all_dependencies_syncing = set()
self.closed_cases = set()
self.potential_elements_to_sync = {}
self.timing_context = timing_context
self._track_load = case_load_counter("cleanowner_restore",
restore_state.domain)
def __init__(self, domain=None, deleted_ok=False,
lock=False, wrap=True, initial=None, xforms=None,
load_src="unknown"):
self._track_load = case_load_counter(load_src, domain)
self._populate_from_initial(initial)
self.domain = domain
self.cached_xforms = xforms if xforms is not None else []
self.deleted_ok = deleted_ok
self.lock = lock
self.wrap = wrap
if self.lock and not self.wrap:
raise ValueError('Currently locking only supports explicitly wrapping cases!')
self.locks = []
self._changed = set()
# this is used to allow casedb to be re-entrant. Each new context pushes the parent context locks
# onto this stack and restores them when the context exits
self.lock_stack = []
self.processor_interface = FormProcessorInterface(self.domain)
default_retry_delay=5 * 60, max_retries=12, bind=True)
def sync_case_for_messaging_rule(self, domain, case_id, rule_id):
try:
with CriticalSection([get_sync_key(case_id)], timeout=5 * 60):
_sync_case_for_messaging_rule(domain, case_id, rule_id)
except Exception as e:
self.retry(exc=e)
def _sync_case_for_messaging(domain, case_id):
try:
case = CaseAccessors(domain).get_case(case_id)
sms_tasks.clear_case_caches(case)
except CaseNotFound:
case = None
case_load_counter("messaging_sync", domain)()
if case is None or case.is_deleted:
sms_tasks.delete_phone_numbers_for_owners([case_id])
delete_schedule_instances_for_cases(domain, [case_id])
return
if use_phone_entries():
sms_tasks._sync_case_phone_number(case)
rules = AutomaticUpdateRule.by_domain_cached(case.domain, AutomaticUpdateRule.WORKFLOW_SCHEDULING)
rules_by_case_type = AutomaticUpdateRule.organize_rules_by_case_type(rules)
for rule in rules_by_case_type.get(case.type, []):
rule.run_rule(case, utcnow())
def _log_case_lookup(domain):
case_load_counter("case_importer", domain)
def do_import(spreadsheet, config, domain, task=None, chunksize=CASEBLOCK_CHUNKSIZE,
record_form_callback=None):
match_count = created_count = too_many_matches = num_chunks = 0
errors = importer_util.ImportErrorDetail()
user = CouchUser.get_by_user_id(config.couch_user_id, domain)
username = user.username
user_id = user._id
# keep a cache of id lookup successes to help performance
id_cache = {}
name_cache = {}
caseblocks = []
ids_seen = set()
track_load = case_load_counter("case_importer", domain)
def _submit_caseblocks(domain, case_type, caseblocks):
err = False
if caseblocks:
try:
form, cases = submit_case_blocks(
[cb.case.as_string().decode('utf-8') for cb in caseblocks],
domain,
username,
user_id,
device_id=__name__ + ".do_import",
)
if form.is_error:
errors.add(
error=ImportErrors.ImportErrorMessage,
row_number=form.problem
)
except Exception:
err = True
for row_number, case in caseblocks:
errors.add(
error=ImportErrors.ImportErrorMessage,
row_number=row_number
)
else:
if record_form_callback:
record_form_callback(form.form_id)
properties = set().union(*[set(c.dynamic_case_properties().keys()) for c in cases])
if case_type and len(properties):
add_inferred_export_properties.delay(
'CaseImporter',
domain,
case_type,
properties,
)
else:
_soft_assert = soft_assert(notify_admins=True)
_soft_assert(
len(properties) == 0,
'error adding inferred export properties in domain '
'({}): {}'.format(domain, ", ".join(properties))
)
return err
row_count = spreadsheet.max_row
for i, row in enumerate(spreadsheet.iter_row_dicts()):
if task:
set_task_progress(task, i, row_count)
# skip first row (header row)
if i == 0:
continue
search_id = importer_util.parse_search_id(config, row)
fields_to_update = importer_util.populate_updated_fields(config, row)
if not any(fields_to_update.values()):
# if the row was blank, just skip it, no errors
continue
if config.search_field == 'external_id' and not search_id:
# do not allow blank external id since we save this
errors.add(ImportErrors.BlankExternalId, i + 1)
continue
external_id = fields_to_update.pop('external_id', None)
parent_id = fields_to_update.pop('parent_id', None)
parent_external_id = fields_to_update.pop('parent_external_id', None)
parent_type = fields_to_update.pop('parent_type', config.case_type)
parent_ref = fields_to_update.pop('parent_ref', 'parent')
to_close = fields_to_update.pop('close', False)
if any([lookup_id and lookup_id in ids_seen for lookup_id in [search_id, parent_id, parent_external_id]]):
# clear out the queue to make sure we've processed any potential
# cases we want to look up
# note: these three lines are repeated a few places, and could be converted
# to a function that makes use of closures (and globals) to do the same thing,
# but that seems sketchier than just beeing a little RY
_submit_caseblocks(domain, config.case_type, caseblocks)
num_chunks += 1
caseblocks = []
ids_seen = set() # also clear ids_seen, since all the cases will now be in the database
case, error = importer_util.lookup_case(
config.search_field,
search_id,
domain,
config.case_type
)
track_load()
if case:
if case.type != config.case_type:
continue
elif error == LookupErrors.NotFound:
if not config.create_new_cases:
continue
elif error == LookupErrors.MultipleResults:
too_many_matches += 1
continue
uploaded_owner_name = fields_to_update.pop('owner_name', None)
uploaded_owner_id = fields_to_update.pop('owner_id', None)
if uploaded_owner_name:
# If an owner name was provided, replace the provided
# uploaded_owner_id with the id of the provided group or owner
try:
uploaded_owner_id = importer_util.get_id_from_name(uploaded_owner_name, domain, name_cache)
except SQLLocation.MultipleObjectsReturned:
errors.add(ImportErrors.DuplicateLocationName, i + 1)
continue
if not uploaded_owner_id:
errors.add(ImportErrors.InvalidOwnerName, i + 1, 'owner_name')
continue
if uploaded_owner_id:
# If an owner_id mapping exists, verify it is a valid user
# or case sharing group
if importer_util.is_valid_id(uploaded_owner_id, domain, id_cache):
owner_id = uploaded_owner_id
id_cache[uploaded_owner_id] = True
else:
errors.add(ImportErrors.InvalidOwnerId, i + 1, 'owner_id')
id_cache[uploaded_owner_id] = False
continue
else:
# if they didn't supply an owner_id mapping, default to current
# user
owner_id = user_id
extras = {}
if parent_id:
try:
parent_case = CaseAccessors(domain).get_case(parent_id)
track_load()
if parent_case.domain == domain:
extras['index'] = {
parent_ref: (parent_case.type, parent_id)
}
except ResourceNotFound:
errors.add(ImportErrors.InvalidParentId, i + 1, 'parent_id')
continue
elif parent_external_id:
parent_case, error = importer_util.lookup_case(
'external_id',
parent_external_id,
domain,
parent_type
)
track_load()
if parent_case:
extras['index'] = {
parent_ref: (parent_type, parent_case.case_id)
}
case_name = fields_to_update.pop('name', None)
if BULK_UPLOAD_DATE_OPENED.enabled(domain):
date_opened = fields_to_update.pop(CASE_TAG_DATE_OPENED, None)
if date_opened:
extras['date_opened'] = date_opened
if not case:
id = uuid.uuid4().hex
if config.search_field == 'external_id':
extras['external_id'] = search_id
elif external_id:
extras['external_id'] = external_id
try:
caseblock = CaseBlock(
create=True,
case_id=id,
owner_id=owner_id,
user_id=user_id,
case_type=config.case_type,
case_name=case_name or '',
update=fields_to_update,
**extras
)
caseblocks.append(RowAndCase(i, caseblock))
created_count += 1
if external_id:
ids_seen.add(external_id)
except CaseBlockError:
errors.add(ImportErrors.CaseGeneration, i + 1)
else:
if external_id:
extras['external_id'] = external_id
if uploaded_owner_id:
extras['owner_id'] = owner_id
if to_close == 'yes':
extras['close'] = True
if case_name is not None:
extras['case_name'] = case_name
try:
caseblock = CaseBlock(
create=False,
case_id=case.case_id,
update=fields_to_update,
**extras
)
caseblocks.append(RowAndCase(i, caseblock))
match_count += 1
except CaseBlockError:
errors.add(ImportErrors.CaseGeneration, i + 1)
# check if we've reached a reasonable chunksize
# and if so submit
if len(caseblocks) >= chunksize:
_submit_caseblocks(domain, config.case_type, caseblocks)
num_chunks += 1
caseblocks = []
# final purge of anything left in the queue
if _submit_caseblocks(domain, config.case_type, caseblocks):
match_count -= 1
num_chunks += 1
return {
'created_count': created_count,
'match_count': match_count,
'too_many_matches': too_many_matches,
'errors': errors.as_dict(),
'num_chunks': num_chunks,
}
def do_import(spreadsheet,
config,
domain,
task=None,
chunksize=CASEBLOCK_CHUNKSIZE,
record_form_callback=None):
match_count = created_count = too_many_matches = num_chunks = 0
errors = importer_util.ImportErrorDetail()
user = CouchUser.get_by_user_id(config.couch_user_id, domain)
username = user.username
user_id = user._id
# keep a cache of id lookup successes to help performance
id_cache = {}
name_cache = {}
caseblocks = []
ids_seen = set()
track_load = case_load_counter("case_importer", domain)
def _submit_caseblocks(domain, case_type, caseblocks):
err = False
if caseblocks:
try:
form, cases = submit_case_blocks(
[cb.case.as_string().decode('utf-8') for cb in caseblocks],
domain,
username,
user_id,
device_id=__name__ + ".do_import",
)
if form.is_error:
errors.add(error=ImportErrors.ImportErrorMessage,
row_number=form.problem)
except Exception:
err = True
for row_number, case in caseblocks:
errors.add(error=ImportErrors.ImportErrorMessage,
row_number=row_number)
else:
if record_form_callback:
record_form_callback(form.form_id)
properties = set().union(
*[set(c.dynamic_case_properties().keys()) for c in cases])
if case_type and len(properties):
add_inferred_export_properties.delay(
'CaseImporter',
domain,
case_type,
properties,
)
else:
_soft_assert = soft_assert(notify_admins=True)
_soft_assert(
len(properties) == 0,
'error adding inferred export properties in domain '
'({}): {}'.format(domain, ", ".join(properties)))
return err
row_count = spreadsheet.max_row
for i, row in enumerate(spreadsheet.iter_row_dicts()):
if task:
set_task_progress(task, i, row_count)
# skip first row (header row)
if i == 0:
continue
search_id = importer_util.parse_search_id(config, row)
fields_to_update = importer_util.populate_updated_fields(config, row)
if not any(fields_to_update.values()):
# if the row was blank, just skip it, no errors
continue
if config.search_field == 'external_id' and not search_id:
# do not allow blank external id since we save this
errors.add(ImportErrors.BlankExternalId, i + 1)
continue
external_id = fields_to_update.pop('external_id', None)
parent_id = fields_to_update.pop('parent_id', None)
parent_external_id = fields_to_update.pop('parent_external_id', None)
parent_type = fields_to_update.pop('parent_type', config.case_type)
parent_ref = fields_to_update.pop('parent_ref', 'parent')
to_close = fields_to_update.pop('close', False)
if any([
lookup_id and lookup_id in ids_seen
for lookup_id in [search_id, parent_id, parent_external_id]
]):
# clear out the queue to make sure we've processed any potential
# cases we want to look up
# note: these three lines are repeated a few places, and could be converted
# to a function that makes use of closures (and globals) to do the same thing,
# but that seems sketchier than just beeing a little RY
_submit_caseblocks(domain, config.case_type, caseblocks)
num_chunks += 1
caseblocks = []
ids_seen = set(
) # also clear ids_seen, since all the cases will now be in the database
case, error = importer_util.lookup_case(config.search_field, search_id,
domain, config.case_type)
track_load()
if case:
if case.type != config.case_type:
continue
elif error == LookupErrors.NotFound:
if not config.create_new_cases:
continue
