def hard_rebuild_case(domain, case_id, detail, lock=True): if lock: # only record metric if locking since otherwise it has been # (most likley) recorded elsewhere case_load_counter("rebuild_case", domain)() case, lock_obj = FormProcessorSQL.get_case_with_lock(case_id, lock=lock) found = bool(case) if not found: case = CommCareCaseSQL(case_id=case_id, domain=domain) if lock: lock_obj = CommCareCaseSQL.get_obj_lock_by_id(case_id) acquire_lock(lock_obj, degrade_gracefully=False) try: assert case.domain == domain, (case.domain, domain) case, rebuild_transaction = FormProcessorSQL._rebuild_case_from_transactions(case, detail) if case.is_deleted and not case.is_saved(): return None case.server_modified_on = rebuild_transaction.server_date CaseAccessorSQL.save_case(case) publish_case_saved(case) return case finally: release_lock(lock_obj, degrade_gracefully=True)
def _sync_case_for_messaging_rule(domain, case_id, rule_id): case_load_counter("messaging_rule_sync", domain)() case = CaseAccessors(domain).get_case(case_id) rule = _get_cached_rule(domain, rule_id) if rule: rule.run_rule(case, utcnow()) MessagingRuleProgressHelper(rule_id).increment_current_case_count()
def hard_rebuild_case(domain, case_id, detail, lock=True): if lock: # only record metric if locking since otherwise it has been # (most likley) recorded elsewhere case_load_counter("rebuild_case", domain)() case, lock_obj = FormProcessorSQL.get_case_with_lock(case_id, lock=lock) found = bool(case) if not found: case = CommCareCaseSQL(case_id=case_id, domain=domain) if lock: lock_obj = CommCareCaseSQL.get_obj_lock_by_id(case_id) acquire_lock(lock_obj, degrade_gracefully=False)
def __init__(self, domain=None, deleted_ok=False, lock=False, wrap=True, initial=None, xforms=None, load_src="unknown"): self._track_load = case_load_counter(load_src, domain) self._populate_from_initial(initial) self.domain = domain self.cached_xforms = xforms if xforms is not None else [] self.deleted_ok = deleted_ok self.lock = lock self.wrap = wrap if self.lock and not self.wrap: raise ValueError( 'Currently locking only supports explicitly wrapping cases!') self.locks = [] self._changed = set() # this is used to allow casedb to be re-entrant. Each new context pushes the parent context locks # onto this stack and restores them when the context exits self.lock_stack = [] self.processor_interface = FormProcessorInterface(self.domain)
def recalculate_stagnant_cases(): domain = 'icds-cas' config_ids = [ 'static-icds-cas-static-ccs_record_cases_monthly_v2', 'static-icds-cas-static-child_cases_monthly_v2', ] track_case_load = case_load_counter("find_stagnant_cases", domain) stagnant_cases = set() for config_id in config_ids: config, is_static = get_datasource_config(config_id, domain) adapter = get_indicator_adapter(config, load_source='find_stagnant_cases') case_ids = _find_stagnant_cases(adapter) num_cases = len(case_ids) adapter.track_load(num_cases) celery_task_logger.info("Found {} stagnant cases in config {}".format( num_cases, config_id)) stagnant_cases = stagnant_cases.union(set(case_ids)) celery_task_logger.info( "Total number of stagant cases is now {}".format( len(stagnant_cases))) case_accessor = CaseAccessors(domain) num_stagnant_cases = len(stagnant_cases) current_case_num = 0 for case_ids in chunked(stagnant_cases, 1000): current_case_num += len(case_ids) cases = case_accessor.get_cases(list(case_ids)) for case in cases: track_case_load() publish_case_saved(case, send_post_save_signal=False) celery_task_logger.info("Resaved {} / {} cases".format( current_case_num, num_stagnant_cases))
def hard_rebuild_case(domain, case_id, detail, save=True, lock=True): if lock: # only record metric if locking since otherwise it has been # (most likley) recorded elsewhere case_load_counter("rebuild_case", domain)() case, lock_obj = FormProcessorCouch.get_case_with_lock(case_id, lock=lock, wrap=True) found = bool(case) if not found: case = CommCareCase() case.case_id = case_id case.domain = domain if lock: lock_obj = CommCareCase.get_obj_lock_by_id(case_id) acquire_lock(lock_obj, degrade_gracefully=False) try: assert case.domain == domain, (case.domain, domain) forms = FormProcessorCouch.get_case_forms(case_id) form_load_counter("rebuild_case", domain)(len(forms)) filtered_forms = [f for f in forms if f.is_normal] sorted_forms = sorted(filtered_forms, key=lambda f: f.received_on) actions = _get_actions_from_forms(domain, sorted_forms, case_id) if not found and case.domain is None: case.domain = domain rebuild_case_from_actions(case, actions) # todo: should this move to case.rebuild? if not case.xform_ids: if not found: return None # there were no more forms. 'delete' the case case.doc_type = 'CommCareCase-Deleted' # add a "rebuild" action case.actions.append(_rebuild_action()) if save: case.save() return case finally: release_lock(lock_obj, degrade_gracefully=True)
def _sync_case_for_messaging(domain, case_id): try: case = CaseAccessors(domain).get_case(case_id) sms_tasks.clear_case_caches(case) except CaseNotFound: case = None case_load_counter("messaging_sync", domain)() if case is None or case.is_deleted: sms_tasks.delete_phone_numbers_for_owners([case_id]) delete_schedule_instances_for_cases(domain, [case_id]) return if use_phone_entries(): sms_tasks._sync_case_phone_number(case) rules = AutomaticUpdateRule.by_domain_cached(case.domain, AutomaticUpdateRule.WORKFLOW_SCHEDULING) rules_by_case_type = AutomaticUpdateRule.organize_rules_by_case_type(rules) for rule in rules_by_case_type.get(case.type, []): rule.run_rule(case, utcnow())
def batch_cases(accessor, case_ids): def take(n, iterable): # https://docs.python.org/2/library/itertools.html#recipes return list(islice(iterable, n)) track_load = case_load_counter("livequery_restore", accessor.domain) ids = iter(case_ids) while True: next_ids = take(1000, ids) if not next_ids: break track_load(len(next_ids)) yield accessor.get_cases(next_ids)
def batch_cases(accessor, case_ids): def take(n, iterable): # https://docs.python.org/2/library/itertools.html#recipes return list(islice(iterable, n)) track_load = case_load_counter("livequery_restore", accessor.domain) ids = iter(case_ids) while True: next_ids = take(1000, ids) if not next_ids: break track_load(len(next_ids)) yield accessor.get_cases(next_ids)
def __init__(self, timing_context, case_ids_to_sync, restore_state): self.restore_state = restore_state self.case_accessor = CaseAccessors(self.restore_state.domain) self.case_ids_to_sync = case_ids_to_sync self.all_maybe_syncing = copy(case_ids_to_sync) self.checked_cases = set() self.child_indices = defaultdict(set) self.extension_indices = defaultdict(set) self.all_dependencies_syncing = set() self.closed_cases = set() self.potential_elements_to_sync = {} self.timing_context = timing_context self._track_load = case_load_counter("cleanowner_restore", restore_state.domain)
def __init__(self, timing_context, case_ids_to_sync, restore_state): self.restore_state = restore_state self.case_accessor = CaseAccessors(self.restore_state.domain) self.case_ids_to_sync = case_ids_to_sync self.all_maybe_syncing = copy(case_ids_to_sync) self.checked_cases = set() self.child_indices = defaultdict(set) self.extension_indices = defaultdict(set) self.all_dependencies_syncing = set() self.closed_cases = set() self.potential_elements_to_sync = {} self.timing_context = timing_context self._track_load = case_load_counter("cleanowner_restore", restore_state.domain)
def __init__(self, domain=None, deleted_ok=False, lock=False, wrap=True, initial=None, xforms=None, load_src="unknown"): self._track_load = case_load_counter(load_src, domain) self._populate_from_initial(initial) self.domain = domain self.cached_xforms = xforms if xforms is not None else [] self.deleted_ok = deleted_ok self.lock = lock self.wrap = wrap if self.lock and not self.wrap: raise ValueError('Currently locking only supports explicitly wrapping cases!') self.locks = [] self._changed = set() # this is used to allow casedb to be re-entrant. Each new context pushes the parent context locks # onto this stack and restores them when the context exits self.lock_stack = [] self.processor_interface = FormProcessorInterface(self.domain)
default_retry_delay=5 * 60, max_retries=12, bind=True) def sync_case_for_messaging_rule(self, domain, case_id, rule_id): try: with CriticalSection([get_sync_key(case_id)], timeout=5 * 60): _sync_case_for_messaging_rule(domain, case_id, rule_id) except Exception as e: self.retry(exc=e) def _sync_case_for_messaging(domain, case_id): try: case = CaseAccessors(domain).get_case(case_id) sms_tasks.clear_case_caches(case) except CaseNotFound: case = None case_load_counter("messaging_sync", domain)() if case is None or case.is_deleted: sms_tasks.delete_phone_numbers_for_owners([case_id]) delete_schedule_instances_for_cases(domain, [case_id]) return if use_phone_entries(): sms_tasks._sync_case_phone_number(case) rules = AutomaticUpdateRule.by_domain_cached(case.domain, AutomaticUpdateRule.WORKFLOW_SCHEDULING) rules_by_case_type = AutomaticUpdateRule.organize_rules_by_case_type(rules) for rule in rules_by_case_type.get(case.type, []): rule.run_rule(case, utcnow())
def _log_case_lookup(domain): case_load_counter("case_importer", domain)
def do_import(spreadsheet, config, domain, task=None, chunksize=CASEBLOCK_CHUNKSIZE, record_form_callback=None): match_count = created_count = too_many_matches = num_chunks = 0 errors = importer_util.ImportErrorDetail() user = CouchUser.get_by_user_id(config.couch_user_id, domain) username = user.username user_id = user._id # keep a cache of id lookup successes to help performance id_cache = {} name_cache = {} caseblocks = [] ids_seen = set() track_load = case_load_counter("case_importer", domain) def _submit_caseblocks(domain, case_type, caseblocks): err = False if caseblocks: try: form, cases = submit_case_blocks( [cb.case.as_string().decode('utf-8') for cb in caseblocks], domain, username, user_id, device_id=__name__ + ".do_import", ) if form.is_error: errors.add( error=ImportErrors.ImportErrorMessage, row_number=form.problem ) except Exception: err = True for row_number, case in caseblocks: errors.add( error=ImportErrors.ImportErrorMessage, row_number=row_number ) else: if record_form_callback: record_form_callback(form.form_id) properties = set().union(*[set(c.dynamic_case_properties().keys()) for c in cases]) if case_type and len(properties): add_inferred_export_properties.delay( 'CaseImporter', domain, case_type, properties, ) else: _soft_assert = soft_assert(notify_admins=True) _soft_assert( len(properties) == 0, 'error adding inferred export properties in domain ' '({}): {}'.format(domain, ", ".join(properties)) ) return err row_count = spreadsheet.max_row for i, row in enumerate(spreadsheet.iter_row_dicts()): if task: set_task_progress(task, i, row_count) # skip first row (header row) if i == 0: continue search_id = importer_util.parse_search_id(config, row) fields_to_update = importer_util.populate_updated_fields(config, row) if not any(fields_to_update.values()): # if the row was blank, just skip it, no errors continue if config.search_field == 'external_id' and not search_id: # do not allow blank external id since we save this errors.add(ImportErrors.BlankExternalId, i + 1) continue external_id = fields_to_update.pop('external_id', None) parent_id = fields_to_update.pop('parent_id', None) parent_external_id = fields_to_update.pop('parent_external_id', None) parent_type = fields_to_update.pop('parent_type', config.case_type) parent_ref = fields_to_update.pop('parent_ref', 'parent') to_close = fields_to_update.pop('close', False) if any([lookup_id and lookup_id in ids_seen for lookup_id in [search_id, parent_id, parent_external_id]]): # clear out the queue to make sure we've processed any potential # cases we want to look up # note: these three lines are repeated a few places, and could be converted # to a function that makes use of closures (and globals) to do the same thing, # but that seems sketchier than just beeing a little RY _submit_caseblocks(domain, config.case_type, caseblocks) num_chunks += 1 caseblocks = [] ids_seen = set() # also clear ids_seen, since all the cases will now be in the database case, error = importer_util.lookup_case( config.search_field, search_id, domain, config.case_type ) track_load() if case: if case.type != config.case_type: continue elif error == LookupErrors.NotFound: if not config.create_new_cases: continue elif error == LookupErrors.MultipleResults: too_many_matches += 1 continue uploaded_owner_name = fields_to_update.pop('owner_name', None) uploaded_owner_id = fields_to_update.pop('owner_id', None) if uploaded_owner_name: # If an owner name was provided, replace the provided # uploaded_owner_id with the id of the provided group or owner try: uploaded_owner_id = importer_util.get_id_from_name(uploaded_owner_name, domain, name_cache) except SQLLocation.MultipleObjectsReturned: errors.add(ImportErrors.DuplicateLocationName, i + 1) continue if not uploaded_owner_id: errors.add(ImportErrors.InvalidOwnerName, i + 1, 'owner_name') continue if uploaded_owner_id: # If an owner_id mapping exists, verify it is a valid user # or case sharing group if importer_util.is_valid_id(uploaded_owner_id, domain, id_cache): owner_id = uploaded_owner_id id_cache[uploaded_owner_id] = True else: errors.add(ImportErrors.InvalidOwnerId, i + 1, 'owner_id') id_cache[uploaded_owner_id] = False continue else: # if they didn't supply an owner_id mapping, default to current # user owner_id = user_id extras = {} if parent_id: try: parent_case = CaseAccessors(domain).get_case(parent_id) track_load() if parent_case.domain == domain: extras['index'] = { parent_ref: (parent_case.type, parent_id) } except ResourceNotFound: errors.add(ImportErrors.InvalidParentId, i + 1, 'parent_id') continue elif parent_external_id: parent_case, error = importer_util.lookup_case( 'external_id', parent_external_id, domain, parent_type ) track_load() if parent_case: extras['index'] = { parent_ref: (parent_type, parent_case.case_id) } case_name = fields_to_update.pop('name', None) if BULK_UPLOAD_DATE_OPENED.enabled(domain): date_opened = fields_to_update.pop(CASE_TAG_DATE_OPENED, None) if date_opened: extras['date_opened'] = date_opened if not case: id = uuid.uuid4().hex if config.search_field == 'external_id': extras['external_id'] = search_id elif external_id: extras['external_id'] = external_id try: caseblock = CaseBlock( create=True, case_id=id, owner_id=owner_id, user_id=user_id, case_type=config.case_type, case_name=case_name or '', update=fields_to_update, **extras ) caseblocks.append(RowAndCase(i, caseblock)) created_count += 1 if external_id: ids_seen.add(external_id) except CaseBlockError: errors.add(ImportErrors.CaseGeneration, i + 1) else: if external_id: extras['external_id'] = external_id if uploaded_owner_id: extras['owner_id'] = owner_id if to_close == 'yes': extras['close'] = True if case_name is not None: extras['case_name'] = case_name try: caseblock = CaseBlock( create=False, case_id=case.case_id, update=fields_to_update, **extras ) caseblocks.append(RowAndCase(i, caseblock)) match_count += 1 except CaseBlockError: errors.add(ImportErrors.CaseGeneration, i + 1) # check if we've reached a reasonable chunksize # and if so submit if len(caseblocks) >= chunksize: _submit_caseblocks(domain, config.case_type, caseblocks) num_chunks += 1 caseblocks = [] # final purge of anything left in the queue if _submit_caseblocks(domain, config.case_type, caseblocks): match_count -= 1 num_chunks += 1 return { 'created_count': created_count, 'match_count': match_count, 'too_many_matches': too_many_matches, 'errors': errors.as_dict(), 'num_chunks': num_chunks, }
def do_import(spreadsheet, config, domain, task=None, chunksize=CASEBLOCK_CHUNKSIZE, record_form_callback=None): match_count = created_count = too_many_matches = num_chunks = 0 errors = importer_util.ImportErrorDetail() user = CouchUser.get_by_user_id(config.couch_user_id, domain) username = user.username user_id = user._id # keep a cache of id lookup successes to help performance id_cache = {} name_cache = {} caseblocks = [] ids_seen = set() track_load = case_load_counter("case_importer", domain) def _submit_caseblocks(domain, case_type, caseblocks): err = False if caseblocks: try: form, cases = submit_case_blocks( [cb.case.as_string().decode('utf-8') for cb in caseblocks], domain, username, user_id, device_id=__name__ + ".do_import", ) if form.is_error: errors.add(error=ImportErrors.ImportErrorMessage, row_number=form.problem) except Exception: err = True for row_number, case in caseblocks: errors.add(error=ImportErrors.ImportErrorMessage, row_number=row_number) else: if record_form_callback: record_form_callback(form.form_id) properties = set().union( *[set(c.dynamic_case_properties().keys()) for c in cases]) if case_type and len(properties): add_inferred_export_properties.delay( 'CaseImporter', domain, case_type, properties, ) else: _soft_assert = soft_assert(notify_admins=True) _soft_assert( len(properties) == 0, 'error adding inferred export properties in domain ' '({}): {}'.format(domain, ", ".join(properties))) return err row_count = spreadsheet.max_row for i, row in enumerate(spreadsheet.iter_row_dicts()): if task: set_task_progress(task, i, row_count) # skip first row (header row) if i == 0: continue search_id = importer_util.parse_search_id(config, row) fields_to_update = importer_util.populate_updated_fields(config, row) if not any(fields_to_update.values()): # if the row was blank, just skip it, no errors continue if config.search_field == 'external_id' and not search_id: # do not allow blank external id since we save this errors.add(ImportErrors.BlankExternalId, i + 1) continue external_id = fields_to_update.pop('external_id', None) parent_id = fields_to_update.pop('parent_id', None) parent_external_id = fields_to_update.pop('parent_external_id', None) parent_type = fields_to_update.pop('parent_type', config.case_type) parent_ref = fields_to_update.pop('parent_ref', 'parent') to_close = fields_to_update.pop('close', False) if any([ lookup_id and lookup_id in ids_seen for lookup_id in [search_id, parent_id, parent_external_id] ]): # clear out the queue to make sure we've processed any potential # cases we want to look up # note: these three lines are repeated a few places, and could be converted # to a function that makes use of closures (and globals) to do the same thing, # but that seems sketchier than just beeing a little RY _submit_caseblocks(domain, config.case_type, caseblocks) num_chunks += 1 caseblocks = [] ids_seen = set( ) # also clear ids_seen, since all the cases will now be in the database case, error = importer_util.lookup_case(config.search_field, search_id, domain, config.case_type) track_load() if case: if case.type != config.case_type: continue elif error == LookupErrors.NotFound: if not config.create_new_cases: continue