def get_call_center_cases(domain_name, case_type, user=None): base_key = ["open type owner", domain_name, case_type] if user: keys = [ base_key + [owner_id] for owner_id in user.get_owner_ids() ] else: keys = [base_key] all_cases = [] for key in keys: rows = paginate_view( CommCareCase.get_db(), 'case/all_cases', chunk_size=10, startkey=key, endkey=key + [{}], reduce=False, include_docs=True ) for row in rows: hq_user_id = row['doc'].get('hq_user_id', None) if hq_user_id: all_cases.append(CallCenterCase( case_id=row['id'], hq_user_id=hq_user_id )) return all_cases
def _get_data_for_couch_backend(run_config): if run_config.case_type: raise CommandError( 'Case type argument is not supported for couch domains!') domain = run_config.domain start_time = datetime.utcnow() chunk_size = 1000 chunked_iterator = chunked( paginate_view(CommCareCase.get_db(), 'cases_by_server_date/by_server_modified_on', chunk_size=chunk_size, startkey=[domain], endkey=[domain, {}], include_docs=False, reduce=False), chunk_size) for chunk in chunked_iterator: case_ids = [row['id'] for row in chunk] es_modified_on_by_ids = _get_es_modified_dates(domain, case_ids) for row in chunk: case_id, couch_modified_on = row['id'], row['value'] if iso_string_to_datetime(couch_modified_on) > start_time: # skip cases modified after the script started continue es_modified_on = es_modified_on_by_ids.get(case_id) if not es_modified_on or (es_modified_on != couch_modified_on): yield (case_id, 'COUCH_TYPE_NOT_SUPPORTED', es_modified_on, couch_modified_on)
def iter_domains(): for row in paginate_view(Domain.get_db(), 'domain/domains', 100, reduce=False, include_docs=False): yield row['key']
def get_all_docs_with_doc_types(db, doc_types): for doc_type in doc_types: results = paginate_view( db, 'all_docs/by_doc_type', chunk_size=100, startkey=[doc_type], endkey=[doc_type, {}], include_docs=True, reduce=False) for result in results: yield result['doc']
def iter_all_cases_most_recent_first(chunk_size=100): return paginate_view( CommCareCase.get_db(), 'hqadmin/cases_over_time', reduce=False, include_docs=True, descending=True, chunk_size=chunk_size, )
def iter_all_forms_most_recent_first(chunk_size=100): return paginate_view( XFormInstance.get_db(), 'hqadmin/forms_over_time', reduce=False, include_docs=True, descending=True, chunk_size=chunk_size, )
def iter_changes(self, start_from=None): view_kwargs = copy(self._view_kwargs) view_kwargs['reduce'] = False # required to paginate a view if start_from is not None: # todo: should we abstract out how the keys work inside this class? view_kwargs['startkey'] = start_from for item in paginate_view(self._couch_db, self._view_name, self._chunk_size, **view_kwargs): # todo: need to transform to a `Change` object yield item
def iter_fixture_items_for_data_type(domain, data_type_id): from corehq.apps.fixtures.models import FixtureDataItem for row in paginate_view(FixtureDataItem.get_db(), 'fixtures/data_items_by_domain_type', chunk_size=1000, startkey=[domain, data_type_id], endkey=[domain, data_type_id, {}], reduce=False, include_docs=True): yield FixtureDataItem.wrap(row['doc'])
def iter_repeat_records_by_repeater(domain, repeater_id, chunk_size=1000): from corehq.motech.repeaters.models import RepeatRecord kwargs = { 'include_docs': True, 'reduce': False, 'descending': True, } kwargs.update(_get_startkey_endkey_all_records(domain, repeater_id)) for doc in paginate_view(RepeatRecord.get_db(), 'repeaters/repeat_records', chunk_size, **kwargs): yield RepeatRecord.wrap(doc['doc'])
def iter_all_changes(self, start_from=None): view_kwargs = copy(self._view_kwargs) view_kwargs['reduce'] = False # required to paginate a view if start_from is not None: # todo: should we abstract out how the keys work inside this class? view_kwargs['startkey'] = start_from for row in paginate_view(self._couch_db, self._view_name, self._chunk_size, **view_kwargs): # todo: if include_docs isn't specified then this will make one request to couch per row # to get the documents. In the future we will likely need to add chunking yield Change(id=row['id'], sequence_id=None, document=row.get('doc'), deleted=False, document_store=CouchDocumentStore(self._couch_db))
def get_all_docs_with_doc_types(db, doc_types): for doc_type in doc_types: results = paginate_view( db, "all_docs/by_doc_type", chunk_size=100, startkey=[doc_type], endkey=[doc_type, {}], attachments=True, include_docs=True, reduce=False, ) for result in results: yield result["doc"]
def iterate_repeat_records(due_before, chunk_size=10000, database=None): from .models import RepeatRecord json_now = json_format_datetime(due_before) view_kwargs = { 'reduce': False, 'startkey': [None], 'endkey': [None, json_now, {}], 'include_docs': True } for doc in paginate_view(RepeatRecord.get_db(), 'receiverwrapper/repeat_records_by_next_check', chunk_size, **view_kwargs): yield RepeatRecord.wrap(doc['doc'])
def iter_repeat_records_by_domain(domain, repeater_id=None, state=None, since=None, chunk_size=1000): from .models import RepeatRecord kwargs = { 'include_docs': True, 'reduce': False, 'descending': True, } kwargs.update(_get_startkey_endkey_all_records(domain, repeater_id, state, last_checked_after=since)) for doc in paginate_view( RepeatRecord.get_db(), 'repeaters/repeat_records', chunk_size, **kwargs): yield RepeatRecord.wrap(doc['doc'])
def get_all_docs_with_doc_types(db, doc_types): """ doc_types must be a sequence of doc_types returns doc JSON (not wrapped) """ if isinstance(doc_types, basestring): raise TypeError('get_all_docs_with_doc_types requires doc_types ' 'to be a sequence of strings, not {!r}'.format(doc_types)) for doc_type in doc_types: results = paginate_view( db, 'all_docs/by_doc_type', chunk_size=100, startkey=[doc_type], endkey=[doc_type, {}], include_docs=True, reduce=False) for result in results: yield result['doc']
def get_all_docs_with_doc_types(db, doc_types): """ doc_types must be a sequence of doc_types returns doc JSON (not wrapped) """ if isinstance(doc_types, str): raise TypeError('get_all_docs_with_doc_types requires doc_types ' 'to be a sequence of strings, not {!r}'.format(doc_types)) for doc_type in doc_types: results = paginate_view( db, 'all_docs/by_doc_type', chunk_size=100, startkey=[doc_type], endkey=[doc_type, {}], include_docs=True, reduce=False) for result in results: yield result['doc']
def iterate_repeat_records(due_before, chunk_size=10000, database=None): from .models import RepeatRecord json_now = json_format_datetime(due_before) view_kwargs = { 'reduce': False, 'startkey': [None], 'endkey': [None, json_now, {}], 'include_docs': True } for doc in paginate_view( RepeatRecord.get_db(), 'repeaters/repeat_records_by_next_check', chunk_size, **view_kwargs): yield RepeatRecord.wrap(doc['doc'])
def _get_couch_case_data(run_config): for couch_domain in _get_matching_couch_domains(run_config): iterator = paginate_view( CommCareCase.get_db(), 'cases_by_server_date/by_server_modified_on', chunk_size=1000, startkey=[couch_domain], endkey=[couch_domain, {}], include_docs=False, reduce=False, ) for row in iterator: case_id, modified_on = row['id'], iso_string_to_datetime( row['value']) if run_config.start_date <= modified_on < run_config.end_date: yield case_id, 'COUCH_TYPE_NOT_SUPPORTED', modified_on, couch_domain
def iterate_doc_ids_in_domain_by_type(domain, doc_type, chunk_size=10000, database=None): if not database: database = get_db_by_doc_type(doc_type) view_kwargs = { 'reduce': False, 'startkey': [domain, doc_type], 'endkey': [domain, doc_type, {}], 'include_docs': False } for doc in paginate_view( database, 'by_domain_doc_type_date/view', chunk_size, **view_kwargs): yield doc['id']
def _yield_records(): for couch_domain in _get_matching_couch_domains(run_config): for doc_type in ['XFormArchived', 'XFormInstance']: iterator = paginate_view(db, view, reduce=False, include_docs=False, chunk_size=1000, **get_kwargs( couch_domain, doc_type)) for row in iterator: form_id = row['id'] domain, doc_type, received_on = row['key'] received_on = iso_string_to_datetime(received_on) assert run_config.domain in (domain, ALL_DOMAINS) yield (form_id, doc_type, 'COUCH_XMLNS_NOT_SUPPORTED', received_on, domain)
def iterate_repeat_record_ids(due_before, chunk_size=10000): """ Yields repeat record ids only. Use chunk_size to optimize db query. Has no effect on # of items returned. """ from .models import RepeatRecord json_due_before = json_format_datetime(due_before) view_kwargs = { 'reduce': False, 'startkey': [None], 'endkey': [None, json_due_before, {}], 'include_docs': False } for doc in paginate_view( RepeatRecord.get_db(), 'repeaters/repeat_records_by_next_check', chunk_size, **view_kwargs): yield doc['id']
def iterate_doc_ids_in_domain_by_type(domain, doc_type, chunk_size=10000, database=None, startkey=None, startkey_docid=None): if not database: database = get_db_by_doc_type(doc_type) view_kwargs = { 'reduce': False, 'startkey': startkey if startkey else [domain, doc_type], 'endkey': [domain, doc_type, {}], 'include_docs': False } if startkey_docid: view_kwargs.update({'startkey_docid': startkey_docid}) for doc in paginate_view(database, 'by_domain_doc_type_date/view', chunk_size, **view_kwargs): yield doc['id']
def paginate_view(self, *args, **kwargs): if 'chunk_size' not in kwargs: kwargs['chunk_size'] = self.chunk_size if 'log_handler' not in kwargs: kwargs['log_handler'] = PaginateViewLogHandler(self) return paginate_view(*args, **kwargs)
def paginate_view(self, *args, **kwargs): if "chunk_size" not in kwargs: kwargs["chunk_size"] = self.chunk_size if "log_handler" not in kwargs: kwargs["log_handler"] = PaginateViewLogHandler(self) return paginate_view(*args, **kwargs)