def get_call_center_cases(domain_name, case_type, user=None):
base_key = ["open type owner", domain_name, case_type]
if user:
keys = [
base_key + [owner_id]
for owner_id in user.get_owner_ids()
]
else:
keys = [base_key]
all_cases = []
for key in keys:
rows = paginate_view(
CommCareCase.get_db(),
'case/all_cases',
chunk_size=10,
startkey=key,
endkey=key + [{}],
reduce=False,
include_docs=True
)
for row in rows:
hq_user_id = row['doc'].get('hq_user_id', None)
if hq_user_id:
all_cases.append(CallCenterCase(
case_id=row['id'],
hq_user_id=hq_user_id
))
return all_cases
def _get_data_for_couch_backend(run_config):
if run_config.case_type:
raise CommandError(
'Case type argument is not supported for couch domains!')
domain = run_config.domain
start_time = datetime.utcnow()
chunk_size = 1000
chunked_iterator = chunked(
paginate_view(CommCareCase.get_db(),
'cases_by_server_date/by_server_modified_on',
chunk_size=chunk_size,
startkey=[domain],
endkey=[domain, {}],
include_docs=False,
reduce=False), chunk_size)
for chunk in chunked_iterator:
case_ids = [row['id'] for row in chunk]
es_modified_on_by_ids = _get_es_modified_dates(domain, case_ids)
for row in chunk:
case_id, couch_modified_on = row['id'], row['value']
if iso_string_to_datetime(couch_modified_on) > start_time:
# skip cases modified after the script started
continue
es_modified_on = es_modified_on_by_ids.get(case_id)
if not es_modified_on or (es_modified_on != couch_modified_on):
yield (case_id, 'COUCH_TYPE_NOT_SUPPORTED', es_modified_on,
couch_modified_on)
def iter_domains():
for row in paginate_view(Domain.get_db(),
'domain/domains',
100,
reduce=False,
include_docs=False):
yield row['key']
def get_all_docs_with_doc_types(db, doc_types):
for doc_type in doc_types:
results = paginate_view(
db, 'all_docs/by_doc_type',
chunk_size=100, startkey=[doc_type], endkey=[doc_type, {}],
include_docs=True, reduce=False)
for result in results:
yield result['doc']
def iter_all_cases_most_recent_first(chunk_size=100):
return paginate_view(
CommCareCase.get_db(),
'hqadmin/cases_over_time',
reduce=False,
include_docs=True,
descending=True,
chunk_size=chunk_size,
)
def iter_all_forms_most_recent_first(chunk_size=100):
return paginate_view(
XFormInstance.get_db(),
'hqadmin/forms_over_time',
reduce=False,
include_docs=True,
descending=True,
chunk_size=chunk_size,
)
def iter_all_forms_most_recent_first(chunk_size=100):
return paginate_view(
XFormInstance.get_db(),
'hqadmin/forms_over_time',
reduce=False,
include_docs=True,
descending=True,
chunk_size=chunk_size,
)
def iter_all_cases_most_recent_first(chunk_size=100):
return paginate_view(
CommCareCase.get_db(),
'hqadmin/cases_over_time',
reduce=False,
include_docs=True,
descending=True,
chunk_size=chunk_size,
)
def iter_changes(self, start_from=None):
view_kwargs = copy(self._view_kwargs)
view_kwargs['reduce'] = False # required to paginate a view
if start_from is not None:
# todo: should we abstract out how the keys work inside this class?
view_kwargs['startkey'] = start_from
for item in paginate_view(self._couch_db, self._view_name, self._chunk_size, **view_kwargs):
# todo: need to transform to a `Change` object
yield item
def iter_fixture_items_for_data_type(domain, data_type_id):
from corehq.apps.fixtures.models import FixtureDataItem
for row in paginate_view(FixtureDataItem.get_db(),
'fixtures/data_items_by_domain_type',
chunk_size=1000,
startkey=[domain, data_type_id],
endkey=[domain, data_type_id, {}],
reduce=False,
include_docs=True):
yield FixtureDataItem.wrap(row['doc'])
def iter_repeat_records_by_repeater(domain, repeater_id, chunk_size=1000):
from corehq.motech.repeaters.models import RepeatRecord
kwargs = {
'include_docs': True,
'reduce': False,
'descending': True,
}
kwargs.update(_get_startkey_endkey_all_records(domain, repeater_id))
for doc in paginate_view(RepeatRecord.get_db(), 'repeaters/repeat_records',
chunk_size, **kwargs):
yield RepeatRecord.wrap(doc['doc'])
def iter_all_changes(self, start_from=None):
view_kwargs = copy(self._view_kwargs)
view_kwargs['reduce'] = False # required to paginate a view
if start_from is not None:
# todo: should we abstract out how the keys work inside this class?
view_kwargs['startkey'] = start_from
for row in paginate_view(self._couch_db, self._view_name, self._chunk_size, **view_kwargs):
# todo: if include_docs isn't specified then this will make one request to couch per row
# to get the documents. In the future we will likely need to add chunking
yield Change(id=row['id'], sequence_id=None, document=row.get('doc'), deleted=False,
document_store=CouchDocumentStore(self._couch_db))
def get_all_docs_with_doc_types(db, doc_types):
for doc_type in doc_types:
results = paginate_view(
db,
"all_docs/by_doc_type",
chunk_size=100,
startkey=[doc_type],
endkey=[doc_type, {}],
attachments=True,
include_docs=True,
reduce=False,
)
for result in results:
yield result["doc"]
def iterate_repeat_records(due_before, chunk_size=10000, database=None):
from .models import RepeatRecord
json_now = json_format_datetime(due_before)
view_kwargs = {
'reduce': False,
'startkey': [None],
'endkey': [None, json_now, {}],
'include_docs': True
}
for doc in paginate_view(RepeatRecord.get_db(),
'receiverwrapper/repeat_records_by_next_check',
chunk_size, **view_kwargs):
yield RepeatRecord.wrap(doc['doc'])
def iter_all_changes(self, start_from=None):
view_kwargs = copy(self._view_kwargs)
view_kwargs['reduce'] = False # required to paginate a view
if start_from is not None:
# todo: should we abstract out how the keys work inside this class?
view_kwargs['startkey'] = start_from
for row in paginate_view(self._couch_db, self._view_name,
self._chunk_size, **view_kwargs):
# todo: if include_docs isn't specified then this will make one request to couch per row
# to get the documents. In the future we will likely need to add chunking
yield Change(id=row['id'],
sequence_id=None,
document=row.get('doc'),
deleted=False,
document_store=CouchDocumentStore(self._couch_db))
def iter_repeat_records_by_domain(domain, repeater_id=None, state=None, since=None, chunk_size=1000):
from .models import RepeatRecord
kwargs = {
'include_docs': True,
'reduce': False,
'descending': True,
}
kwargs.update(_get_startkey_endkey_all_records(domain, repeater_id, state,
last_checked_after=since))
for doc in paginate_view(
RepeatRecord.get_db(),
'repeaters/repeat_records',
chunk_size,
**kwargs):
yield RepeatRecord.wrap(doc['doc'])
def get_all_docs_with_doc_types(db, doc_types):
"""
doc_types must be a sequence of doc_types
returns doc JSON (not wrapped)
"""
if isinstance(doc_types, basestring):
raise TypeError('get_all_docs_with_doc_types requires doc_types '
'to be a sequence of strings, not {!r}'.format(doc_types))
for doc_type in doc_types:
results = paginate_view(
db, 'all_docs/by_doc_type',
chunk_size=100, startkey=[doc_type], endkey=[doc_type, {}],
include_docs=True, reduce=False)
for result in results:
yield result['doc']
def get_all_docs_with_doc_types(db, doc_types):
"""
doc_types must be a sequence of doc_types
returns doc JSON (not wrapped)
"""
if isinstance(doc_types, str):
raise TypeError('get_all_docs_with_doc_types requires doc_types '
'to be a sequence of strings, not {!r}'.format(doc_types))
for doc_type in doc_types:
results = paginate_view(
db, 'all_docs/by_doc_type',
chunk_size=100, startkey=[doc_type], endkey=[doc_type, {}],
include_docs=True, reduce=False)
for result in results:
yield result['doc']
def iterate_repeat_records(due_before, chunk_size=10000, database=None):
from .models import RepeatRecord
json_now = json_format_datetime(due_before)
view_kwargs = {
'reduce': False,
'startkey': [None],
'endkey': [None, json_now, {}],
'include_docs': True
}
for doc in paginate_view(
RepeatRecord.get_db(),
'repeaters/repeat_records_by_next_check',
chunk_size,
**view_kwargs):
yield RepeatRecord.wrap(doc['doc'])
def iter_repeat_records_by_domain(domain, repeater_id=None, state=None, since=None, chunk_size=1000):
from .models import RepeatRecord
kwargs = {
'include_docs': True,
'reduce': False,
'descending': True,
}
kwargs.update(_get_startkey_endkey_all_records(domain, repeater_id, state,
last_checked_after=since))
for doc in paginate_view(
RepeatRecord.get_db(),
'repeaters/repeat_records',
chunk_size,
**kwargs):
yield RepeatRecord.wrap(doc['doc'])
def _get_couch_case_data(run_config):
for couch_domain in _get_matching_couch_domains(run_config):
iterator = paginate_view(
CommCareCase.get_db(),
'cases_by_server_date/by_server_modified_on',
chunk_size=1000,
startkey=[couch_domain],
endkey=[couch_domain, {}],
include_docs=False,
reduce=False,
)
for row in iterator:
case_id, modified_on = row['id'], iso_string_to_datetime(
row['value'])
if run_config.start_date <= modified_on < run_config.end_date:
yield case_id, 'COUCH_TYPE_NOT_SUPPORTED', modified_on, couch_domain
def iterate_doc_ids_in_domain_by_type(domain, doc_type, chunk_size=10000, database=None):
if not database:
database = get_db_by_doc_type(doc_type)
view_kwargs = {
'reduce': False,
'startkey': [domain, doc_type],
'endkey': [domain, doc_type, {}],
'include_docs': False
}
for doc in paginate_view(
database,
'by_domain_doc_type_date/view',
chunk_size,
**view_kwargs):
yield doc['id']
def _yield_records():
for couch_domain in _get_matching_couch_domains(run_config):
for doc_type in ['XFormArchived', 'XFormInstance']:
iterator = paginate_view(db,
view,
reduce=False,
include_docs=False,
chunk_size=1000,
**get_kwargs(
couch_domain, doc_type))
for row in iterator:
form_id = row['id']
domain, doc_type, received_on = row['key']
received_on = iso_string_to_datetime(received_on)
assert run_config.domain in (domain, ALL_DOMAINS)
yield (form_id, doc_type, 'COUCH_XMLNS_NOT_SUPPORTED',
received_on, domain)
def iterate_repeat_record_ids(due_before, chunk_size=10000):
"""
Yields repeat record ids only.
Use chunk_size to optimize db query. Has no effect on # of items returned.
"""
from .models import RepeatRecord
json_due_before = json_format_datetime(due_before)
view_kwargs = {
'reduce': False,
'startkey': [None],
'endkey': [None, json_due_before, {}],
'include_docs': False
}
for doc in paginate_view(
RepeatRecord.get_db(),
'repeaters/repeat_records_by_next_check',
chunk_size,
**view_kwargs):
yield doc['id']
def iterate_doc_ids_in_domain_by_type(domain,
doc_type,
chunk_size=10000,
database=None,
startkey=None,
startkey_docid=None):
if not database:
database = get_db_by_doc_type(doc_type)
view_kwargs = {
'reduce': False,
'startkey': startkey if startkey else [domain, doc_type],
'endkey': [domain, doc_type, {}],
'include_docs': False
}
if startkey_docid:
view_kwargs.update({'startkey_docid': startkey_docid})
for doc in paginate_view(database, 'by_domain_doc_type_date/view',
chunk_size, **view_kwargs):
yield doc['id']
def paginate_view(self, *args, **kwargs):
if 'chunk_size' not in kwargs:
kwargs['chunk_size'] = self.chunk_size
if 'log_handler' not in kwargs:
kwargs['log_handler'] = PaginateViewLogHandler(self)
return paginate_view(*args, **kwargs)
def paginate_view(self, *args, **kwargs):
if "chunk_size" not in kwargs:
kwargs["chunk_size"] = self.chunk_size
if "log_handler" not in kwargs:
kwargs["log_handler"] = PaginateViewLogHandler(self)
return paginate_view(*args, **kwargs)
def paginate_view(self, *args, **kwargs):
if 'chunk_size' not in kwargs:
kwargs['chunk_size'] = self.chunk_size
if 'log_handler' not in kwargs:
kwargs['log_handler'] = PaginateViewLogHandler(self)
return paginate_view(*args, **kwargs)
