def iter_all_changes(self, start_from=None):
if not self.domains:
return
def data_function(**view_kwargs):
return self.couch_db.view('by_domain_doc_type_date/view',
**view_kwargs)
keys = []
for domain in self.domains:
for doc_type in self.doc_types:
keys.append([domain, doc_type])
args_provider = MultiKeyViewArgsProvider(keys,
include_docs=True,
chunk_size=self.chunk_size)
for row in paginate_function(data_function,
args_provider,
event_handler=self.event_handler):
yield Change(id=row['id'],
sequence_id=None,
document=row.get('doc'),
deleted=False,
document_store=None)
def _get_paginated_iterable(data_function, args_provider, event_handler=None, resumable_key=None):
if resumable_key:
return ResumableFunctionIterator(
resumable_key,
data_function,
args_provider,
lambda x: x.id,
event_handler=event_handler
)
else:
return paginate_function(data_function, args_provider, event_handler=event_handler)
def _get_paginated_iterable(data_function,
args_provider,
event_handler=None,
resumable_key=None):
if resumable_key:
return ResumableFunctionIterator(resumable_key,
data_function,
args_provider,
lambda x: x.id,
event_handler=event_handler)
else:
return paginate_function(data_function,
args_provider,
event_handler=event_handler)
def _iter_form_id_chunks(self):
kwargs = []
for domain in self.domains:
for doc_type in doc_type_to_state:
kwargs.append({'domain': domain, 'type_': doc_type})
args_provider = ArgsListProvider(kwargs)
data_function = FormAccessorSQL.get_form_ids_in_domain_by_type
chunk = []
for form_id in paginate_function(data_function, args_provider):
chunk.append(form_id)
if len(chunk) >= self.chunk_size:
yield chunk
chunk = []
if chunk:
yield chunk
def _iter_form_id_chunks(self):
kwargs = []
for domain in self.domains:
for doc_type in doc_type_to_state:
kwargs.append({'domain': domain, 'type_': doc_type})
args_provider = ArgsListProvider(kwargs)
data_function = FormAccessorSQL.get_form_ids_in_domain_by_type
chunk = []
for form_id in paginate_function(data_function, args_provider):
chunk.append(form_id)
if len(chunk) >= self.chunk_size:
yield chunk
chunk = []
if chunk:
yield chunk
def _iter_form_id_chunks(self):
kwargs = []
for domain in self.domains:
for doc_type in XFormInstance.DOC_TYPE_TO_STATE:
kwargs.append({'domain': domain, 'doc_type': doc_type})
args_provider = ArgsListProvider(kwargs)
data_function = XFormInstance.objects.get_form_ids_in_domain
chunk = []
for form_id in paginate_function(data_function, args_provider):
chunk.append(form_id)
if len(chunk) >= self.chunk_size:
yield chunk
chunk = []
if chunk:
yield chunk
def paginate_view(db,
view_name,
chunk_size,
event_handler=PaginationEventHandler(),
**view_kwargs):
"""
intended as a more performant drop-in replacement for
iter(db.view(view_name, **kwargs))
intended specifically to be more performant when dealing with
large numbers of rows
Note: If the contents of the couch view do not change over the duration of
the paginate_view call, this is guaranteed to have the same results
as a direct view call. If the view is updated, however,
paginate_views may skip docs that were added/updated during this period
or may include docs that were removed/updated during this period.
For this reason, it's best to use this with views that update infrequently
or that are sorted by date modified and/or add-only,
or when exactness is not a strict requirement
chunk_size is how many docs to fetch per request to couchdb
"""
if view_kwargs.get('reduce', True):
raise ValueError('paginate_view must be called with reduce=False')
if 'limit' in view_kwargs:
raise ValueError('paginate_view cannot be called with limit')
if 'skip' in view_kwargs:
raise ValueError('paginate_view cannot be called with skip')
view_kwargs['limit'] = chunk_size
def call_view(**view_kwargs):
return db.view(view_name, **view_kwargs)
args_provider = PaginatedViewArgsProvider(view_kwargs)
for result in paginate_function(call_view, args_provider, event_handler):
yield result
def iter_all_changes(self, start_from=None):
if not self.domains:
return
def data_function(**view_kwargs):
return self.couch_db.view('by_domain_doc_type_date/view', **view_kwargs)
keys = []
for domain in self.domains:
for doc_type in self.doc_types:
keys.append([domain, doc_type])
args_provider = MultiKeyViewArgsProvider(keys, include_docs=True, chunk_size=self.chunk_size)
for row in paginate_function(data_function, args_provider, event_handler=self.event_handler):
yield Change(
id=row['id'],
sequence_id=None,
document=row.get('doc'),
deleted=False,
document_store=None
)
def paginate_view(db, view_name, chunk_size, event_handler=PaginationEventHandler(), **view_kwargs):
"""
intended as a more performant drop-in replacement for
iter(db.view(view_name, **kwargs))
intended specifically to be more performant when dealing with
large numbers of rows
Note: If the contents of the couch view do not change over the duration of
the paginate_view call, this is guaranteed to have the same results
as a direct view call. If the view is updated, however,
paginate_views may skip docs that were added/updated during this period
or may include docs that were removed/updated during this period.
For this reason, it's best to use this with views that update infrequently
or that are sorted by date modified and/or add-only,
or when exactness is not a strict requirement
chunk_size is how many docs to fetch per request to couchdb
"""
if view_kwargs.get('reduce', True):
raise ValueError('paginate_view must be called with reduce=False')
if 'limit' in view_kwargs:
raise ValueError('paginate_view cannot be called with limit')
if 'skip' in view_kwargs:
raise ValueError('paginate_view cannot be called with skip')
view_kwargs['limit'] = chunk_size
def call_view(**view_kwargs):
return db.view(view_name, **view_kwargs)
args_provider = PaginatedViewArgsProvider(view_kwargs)
for result in paginate_function(call_view, args_provider, event_handler):
yield result
