def iter_all_changes(self, start_from=None): if not self.domains: return def data_function(**view_kwargs): return self.couch_db.view('by_domain_doc_type_date/view', **view_kwargs) keys = [] for domain in self.domains: for doc_type in self.doc_types: keys.append([domain, doc_type]) args_provider = MultiKeyViewArgsProvider(keys, include_docs=True, chunk_size=self.chunk_size) for row in paginate_function(data_function, args_provider, event_handler=self.event_handler): yield Change(id=row['id'], sequence_id=None, document=row.get('doc'), deleted=False, document_store=None)
def _get_paginated_iterable(data_function, args_provider, event_handler=None, resumable_key=None): if resumable_key: return ResumableFunctionIterator( resumable_key, data_function, args_provider, lambda x: x.id, event_handler=event_handler ) else: return paginate_function(data_function, args_provider, event_handler=event_handler)
def _get_paginated_iterable(data_function, args_provider, event_handler=None, resumable_key=None): if resumable_key: return ResumableFunctionIterator(resumable_key, data_function, args_provider, lambda x: x.id, event_handler=event_handler) else: return paginate_function(data_function, args_provider, event_handler=event_handler)
def _iter_form_id_chunks(self): kwargs = [] for domain in self.domains: for doc_type in doc_type_to_state: kwargs.append({'domain': domain, 'type_': doc_type}) args_provider = ArgsListProvider(kwargs) data_function = FormAccessorSQL.get_form_ids_in_domain_by_type chunk = [] for form_id in paginate_function(data_function, args_provider): chunk.append(form_id) if len(chunk) >= self.chunk_size: yield chunk chunk = [] if chunk: yield chunk
def _iter_form_id_chunks(self): kwargs = [] for domain in self.domains: for doc_type in doc_type_to_state: kwargs.append({'domain': domain, 'type_': doc_type}) args_provider = ArgsListProvider(kwargs) data_function = FormAccessorSQL.get_form_ids_in_domain_by_type chunk = [] for form_id in paginate_function(data_function, args_provider): chunk.append(form_id) if len(chunk) >= self.chunk_size: yield chunk chunk = [] if chunk: yield chunk
def _iter_form_id_chunks(self): kwargs = [] for domain in self.domains: for doc_type in XFormInstance.DOC_TYPE_TO_STATE: kwargs.append({'domain': domain, 'doc_type': doc_type}) args_provider = ArgsListProvider(kwargs) data_function = XFormInstance.objects.get_form_ids_in_domain chunk = [] for form_id in paginate_function(data_function, args_provider): chunk.append(form_id) if len(chunk) >= self.chunk_size: yield chunk chunk = [] if chunk: yield chunk
def paginate_view(db, view_name, chunk_size, event_handler=PaginationEventHandler(), **view_kwargs): """ intended as a more performant drop-in replacement for iter(db.view(view_name, **kwargs)) intended specifically to be more performant when dealing with large numbers of rows Note: If the contents of the couch view do not change over the duration of the paginate_view call, this is guaranteed to have the same results as a direct view call. If the view is updated, however, paginate_views may skip docs that were added/updated during this period or may include docs that were removed/updated during this period. For this reason, it's best to use this with views that update infrequently or that are sorted by date modified and/or add-only, or when exactness is not a strict requirement chunk_size is how many docs to fetch per request to couchdb """ if view_kwargs.get('reduce', True): raise ValueError('paginate_view must be called with reduce=False') if 'limit' in view_kwargs: raise ValueError('paginate_view cannot be called with limit') if 'skip' in view_kwargs: raise ValueError('paginate_view cannot be called with skip') view_kwargs['limit'] = chunk_size def call_view(**view_kwargs): return db.view(view_name, **view_kwargs) args_provider = PaginatedViewArgsProvider(view_kwargs) for result in paginate_function(call_view, args_provider, event_handler): yield result
def iter_all_changes(self, start_from=None): if not self.domains: return def data_function(**view_kwargs): return self.couch_db.view('by_domain_doc_type_date/view', **view_kwargs) keys = [] for domain in self.domains: for doc_type in self.doc_types: keys.append([domain, doc_type]) args_provider = MultiKeyViewArgsProvider(keys, include_docs=True, chunk_size=self.chunk_size) for row in paginate_function(data_function, args_provider, event_handler=self.event_handler): yield Change( id=row['id'], sequence_id=None, document=row.get('doc'), deleted=False, document_store=None )
def paginate_view(db, view_name, chunk_size, event_handler=PaginationEventHandler(), **view_kwargs): """ intended as a more performant drop-in replacement for iter(db.view(view_name, **kwargs)) intended specifically to be more performant when dealing with large numbers of rows Note: If the contents of the couch view do not change over the duration of the paginate_view call, this is guaranteed to have the same results as a direct view call. If the view is updated, however, paginate_views may skip docs that were added/updated during this period or may include docs that were removed/updated during this period. For this reason, it's best to use this with views that update infrequently or that are sorted by date modified and/or add-only, or when exactness is not a strict requirement chunk_size is how many docs to fetch per request to couchdb """ if view_kwargs.get('reduce', True): raise ValueError('paginate_view must be called with reduce=False') if 'limit' in view_kwargs: raise ValueError('paginate_view cannot be called with limit') if 'skip' in view_kwargs: raise ValueError('paginate_view cannot be called with skip') view_kwargs['limit'] = chunk_size def call_view(**view_kwargs): return db.view(view_name, **view_kwargs) args_provider = PaginatedViewArgsProvider(view_kwargs) for result in paginate_function(call_view, args_provider, event_handler): yield result