def test_change_provider_empty(self): provider = CouchDomainDocTypeChangeProvider( couch_db=XFormInstance.get_db(), domains=[], doc_types=all_known_formlike_doc_types()) self.assertEqual([], [change for change in provider.iter_all_changes()])
def test_change_provider_empty(self): provider = CouchDomainDocTypeChangeProvider( couch_db=XFormInstance.get_db(), domains=[], doc_types=all_known_formlike_doc_types() ) self.assertEqual([], [change for change in provider.iter_all_changes()])
def test_change_provider(self): provider = CouchDomainDocTypeChangeProvider( couch_db=XFormInstance.get_db(), domains=self.domains, doc_types=all_known_formlike_doc_types()) doc_ids = {change.id for change in provider.iter_all_changes()} self.assertEqual(doc_ids, set(itertools.chain(*self.form_ids.values())))
def test_change_provider(self): provider = CouchDomainDocTypeChangeProvider( couch_db=XFormInstance.get_db(), domains=self.domains, doc_types=all_known_formlike_doc_types() ) doc_ids = {change.id for change in provider.iter_all_changes()} self.assertEqual(doc_ids, set(itertools.chain(*list(self.form_ids.values()))))
def _get_unprocessed_form_iterator(domain): return CouchDomainDocTypeChangeProvider( couch_db=XFormInstance.get_db(), domains=[domain], doc_types=UNPROCESSED_DOC_TYPES, event_handler=MigrationPaginationEventHandler(domain), )
def _get_main_form_iterator(domain): return CouchDomainDocTypeChangeProvider( couch_db=XFormInstance.get_db(), domains=[domain], doc_types=['XFormInstance'], event_handler=MigrationPaginationEventHandler(domain), )
def _get_unprocessed_form_iterator(domain): return CouchDomainDocTypeChangeProvider( couch_db=XFormInstance.get_db(), domains=[domain], doc_types=UNPROCESSED_DOC_TYPES, event_handler=ReindexEventHandler(u'couch to sql migrator ({} unprocessed forms)'.format(domain)), )
def _get_main_form_iterator(domain): return CouchDomainDocTypeChangeProvider( couch_db=XFormInstance.get_db(), domains=[domain], doc_types=['XFormInstance'], event_handler=ReindexEventHandler(u'couch to sql migrator ({})'.format(domain)), )
def _iter_changes(domain, doc_types, **kw): return CouchDomainDocTypeChangeProvider( couch_db=XFormInstance.get_db(), domains=[domain], doc_types=doc_types, event_handler=MigrationPaginationEventHandler(domain), ).iter_all_changes(**kw)
def _get_case_iterator(domain, doc_types=None): doc_types = doc_types or CASE_DOC_TYPES return CouchDomainDocTypeChangeProvider( couch_db=XFormInstance.get_db(), domains=[domain], doc_types=doc_types, event_handler=MigrationPaginationEventHandler(domain), )
def _get_case_iterator(domain, doc_types=None): doc_types = doc_types or CASE_DOC_TYPES return CouchDomainDocTypeChangeProvider( couch_db=XFormInstance.get_db(), domains=[domain], doc_types=doc_types, event_handler=ReindexEventHandler(u'couch to sql migrator ({})'.format(domain)), )
def get_domain_form_change_provider(domains): sql_domains = { domain for domain in domains if should_use_sql_backend(domain) } couch_domains = set(domains) - sql_domains return CompositeChangeProvider([ SqlDomainXFormChangeProvider(sql_domains), CouchDomainDocTypeChangeProvider( couch_db=XFormInstance.get_db(), domains=couch_domains, doc_types=all_known_formlike_doc_types()), ])
def handle(self, *args, **options): if len(args) < 1: raise CommandError('Usage is ptop_reindexer_fluff %s' % self.args) fluff_configs = { config.name: config for config in get_fluff_pillow_configs() } pillow_name = args[0] if pillow_name not in fluff_configs: raise CommandError( 'Unrecognised fluff pillow: "{}". Options are:\n\t{}'.format( pillow_name, '\n\t'.join(fluff_configs))) pillow_getter = get_pillow_by_name(pillow_name, instantiate=False) pillow = pillow_getter(delete_filtered=True) if len(args) == 1: domains = pillow.domains else: domains = args[1:] domains_not_in_pillow = set(domains) - set(pillow.domains) if domains_not_in_pillow: bad_domains = ', '.join(domains_not_in_pillow) available_domains = ', '.join(pillow.domains) raise CommandError( "The following domains aren't for this pillow: {}.\nAvailable domains are: {}" .format(bad_domains, available_domains)) if pillow.kafka_topic in (topics.CASE, topics.FORM): couch_db = couch_config.get_db(None) elif pillow.kafka_topic == topics.COMMCARE_USER: couch_db = couch_config.get_db(settings.NEW_USERS_GROUPS_DB) else: raise CommandError('Reindexer not configured for topic: {}'.format( pillow.kafka_topic)) change_provider = CouchDomainDocTypeChangeProvider( couch_db=couch_db, domains=domains, doc_types=[pillow.doc_type], event_handler=ReindexEventHandler(pillow_name), ) PillowChangeProviderReindexer(pillow, change_provider).reindex()
def _get_unprocessed_form_iterator(domain): return CouchDomainDocTypeChangeProvider( couch_db=XFormInstance.get_db(), domains=[domain], doc_types=[ 'XFormArchived', 'XFormError', 'XFormDeprecated', 'XFormDuplicate', # todo: need to figure out which of these we plan on supporting 'XFormInstance-Deleted', 'HQSubmission', 'SubmissionErrorLog', ], event_handler=ReindexEventHandler( u'couch to sql migrator ({} unprocessed forms)'.format(domain)), )